projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
bnxt_en: Enable NPAR (NIC Partitioning) Support.
[deliverable/linux.git] / drivers / net / ethernet / broadcom / bnxt / bnxt.c
1 /* Broadcom NetXtreme-C/E network driver.
2 *
3 * Copyright (c) 2014-2016 Broadcom Corporation
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation.
8 */
9
10 #include <linux/module.h>
11
12 #include <linux/stringify.h>
13 #include <linux/kernel.h>
14 #include <linux/timer.h>
15 #include <linux/errno.h>
16 #include <linux/ioport.h>
17 #include <linux/slab.h>
18 #include <linux/vmalloc.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <linux/netdevice.h>
22 #include <linux/etherdevice.h>
23 #include <linux/skbuff.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/bitops.h>
26 #include <linux/io.h>
27 #include <linux/irq.h>
28 #include <linux/delay.h>
29 #include <asm/byteorder.h>
30 #include <asm/page.h>
31 #include <linux/time.h>
32 #include <linux/mii.h>
33 #include <linux/if.h>
34 #include <linux/if_vlan.h>
35 #include <net/ip.h>
36 #include <net/tcp.h>
37 #include <net/udp.h>
38 #include <net/checksum.h>
39 #include <net/ip6_checksum.h>
40 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
41 #include <net/vxlan.h>
42 #endif
43 #ifdef CONFIG_NET_RX_BUSY_POLL
44 #include <net/busy_poll.h>
45 #endif
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53
54 #include "bnxt_hsi.h"
55 #include "bnxt.h"
56 #include "bnxt_sriov.h"
57 #include "bnxt_ethtool.h"
58
59 #define BNXT_TX_TIMEOUT (5 * HZ)
60
61 static const char version[] =
62 "Broadcom NetXtreme-C/E driver " DRV_MODULE_NAME " v" DRV_MODULE_VERSION "\n";
63
64 MODULE_LICENSE("GPL");
65 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
66 MODULE_VERSION(DRV_MODULE_VERSION);
67
68 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
69 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
70 #define BNXT_RX_COPY_THRESH 256
71
72 #define BNXT_TX_PUSH_THRESH 164
73
74 enum board_idx {
75 BCM57301,
76 BCM57302,
77 BCM57304,
78 BCM57402,
79 BCM57404,
80 BCM57406,
81 BCM57314,
82 BCM57304_VF,
83 BCM57404_VF,
84 };
85
86 /* indexed by enum above */
87 static const struct {
88 char *name;
89 } board_info[] = {
90 { "Broadcom BCM57301 NetXtreme-C Single-port 10Gb Ethernet" },
91 { "Broadcom BCM57302 NetXtreme-C Dual-port 10Gb/25Gb Ethernet" },
92 { "Broadcom BCM57304 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
93 { "Broadcom BCM57402 NetXtreme-E Dual-port 10Gb Ethernet" },
94 { "Broadcom BCM57404 NetXtreme-E Dual-port 10Gb/25Gb Ethernet" },
95 { "Broadcom BCM57406 NetXtreme-E Dual-port 10GBase-T Ethernet" },
96 { "Broadcom BCM57314 NetXtreme-C Dual-port 10Gb/25Gb/40Gb/50Gb Ethernet" },
97 { "Broadcom BCM57304 NetXtreme-C Ethernet Virtual Function" },
98 { "Broadcom BCM57404 NetXtreme-E Ethernet Virtual Function" },
99 };
100
101 static const struct pci_device_id bnxt_pci_tbl[] = {
102 { PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
103 { PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
104 { PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
105 { PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
106 { PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
107 { PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
108 { PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
109 #ifdef CONFIG_BNXT_SRIOV
110 { PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = BCM57304_VF },
111 { PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = BCM57404_VF },
112 #endif
113 { 0 }
114 };
115
116 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
117
118 static const u16 bnxt_vf_req_snif[] = {
119 HWRM_FUNC_CFG,
120 HWRM_PORT_PHY_QCFG,
121 HWRM_CFA_L2_FILTER_ALLOC,
122 };
123
124 static const u16 bnxt_async_events_arr[] = {
125 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
126 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
127 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
128 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
129 HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
130 };
131
132 static bool bnxt_vf_pciid(enum board_idx idx)
133 {
134 return (idx == BCM57304_VF || idx == BCM57404_VF);
135 }
136
137 #define DB_CP_REARM_FLAGS (DB_KEY_CP | DB_IDX_VALID)
138 #define DB_CP_FLAGS (DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
139 #define DB_CP_IRQ_DIS_FLAGS (DB_KEY_CP | DB_IRQ_DIS)
140
141 #define BNXT_CP_DB_REARM(db, raw_cons) \
142 writel(DB_CP_REARM_FLAGS | RING_CMP(raw_cons), db)
143
144 #define BNXT_CP_DB(db, raw_cons) \
145 writel(DB_CP_FLAGS | RING_CMP(raw_cons), db)
146
147 #define BNXT_CP_DB_IRQ_DIS(db) \
148 writel(DB_CP_IRQ_DIS_FLAGS, db)
149
150 static inline u32 bnxt_tx_avail(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
151 {
152 /* Tell compiler to fetch tx indices from memory. */
153 barrier();
154
155 return bp->tx_ring_size -
156 ((txr->tx_prod - txr->tx_cons) & bp->tx_ring_mask);
157 }
158
159 static const u16 bnxt_lhint_arr[] = {
160 TX_BD_FLAGS_LHINT_512_AND_SMALLER,
161 TX_BD_FLAGS_LHINT_512_TO_1023,
162 TX_BD_FLAGS_LHINT_1024_TO_2047,
163 TX_BD_FLAGS_LHINT_1024_TO_2047,
164 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
165 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
166 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
167 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
168 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
169 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
170 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
171 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
172 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
173 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
174 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
175 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
176 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
177 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
178 TX_BD_FLAGS_LHINT_2048_AND_LARGER,
179 };
180
181 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
182 {
183 struct bnxt *bp = netdev_priv(dev);
184 struct tx_bd *txbd;
185 struct tx_bd_ext *txbd1;
186 struct netdev_queue *txq;
187 int i;
188 dma_addr_t mapping;
189 unsigned int length, pad = 0;
190 u32 len, free_size, vlan_tag_flags, cfa_action, flags;
191 u16 prod, last_frag;
192 struct pci_dev *pdev = bp->pdev;
193 struct bnxt_tx_ring_info *txr;
194 struct bnxt_sw_tx_bd *tx_buf;
195
196 i = skb_get_queue_mapping(skb);
197 if (unlikely(i >= bp->tx_nr_rings)) {
198 dev_kfree_skb_any(skb);
199 return NETDEV_TX_OK;
200 }
201
202 txr = &bp->tx_ring[i];
203 txq = netdev_get_tx_queue(dev, i);
204 prod = txr->tx_prod;
205
206 free_size = bnxt_tx_avail(bp, txr);
207 if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
208 netif_tx_stop_queue(txq);
209 return NETDEV_TX_BUSY;
210 }
211
212 length = skb->len;
213 len = skb_headlen(skb);
214 last_frag = skb_shinfo(skb)->nr_frags;
215
216 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
217
218 txbd->tx_bd_opaque = prod;
219
220 tx_buf = &txr->tx_buf_ring[prod];
221 tx_buf->skb = skb;
222 tx_buf->nr_frags = last_frag;
223
224 vlan_tag_flags = 0;
225 cfa_action = 0;
226 if (skb_vlan_tag_present(skb)) {
227 vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
228 skb_vlan_tag_get(skb);
229 /* Currently supports 8021Q, 8021AD vlan offloads
230 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
231 */
232 if (skb->vlan_proto == htons(ETH_P_8021Q))
233 vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
234 }
235
236 if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
237 struct tx_push_buffer *tx_push_buf = txr->tx_push;
238 struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
239 struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
240 void *pdata = tx_push_buf->data;
241 u64 *end;
242 int j, push_len;
243
244 /* Set COAL_NOW to be ready quickly for the next push */
245 tx_push->tx_bd_len_flags_type =
246 cpu_to_le32((length << TX_BD_LEN_SHIFT) |
247 TX_BD_TYPE_LONG_TX_BD |
248 TX_BD_FLAGS_LHINT_512_AND_SMALLER |
249 TX_BD_FLAGS_COAL_NOW |
250 TX_BD_FLAGS_PACKET_END |
251 (2 << TX_BD_FLAGS_BD_CNT_SHIFT));
252
253 if (skb->ip_summed == CHECKSUM_PARTIAL)
254 tx_push1->tx_bd_hsize_lflags =
255 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
256 else
257 tx_push1->tx_bd_hsize_lflags = 0;
258
259 tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
260 tx_push1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
261
262 end = pdata + length;
263 end = PTR_ALIGN(end, 8) - 1;
264 *end = 0;
265
266 skb_copy_from_linear_data(skb, pdata, len);
267 pdata += len;
268 for (j = 0; j < last_frag; j++) {
269 skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
270 void *fptr;
271
272 fptr = skb_frag_address_safe(frag);
273 if (!fptr)
274 goto normal_tx;
275
276 memcpy(pdata, fptr, skb_frag_size(frag));
277 pdata += skb_frag_size(frag);
278 }
279
280 txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
281 txbd->tx_bd_haddr = txr->data_mapping;
282 prod = NEXT_TX(prod);
283 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
284 memcpy(txbd, tx_push1, sizeof(*txbd));
285 prod = NEXT_TX(prod);
286 tx_push->doorbell =
287 cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
288 txr->tx_prod = prod;
289
290 tx_buf->is_push = 1;
291 netdev_tx_sent_queue(txq, skb->len);
292 wmb(); /* Sync is_push and byte queue before pushing data */
293
294 push_len = (length + sizeof(*tx_push) + 7) / 8;
295 if (push_len > 16) {
296 __iowrite64_copy(txr->tx_doorbell, tx_push_buf, 16);
297 __iowrite64_copy(txr->tx_doorbell + 4, tx_push_buf + 1,
298 push_len - 16);
299 } else {
300 __iowrite64_copy(txr->tx_doorbell, tx_push_buf,
301 push_len);
302 }
303
304 goto tx_done;
305 }
306
307 normal_tx:
308 if (length < BNXT_MIN_PKT_SIZE) {
309 pad = BNXT_MIN_PKT_SIZE - length;
310 if (skb_pad(skb, pad)) {
311 /* SKB already freed. */
312 tx_buf->skb = NULL;
313 return NETDEV_TX_OK;
314 }
315 length = BNXT_MIN_PKT_SIZE;
316 }
317
318 mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
319
320 if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
321 dev_kfree_skb_any(skb);
322 tx_buf->skb = NULL;
323 return NETDEV_TX_OK;
324 }
325
326 dma_unmap_addr_set(tx_buf, mapping, mapping);
327 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
328 ((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
329
330 txbd->tx_bd_haddr = cpu_to_le64(mapping);
331
332 prod = NEXT_TX(prod);
333 txbd1 = (struct tx_bd_ext *)
334 &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
335
336 txbd1->tx_bd_hsize_lflags = 0;
337 if (skb_is_gso(skb)) {
338 u32 hdr_len;
339
340 if (skb->encapsulation)
341 hdr_len = skb_inner_network_offset(skb) +
342 skb_inner_network_header_len(skb) +
343 inner_tcp_hdrlen(skb);
344 else
345 hdr_len = skb_transport_offset(skb) +
346 tcp_hdrlen(skb);
347
348 txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
349 TX_BD_FLAGS_T_IPID |
350 (hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
351 length = skb_shinfo(skb)->gso_size;
352 txbd1->tx_bd_mss = cpu_to_le32(length);
353 length += hdr_len;
354 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
355 txbd1->tx_bd_hsize_lflags =
356 cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
357 txbd1->tx_bd_mss = 0;
358 }
359
360 length >>= 9;
361 flags |= bnxt_lhint_arr[length];
362 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
363
364 txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
365 txbd1->tx_bd_cfa_action = cpu_to_le32(cfa_action);
366 for (i = 0; i < last_frag; i++) {
367 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
368
369 prod = NEXT_TX(prod);
370 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
371
372 len = skb_frag_size(frag);
373 mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
374 DMA_TO_DEVICE);
375
376 if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
377 goto tx_dma_error;
378
379 tx_buf = &txr->tx_buf_ring[prod];
380 dma_unmap_addr_set(tx_buf, mapping, mapping);
381
382 txbd->tx_bd_haddr = cpu_to_le64(mapping);
383
384 flags = len << TX_BD_LEN_SHIFT;
385 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
386 }
387
388 flags &= ~TX_BD_LEN;
389 txbd->tx_bd_len_flags_type =
390 cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
391 TX_BD_FLAGS_PACKET_END);
392
393 netdev_tx_sent_queue(txq, skb->len);
394
395 /* Sync BD data before updating doorbell */
396 wmb();
397
398 prod = NEXT_TX(prod);
399 txr->tx_prod = prod;
400
401 writel(DB_KEY_TX | prod, txr->tx_doorbell);
402 writel(DB_KEY_TX | prod, txr->tx_doorbell);
403
404 tx_done:
405
406 mmiowb();
407
408 if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
409 netif_tx_stop_queue(txq);
410
411 /* netif_tx_stop_queue() must be done before checking
412 * tx index in bnxt_tx_avail() below, because in
413 * bnxt_tx_int(), we update tx index before checking for
414 * netif_tx_queue_stopped().
415 */
416 smp_mb();
417 if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
418 netif_tx_wake_queue(txq);
419 }
420 return NETDEV_TX_OK;
421
422 tx_dma_error:
423 last_frag = i;
424
425 /* start back at beginning and unmap skb */
426 prod = txr->tx_prod;
427 tx_buf = &txr->tx_buf_ring[prod];
428 tx_buf->skb = NULL;
429 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
430 skb_headlen(skb), PCI_DMA_TODEVICE);
431 prod = NEXT_TX(prod);
432
433 /* unmap remaining mapped pages */
434 for (i = 0; i < last_frag; i++) {
435 prod = NEXT_TX(prod);
436 tx_buf = &txr->tx_buf_ring[prod];
437 dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
438 skb_frag_size(&skb_shinfo(skb)->frags[i]),
439 PCI_DMA_TODEVICE);
440 }
441
442 dev_kfree_skb_any(skb);
443 return NETDEV_TX_OK;
444 }
445
446 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
447 {
448 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
449 int index = txr - &bp->tx_ring[0];
450 struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, index);
451 u16 cons = txr->tx_cons;
452 struct pci_dev *pdev = bp->pdev;
453 int i;
454 unsigned int tx_bytes = 0;
455
456 for (i = 0; i < nr_pkts; i++) {
457 struct bnxt_sw_tx_bd *tx_buf;
458 struct sk_buff *skb;
459 int j, last;
460
461 tx_buf = &txr->tx_buf_ring[cons];
462 cons = NEXT_TX(cons);
463 skb = tx_buf->skb;
464 tx_buf->skb = NULL;
465
466 if (tx_buf->is_push) {
467 tx_buf->is_push = 0;
468 goto next_tx_int;
469 }
470
471 dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
472 skb_headlen(skb), PCI_DMA_TODEVICE);
473 last = tx_buf->nr_frags;
474
475 for (j = 0; j < last; j++) {
476 cons = NEXT_TX(cons);
477 tx_buf = &txr->tx_buf_ring[cons];
478 dma_unmap_page(
479 &pdev->dev,
480 dma_unmap_addr(tx_buf, mapping),
481 skb_frag_size(&skb_shinfo(skb)->frags[j]),
482 PCI_DMA_TODEVICE);
483 }
484
485 next_tx_int:
486 cons = NEXT_TX(cons);
487
488 tx_bytes += skb->len;
489 dev_kfree_skb_any(skb);
490 }
491
492 netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
493 txr->tx_cons = cons;
494
495 /* Need to make the tx_cons update visible to bnxt_start_xmit()
496 * before checking for netif_tx_queue_stopped(). Without the
497 * memory barrier, there is a small possibility that bnxt_start_xmit()
498 * will miss it and cause the queue to be stopped forever.
499 */
500 smp_mb();
501
502 if (unlikely(netif_tx_queue_stopped(txq)) &&
503 (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
504 __netif_tx_lock(txq, smp_processor_id());
505 if (netif_tx_queue_stopped(txq) &&
506 bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
507 txr->dev_state != BNXT_DEV_STATE_CLOSING)
508 netif_tx_wake_queue(txq);
509 __netif_tx_unlock(txq);
510 }
511 }
512
513 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
514 gfp_t gfp)
515 {
516 u8 *data;
517 struct pci_dev *pdev = bp->pdev;
518
519 data = kmalloc(bp->rx_buf_size, gfp);
520 if (!data)
521 return NULL;
522
523 *mapping = dma_map_single(&pdev->dev, data + BNXT_RX_DMA_OFFSET,
524 bp->rx_buf_use_size, PCI_DMA_FROMDEVICE);
525
526 if (dma_mapping_error(&pdev->dev, *mapping)) {
527 kfree(data);
528 data = NULL;
529 }
530 return data;
531 }
532
533 static inline int bnxt_alloc_rx_data(struct bnxt *bp,
534 struct bnxt_rx_ring_info *rxr,
535 u16 prod, gfp_t gfp)
536 {
537 struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
538 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
539 u8 *data;
540 dma_addr_t mapping;
541
542 data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
543 if (!data)
544 return -ENOMEM;
545
546 rx_buf->data = data;
547 dma_unmap_addr_set(rx_buf, mapping, mapping);
548
549 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
550
551 return 0;
552 }
553
554 static void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons,
555 u8 *data)
556 {
557 u16 prod = rxr->rx_prod;
558 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
559 struct rx_bd *cons_bd, *prod_bd;
560
561 prod_rx_buf = &rxr->rx_buf_ring[prod];
562 cons_rx_buf = &rxr->rx_buf_ring[cons];
563
564 prod_rx_buf->data = data;
565
566 dma_unmap_addr_set(prod_rx_buf, mapping,
567 dma_unmap_addr(cons_rx_buf, mapping));
568
569 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
570 cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
571
572 prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
573 }
574
575 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
576 {
577 u16 next, max = rxr->rx_agg_bmap_size;
578
579 next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
580 if (next >= max)
581 next = find_first_zero_bit(rxr->rx_agg_bmap, max);
582 return next;
583 }
584
585 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
586 struct bnxt_rx_ring_info *rxr,
587 u16 prod, gfp_t gfp)
588 {
589 struct rx_bd *rxbd =
590 &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
591 struct bnxt_sw_rx_agg_bd *rx_agg_buf;
592 struct pci_dev *pdev = bp->pdev;
593 struct page *page;
594 dma_addr_t mapping;
595 u16 sw_prod = rxr->rx_sw_agg_prod;
596 unsigned int offset = 0;
597
598 if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
599 page = rxr->rx_page;
600 if (!page) {
601 page = alloc_page(gfp);
602 if (!page)
603 return -ENOMEM;
604 rxr->rx_page = page;
605 rxr->rx_page_offset = 0;
606 }
607 offset = rxr->rx_page_offset;
608 rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
609 if (rxr->rx_page_offset == PAGE_SIZE)
610 rxr->rx_page = NULL;
611 else
612 get_page(page);
613 } else {
614 page = alloc_page(gfp);
615 if (!page)
616 return -ENOMEM;
617 }
618
619 mapping = dma_map_page(&pdev->dev, page, offset, BNXT_RX_PAGE_SIZE,
620 PCI_DMA_FROMDEVICE);
621 if (dma_mapping_error(&pdev->dev, mapping)) {
622 __free_page(page);
623 return -EIO;
624 }
625
626 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
627 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
628
629 __set_bit(sw_prod, rxr->rx_agg_bmap);
630 rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
631 rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
632
633 rx_agg_buf->page = page;
634 rx_agg_buf->offset = offset;
635 rx_agg_buf->mapping = mapping;
636 rxbd->rx_bd_haddr = cpu_to_le64(mapping);
637 rxbd->rx_bd_opaque = sw_prod;
638 return 0;
639 }
640
641 static void bnxt_reuse_rx_agg_bufs(struct bnxt_napi *bnapi, u16 cp_cons,
642 u32 agg_bufs)
643 {
644 struct bnxt *bp = bnapi->bp;
645 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
646 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
647 u16 prod = rxr->rx_agg_prod;
648 u16 sw_prod = rxr->rx_sw_agg_prod;
649 u32 i;
650
651 for (i = 0; i < agg_bufs; i++) {
652 u16 cons;
653 struct rx_agg_cmp *agg;
654 struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
655 struct rx_bd *prod_bd;
656 struct page *page;
657
658 agg = (struct rx_agg_cmp *)
659 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
660 cons = agg->rx_agg_cmp_opaque;
661 __clear_bit(cons, rxr->rx_agg_bmap);
662
663 if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
664 sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
665
666 __set_bit(sw_prod, rxr->rx_agg_bmap);
667 prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
668 cons_rx_buf = &rxr->rx_agg_ring[cons];
669
670 /* It is possible for sw_prod to be equal to cons, so
671 * set cons_rx_buf->page to NULL first.
672 */
673 page = cons_rx_buf->page;
674 cons_rx_buf->page = NULL;
675 prod_rx_buf->page = page;
676 prod_rx_buf->offset = cons_rx_buf->offset;
677
678 prod_rx_buf->mapping = cons_rx_buf->mapping;
679
680 prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
681
682 prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
683 prod_bd->rx_bd_opaque = sw_prod;
684
685 prod = NEXT_RX_AGG(prod);
686 sw_prod = NEXT_RX_AGG(sw_prod);
687 cp_cons = NEXT_CMP(cp_cons);
688 }
689 rxr->rx_agg_prod = prod;
690 rxr->rx_sw_agg_prod = sw_prod;
691 }
692
693 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
694 struct bnxt_rx_ring_info *rxr, u16 cons,
695 u16 prod, u8 *data, dma_addr_t dma_addr,
696 unsigned int len)
697 {
698 int err;
699 struct sk_buff *skb;
700
701 err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
702 if (unlikely(err)) {
703 bnxt_reuse_rx_data(rxr, cons, data);
704 return NULL;
705 }
706
707 skb = build_skb(data, 0);
708 dma_unmap_single(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
709 PCI_DMA_FROMDEVICE);
710 if (!skb) {
711 kfree(data);
712 return NULL;
713 }
714
715 skb_reserve(skb, BNXT_RX_OFFSET);
716 skb_put(skb, len);
717 return skb;
718 }
719
720 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp, struct bnxt_napi *bnapi,
721 struct sk_buff *skb, u16 cp_cons,
722 u32 agg_bufs)
723 {
724 struct pci_dev *pdev = bp->pdev;
725 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
726 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
727 u16 prod = rxr->rx_agg_prod;
728 u32 i;
729
730 for (i = 0; i < agg_bufs; i++) {
731 u16 cons, frag_len;
732 struct rx_agg_cmp *agg;
733 struct bnxt_sw_rx_agg_bd *cons_rx_buf;
734 struct page *page;
735 dma_addr_t mapping;
736
737 agg = (struct rx_agg_cmp *)
738 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
739 cons = agg->rx_agg_cmp_opaque;
740 frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
741 RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
742
743 cons_rx_buf = &rxr->rx_agg_ring[cons];
744 skb_fill_page_desc(skb, i, cons_rx_buf->page,
745 cons_rx_buf->offset, frag_len);
746 __clear_bit(cons, rxr->rx_agg_bmap);
747
748 /* It is possible for bnxt_alloc_rx_page() to allocate
749 * a sw_prod index that equals the cons index, so we
750 * need to clear the cons entry now.
751 */
752 mapping = dma_unmap_addr(cons_rx_buf, mapping);
753 page = cons_rx_buf->page;
754 cons_rx_buf->page = NULL;
755
756 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
757 struct skb_shared_info *shinfo;
758 unsigned int nr_frags;
759
760 shinfo = skb_shinfo(skb);
761 nr_frags = --shinfo->nr_frags;
762 __skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
763
764 dev_kfree_skb(skb);
765
766 cons_rx_buf->page = page;
767
768 /* Update prod since possibly some pages have been
769 * allocated already.
770 */
771 rxr->rx_agg_prod = prod;
772 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs - i);
773 return NULL;
774 }
775
776 dma_unmap_page(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
777 PCI_DMA_FROMDEVICE);
778
779 skb->data_len += frag_len;
780 skb->len += frag_len;
781 skb->truesize += PAGE_SIZE;
782
783 prod = NEXT_RX_AGG(prod);
784 cp_cons = NEXT_CMP(cp_cons);
785 }
786 rxr->rx_agg_prod = prod;
787 return skb;
788 }
789
790 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
791 u8 agg_bufs, u32 *raw_cons)
792 {
793 u16 last;
794 struct rx_agg_cmp *agg;
795
796 *raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
797 last = RING_CMP(*raw_cons);
798 agg = (struct rx_agg_cmp *)
799 &cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
800 return RX_AGG_CMP_VALID(agg, *raw_cons);
801 }
802
803 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
804 unsigned int len,
805 dma_addr_t mapping)
806 {
807 struct bnxt *bp = bnapi->bp;
808 struct pci_dev *pdev = bp->pdev;
809 struct sk_buff *skb;
810
811 skb = napi_alloc_skb(&bnapi->napi, len);
812 if (!skb)
813 return NULL;
814
815 dma_sync_single_for_cpu(&pdev->dev, mapping,
816 bp->rx_copy_thresh, PCI_DMA_FROMDEVICE);
817
818 memcpy(skb->data - BNXT_RX_OFFSET, data, len + BNXT_RX_OFFSET);
819
820 dma_sync_single_for_device(&pdev->dev, mapping,
821 bp->rx_copy_thresh,
822 PCI_DMA_FROMDEVICE);
823
824 skb_put(skb, len);
825 return skb;
826 }
827
828 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_napi *bnapi,
829 u32 *raw_cons, void *cmp)
830 {
831 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
832 struct rx_cmp *rxcmp = cmp;
833 u32 tmp_raw_cons = *raw_cons;
834 u8 cmp_type, agg_bufs = 0;
835
836 cmp_type = RX_CMP_TYPE(rxcmp);
837
838 if (cmp_type == CMP_TYPE_RX_L2_CMP) {
839 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
840 RX_CMP_AGG_BUFS) >>
841 RX_CMP_AGG_BUFS_SHIFT;
842 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
843 struct rx_tpa_end_cmp *tpa_end = cmp;
844
845 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
846 RX_TPA_END_CMP_AGG_BUFS) >>
847 RX_TPA_END_CMP_AGG_BUFS_SHIFT;
848 }
849
850 if (agg_bufs) {
851 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
852 return -EBUSY;
853 }
854 *raw_cons = tmp_raw_cons;
855 return 0;
856 }
857
858 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
859 {
860 if (!rxr->bnapi->in_reset) {
861 rxr->bnapi->in_reset = true;
862 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
863 schedule_work(&bp->sp_task);
864 }
865 rxr->rx_next_cons = 0xffff;
866 }
867
868 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
869 struct rx_tpa_start_cmp *tpa_start,
870 struct rx_tpa_start_cmp_ext *tpa_start1)
871 {
872 u8 agg_id = TPA_START_AGG_ID(tpa_start);
873 u16 cons, prod;
874 struct bnxt_tpa_info *tpa_info;
875 struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
876 struct rx_bd *prod_bd;
877 dma_addr_t mapping;
878
879 cons = tpa_start->rx_tpa_start_cmp_opaque;
880 prod = rxr->rx_prod;
881 cons_rx_buf = &rxr->rx_buf_ring[cons];
882 prod_rx_buf = &rxr->rx_buf_ring[prod];
883 tpa_info = &rxr->rx_tpa[agg_id];
884
885 if (unlikely(cons != rxr->rx_next_cons)) {
886 bnxt_sched_reset(bp, rxr);
887 return;
888 }
889
890 prod_rx_buf->data = tpa_info->data;
891
892 mapping = tpa_info->mapping;
893 dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
894
895 prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
896
897 prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
898
899 tpa_info->data = cons_rx_buf->data;
900 cons_rx_buf->data = NULL;
901 tpa_info->mapping = dma_unmap_addr(cons_rx_buf, mapping);
902
903 tpa_info->len =
904 le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
905 RX_TPA_START_CMP_LEN_SHIFT;
906 if (likely(TPA_START_HASH_VALID(tpa_start))) {
907 u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
908
909 tpa_info->hash_type = PKT_HASH_TYPE_L4;
910 tpa_info->gso_type = SKB_GSO_TCPV4;
911 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
912 if (hash_type == 3)
913 tpa_info->gso_type = SKB_GSO_TCPV6;
914 tpa_info->rss_hash =
915 le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
916 } else {
917 tpa_info->hash_type = PKT_HASH_TYPE_NONE;
918 tpa_info->gso_type = 0;
919 if (netif_msg_rx_err(bp))
920 netdev_warn(bp->dev, "TPA packet without valid hash\n");
921 }
922 tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
923 tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
924
925 rxr->rx_prod = NEXT_RX(prod);
926 cons = NEXT_RX(cons);
927 rxr->rx_next_cons = NEXT_RX(cons);
928 cons_rx_buf = &rxr->rx_buf_ring[cons];
929
930 bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
931 rxr->rx_prod = NEXT_RX(rxr->rx_prod);
932 cons_rx_buf->data = NULL;
933 }
934
935 static void bnxt_abort_tpa(struct bnxt *bp, struct bnxt_napi *bnapi,
936 u16 cp_cons, u32 agg_bufs)
937 {
938 if (agg_bufs)
939 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
940 }
941
942 #define BNXT_IPV4_HDR_SIZE (sizeof(struct iphdr) + sizeof(struct tcphdr))
943 #define BNXT_IPV6_HDR_SIZE (sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
944
945 static inline struct sk_buff *bnxt_gro_skb(struct bnxt_tpa_info *tpa_info,
946 struct rx_tpa_end_cmp *tpa_end,
947 struct rx_tpa_end_cmp_ext *tpa_end1,
948 struct sk_buff *skb)
949 {
950 #ifdef CONFIG_INET
951 struct tcphdr *th;
952 int payload_off, tcp_opt_len = 0;
953 int len, nw_off;
954 u16 segs;
955
956 segs = TPA_END_TPA_SEGS(tpa_end);
957 if (segs == 1)
958 return skb;
959
960 NAPI_GRO_CB(skb)->count = segs;
961 skb_shinfo(skb)->gso_size =
962 le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
963 skb_shinfo(skb)->gso_type = tpa_info->gso_type;
964 payload_off = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
965 RX_TPA_END_CMP_PAYLOAD_OFFSET) >>
966 RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT;
967 if (TPA_END_GRO_TS(tpa_end))
968 tcp_opt_len = 12;
969
970 if (tpa_info->gso_type == SKB_GSO_TCPV4) {
971 struct iphdr *iph;
972
973 nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
974 ETH_HLEN;
975 skb_set_network_header(skb, nw_off);
976 iph = ip_hdr(skb);
977 skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
978 len = skb->len - skb_transport_offset(skb);
979 th = tcp_hdr(skb);
980 th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
981 } else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
982 struct ipv6hdr *iph;
983
984 nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
985 ETH_HLEN;
986 skb_set_network_header(skb, nw_off);
987 iph = ipv6_hdr(skb);
988 skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
989 len = skb->len - skb_transport_offset(skb);
990 th = tcp_hdr(skb);
991 th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
992 } else {
993 dev_kfree_skb_any(skb);
994 return NULL;
995 }
996 tcp_gro_complete(skb);
997
998 if (nw_off) { /* tunnel */
999 struct udphdr *uh = NULL;
1000
1001 if (skb->protocol == htons(ETH_P_IP)) {
1002 struct iphdr *iph = (struct iphdr *)skb->data;
1003
1004 if (iph->protocol == IPPROTO_UDP)
1005 uh = (struct udphdr *)(iph + 1);
1006 } else {
1007 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1008
1009 if (iph->nexthdr == IPPROTO_UDP)
1010 uh = (struct udphdr *)(iph + 1);
1011 }
1012 if (uh) {
1013 if (uh->check)
1014 skb_shinfo(skb)->gso_type |=
1015 SKB_GSO_UDP_TUNNEL_CSUM;
1016 else
1017 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1018 }
1019 }
1020 #endif
1021 return skb;
1022 }
1023
1024 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1025 struct bnxt_napi *bnapi,
1026 u32 *raw_cons,
1027 struct rx_tpa_end_cmp *tpa_end,
1028 struct rx_tpa_end_cmp_ext *tpa_end1,
1029 bool *agg_event)
1030 {
1031 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1032 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1033 u8 agg_id = TPA_END_AGG_ID(tpa_end);
1034 u8 *data, agg_bufs;
1035 u16 cp_cons = RING_CMP(*raw_cons);
1036 unsigned int len;
1037 struct bnxt_tpa_info *tpa_info;
1038 dma_addr_t mapping;
1039 struct sk_buff *skb;
1040
1041 if (unlikely(bnapi->in_reset)) {
1042 int rc = bnxt_discard_rx(bp, bnapi, raw_cons, tpa_end);
1043
1044 if (rc < 0)
1045 return ERR_PTR(-EBUSY);
1046 return NULL;
1047 }
1048
1049 tpa_info = &rxr->rx_tpa[agg_id];
1050 data = tpa_info->data;
1051 prefetch(data);
1052 len = tpa_info->len;
1053 mapping = tpa_info->mapping;
1054
1055 agg_bufs = (le32_to_cpu(tpa_end->rx_tpa_end_cmp_misc_v1) &
1056 RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT;
1057
1058 if (agg_bufs) {
1059 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1060 return ERR_PTR(-EBUSY);
1061
1062 *agg_event = true;
1063 cp_cons = NEXT_CMP(cp_cons);
1064 }
1065
1066 if (unlikely(agg_bufs > MAX_SKB_FRAGS)) {
1067 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1068 netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1069 agg_bufs, (int)MAX_SKB_FRAGS);
1070 return NULL;
1071 }
1072
1073 if (len <= bp->rx_copy_thresh) {
1074 skb = bnxt_copy_skb(bnapi, data, len, mapping);
1075 if (!skb) {
1076 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1077 return NULL;
1078 }
1079 } else {
1080 u8 *new_data;
1081 dma_addr_t new_mapping;
1082
1083 new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1084 if (!new_data) {
1085 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1086 return NULL;
1087 }
1088
1089 tpa_info->data = new_data;
1090 tpa_info->mapping = new_mapping;
1091
1092 skb = build_skb(data, 0);
1093 dma_unmap_single(&bp->pdev->dev, mapping, bp->rx_buf_use_size,
1094 PCI_DMA_FROMDEVICE);
1095
1096 if (!skb) {
1097 kfree(data);
1098 bnxt_abort_tpa(bp, bnapi, cp_cons, agg_bufs);
1099 return NULL;
1100 }
1101 skb_reserve(skb, BNXT_RX_OFFSET);
1102 skb_put(skb, len);
1103 }
1104
1105 if (agg_bufs) {
1106 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1107 if (!skb) {
1108 /* Page reuse already handled by bnxt_rx_pages(). */
1109 return NULL;
1110 }
1111 }
1112 skb->protocol = eth_type_trans(skb, bp->dev);
1113
1114 if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1115 skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1116
1117 if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1118 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1119 u16 vlan_proto = tpa_info->metadata >>
1120 RX_CMP_FLAGS2_METADATA_TPID_SFT;
1121 u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_VID_MASK;
1122
1123 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1124 }
1125
1126 skb_checksum_none_assert(skb);
1127 if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1128 skb->ip_summed = CHECKSUM_UNNECESSARY;
1129 skb->csum_level =
1130 (tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1131 }
1132
1133 if (TPA_END_GRO(tpa_end))
1134 skb = bnxt_gro_skb(tpa_info, tpa_end, tpa_end1, skb);
1135
1136 return skb;
1137 }
1138
1139 /* returns the following:
1140 * 1 - 1 packet successfully received
1141 * 0 - successful TPA_START, packet not completed yet
1142 * -EBUSY - completion ring does not have all the agg buffers yet
1143 * -ENOMEM - packet aborted due to out of memory
1144 * -EIO - packet aborted due to hw error indicated in BD
1145 */
1146 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_napi *bnapi, u32 *raw_cons,
1147 bool *agg_event)
1148 {
1149 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1150 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1151 struct net_device *dev = bp->dev;
1152 struct rx_cmp *rxcmp;
1153 struct rx_cmp_ext *rxcmp1;
1154 u32 tmp_raw_cons = *raw_cons;
1155 u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1156 struct bnxt_sw_rx_bd *rx_buf;
1157 unsigned int len;
1158 u8 *data, agg_bufs, cmp_type;
1159 dma_addr_t dma_addr;
1160 struct sk_buff *skb;
1161 int rc = 0;
1162
1163 rxcmp = (struct rx_cmp *)
1164 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1165
1166 tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1167 cp_cons = RING_CMP(tmp_raw_cons);
1168 rxcmp1 = (struct rx_cmp_ext *)
1169 &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1170
1171 if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1172 return -EBUSY;
1173
1174 cmp_type = RX_CMP_TYPE(rxcmp);
1175
1176 prod = rxr->rx_prod;
1177
1178 if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1179 bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1180 (struct rx_tpa_start_cmp_ext *)rxcmp1);
1181
1182 goto next_rx_no_prod;
1183
1184 } else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1185 skb = bnxt_tpa_end(bp, bnapi, &tmp_raw_cons,
1186 (struct rx_tpa_end_cmp *)rxcmp,
1187 (struct rx_tpa_end_cmp_ext *)rxcmp1,
1188 agg_event);
1189
1190 if (unlikely(IS_ERR(skb)))
1191 return -EBUSY;
1192
1193 rc = -ENOMEM;
1194 if (likely(skb)) {
1195 skb_record_rx_queue(skb, bnapi->index);
1196 skb_mark_napi_id(skb, &bnapi->napi);
1197 if (bnxt_busy_polling(bnapi))
1198 netif_receive_skb(skb);
1199 else
1200 napi_gro_receive(&bnapi->napi, skb);
1201 rc = 1;
1202 }
1203 goto next_rx_no_prod;
1204 }
1205
1206 cons = rxcmp->rx_cmp_opaque;
1207 rx_buf = &rxr->rx_buf_ring[cons];
1208 data = rx_buf->data;
1209 if (unlikely(cons != rxr->rx_next_cons)) {
1210 int rc1 = bnxt_discard_rx(bp, bnapi, raw_cons, rxcmp);
1211
1212 bnxt_sched_reset(bp, rxr);
1213 return rc1;
1214 }
1215 prefetch(data);
1216
1217 agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) & RX_CMP_AGG_BUFS) >>
1218 RX_CMP_AGG_BUFS_SHIFT;
1219
1220 if (agg_bufs) {
1221 if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1222 return -EBUSY;
1223
1224 cp_cons = NEXT_CMP(cp_cons);
1225 *agg_event = true;
1226 }
1227
1228 rx_buf->data = NULL;
1229 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1230 bnxt_reuse_rx_data(rxr, cons, data);
1231 if (agg_bufs)
1232 bnxt_reuse_rx_agg_bufs(bnapi, cp_cons, agg_bufs);
1233
1234 rc = -EIO;
1235 goto next_rx;
1236 }
1237
1238 len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1239 dma_addr = dma_unmap_addr(rx_buf, mapping);
1240
1241 if (len <= bp->rx_copy_thresh) {
1242 skb = bnxt_copy_skb(bnapi, data, len, dma_addr);
1243 bnxt_reuse_rx_data(rxr, cons, data);
1244 if (!skb) {
1245 rc = -ENOMEM;
1246 goto next_rx;
1247 }
1248 } else {
1249 skb = bnxt_rx_skb(bp, rxr, cons, prod, data, dma_addr, len);
1250 if (!skb) {
1251 rc = -ENOMEM;
1252 goto next_rx;
1253 }
1254 }
1255
1256 if (agg_bufs) {
1257 skb = bnxt_rx_pages(bp, bnapi, skb, cp_cons, agg_bufs);
1258 if (!skb) {
1259 rc = -ENOMEM;
1260 goto next_rx;
1261 }
1262 }
1263
1264 if (RX_CMP_HASH_VALID(rxcmp)) {
1265 u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1266 enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1267
1268 /* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1269 if (hash_type != 1 && hash_type != 3)
1270 type = PKT_HASH_TYPE_L3;
1271 skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1272 }
1273
1274 skb->protocol = eth_type_trans(skb, dev);
1275
1276 if ((rxcmp1->rx_cmp_flags2 &
1277 cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1278 (skb->dev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
1279 u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1280 u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_VID_MASK;
1281 u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1282
1283 __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1284 }
1285
1286 skb_checksum_none_assert(skb);
1287 if (RX_CMP_L4_CS_OK(rxcmp1)) {
1288 if (dev->features & NETIF_F_RXCSUM) {
1289 skb->ip_summed = CHECKSUM_UNNECESSARY;
1290 skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1291 }
1292 } else {
1293 if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1294 if (dev->features & NETIF_F_RXCSUM)
1295 cpr->rx_l4_csum_errors++;
1296 }
1297 }
1298
1299 skb_record_rx_queue(skb, bnapi->index);
1300 skb_mark_napi_id(skb, &bnapi->napi);
1301 if (bnxt_busy_polling(bnapi))
1302 netif_receive_skb(skb);
1303 else
1304 napi_gro_receive(&bnapi->napi, skb);
1305 rc = 1;
1306
1307 next_rx:
1308 rxr->rx_prod = NEXT_RX(prod);
1309 rxr->rx_next_cons = NEXT_RX(cons);
1310
1311 next_rx_no_prod:
1312 *raw_cons = tmp_raw_cons;
1313
1314 return rc;
1315 }
1316
1317 #define BNXT_GET_EVENT_PORT(data) \
1318 ((data) & \
1319 HWRM_ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1320
1321 static int bnxt_async_event_process(struct bnxt *bp,
1322 struct hwrm_async_event_cmpl *cmpl)
1323 {
1324 u16 event_id = le16_to_cpu(cmpl->event_id);
1325
1326 /* TODO CHIMP_FW: Define event id's for link change, error etc */
1327 switch (event_id) {
1328 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1329 u32 data1 = le32_to_cpu(cmpl->event_data1);
1330 struct bnxt_link_info *link_info = &bp->link_info;
1331
1332 if (BNXT_VF(bp))
1333 goto async_event_process_exit;
1334 if (data1 & 0x20000) {
1335 u16 fw_speed = link_info->force_link_speed;
1336 u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1337
1338 netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1339 speed);
1340 }
1341 /* fall thru */
1342 }
1343 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1344 set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1345 break;
1346 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1347 set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1348 break;
1349 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1350 u32 data1 = le32_to_cpu(cmpl->event_data1);
1351 u16 port_id = BNXT_GET_EVENT_PORT(data1);
1352
1353 if (BNXT_VF(bp))
1354 break;
1355
1356 if (bp->pf.port_id != port_id)
1357 break;
1358
1359 set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1360 break;
1361 }
1362 case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1363 if (BNXT_PF(bp))
1364 goto async_event_process_exit;
1365 set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
1366 break;
1367 default:
1368 netdev_err(bp->dev, "unhandled ASYNC event (id 0x%x)\n",
1369 event_id);
1370 goto async_event_process_exit;
1371 }
1372 schedule_work(&bp->sp_task);
1373 async_event_process_exit:
1374 return 0;
1375 }
1376
1377 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
1378 {
1379 u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
1380 struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1381 struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
1382 (struct hwrm_fwd_req_cmpl *)txcmp;
1383
1384 switch (cmpl_type) {
1385 case CMPL_BASE_TYPE_HWRM_DONE:
1386 seq_id = le16_to_cpu(h_cmpl->sequence_id);
1387 if (seq_id == bp->hwrm_intr_seq_id)
1388 bp->hwrm_intr_seq_id = HWRM_SEQ_ID_INVALID;
1389 else
1390 netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
1391 break;
1392
1393 case CMPL_BASE_TYPE_HWRM_FWD_REQ:
1394 vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
1395
1396 if ((vf_id < bp->pf.first_vf_id) ||
1397 (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
1398 netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
1399 vf_id);
1400 return -EINVAL;
1401 }
1402
1403 set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
1404 set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
1405 schedule_work(&bp->sp_task);
1406 break;
1407
1408 case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1409 bnxt_async_event_process(bp,
1410 (struct hwrm_async_event_cmpl *)txcmp);
1411
1412 default:
1413 break;
1414 }
1415
1416 return 0;
1417 }
1418
1419 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
1420 {
1421 struct bnxt_napi *bnapi = dev_instance;
1422 struct bnxt *bp = bnapi->bp;
1423 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1424 u32 cons = RING_CMP(cpr->cp_raw_cons);
1425
1426 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1427 napi_schedule(&bnapi->napi);
1428 return IRQ_HANDLED;
1429 }
1430
1431 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
1432 {
1433 u32 raw_cons = cpr->cp_raw_cons;
1434 u16 cons = RING_CMP(raw_cons);
1435 struct tx_cmp *txcmp;
1436
1437 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1438
1439 return TX_CMP_VALID(txcmp, raw_cons);
1440 }
1441
1442 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
1443 {
1444 struct bnxt_napi *bnapi = dev_instance;
1445 struct bnxt *bp = bnapi->bp;
1446 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1447 u32 cons = RING_CMP(cpr->cp_raw_cons);
1448 u32 int_status;
1449
1450 prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
1451
1452 if (!bnxt_has_work(bp, cpr)) {
1453 int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
1454 /* return if erroneous interrupt */
1455 if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
1456 return IRQ_NONE;
1457 }
1458
1459 /* disable ring IRQ */
1460 BNXT_CP_DB_IRQ_DIS(cpr->cp_doorbell);
1461
1462 /* Return here if interrupt is shared and is disabled. */
1463 if (unlikely(atomic_read(&bp->intr_sem) != 0))
1464 return IRQ_HANDLED;
1465
1466 napi_schedule(&bnapi->napi);
1467 return IRQ_HANDLED;
1468 }
1469
1470 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
1471 {
1472 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1473 u32 raw_cons = cpr->cp_raw_cons;
1474 u32 cons;
1475 int tx_pkts = 0;
1476 int rx_pkts = 0;
1477 bool rx_event = false;
1478 bool agg_event = false;
1479 struct tx_cmp *txcmp;
1480
1481 while (1) {
1482 int rc;
1483
1484 cons = RING_CMP(raw_cons);
1485 txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
1486
1487 if (!TX_CMP_VALID(txcmp, raw_cons))
1488 break;
1489
1490 /* The valid test of the entry must be done first before
1491 * reading any further.
1492 */
1493 dma_rmb();
1494 if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
1495 tx_pkts++;
1496 /* return full budget so NAPI will complete. */
1497 if (unlikely(tx_pkts > bp->tx_wake_thresh))
1498 rx_pkts = budget;
1499 } else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
1500 rc = bnxt_rx_pkt(bp, bnapi, &raw_cons, &agg_event);
1501 if (likely(rc >= 0))
1502 rx_pkts += rc;
1503 else if (rc == -EBUSY) /* partial completion */
1504 break;
1505 rx_event = true;
1506 } else if (unlikely((TX_CMP_TYPE(txcmp) ==
1507 CMPL_BASE_TYPE_HWRM_DONE) ||
1508 (TX_CMP_TYPE(txcmp) ==
1509 CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
1510 (TX_CMP_TYPE(txcmp) ==
1511 CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
1512 bnxt_hwrm_handler(bp, txcmp);
1513 }
1514 raw_cons = NEXT_RAW_CMP(raw_cons);
1515
1516 if (rx_pkts == budget)
1517 break;
1518 }
1519
1520 cpr->cp_raw_cons = raw_cons;
1521 /* ACK completion ring before freeing tx ring and producing new
1522 * buffers in rx/agg rings to prevent overflowing the completion
1523 * ring.
1524 */
1525 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
1526
1527 if (tx_pkts)
1528 bnxt_tx_int(bp, bnapi, tx_pkts);
1529
1530 if (rx_event) {
1531 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1532
1533 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1534 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
1535 if (agg_event) {
1536 writel(DB_KEY_RX | rxr->rx_agg_prod,
1537 rxr->rx_agg_doorbell);
1538 writel(DB_KEY_RX | rxr->rx_agg_prod,
1539 rxr->rx_agg_doorbell);
1540 }
1541 }
1542 return rx_pkts;
1543 }
1544
1545 static int bnxt_poll(struct napi_struct *napi, int budget)
1546 {
1547 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1548 struct bnxt *bp = bnapi->bp;
1549 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1550 int work_done = 0;
1551
1552 if (!bnxt_lock_napi(bnapi))
1553 return budget;
1554
1555 while (1) {
1556 work_done += bnxt_poll_work(bp, bnapi, budget - work_done);
1557
1558 if (work_done >= budget)
1559 break;
1560
1561 if (!bnxt_has_work(bp, cpr)) {
1562 napi_complete(napi);
1563 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1564 break;
1565 }
1566 }
1567 mmiowb();
1568 bnxt_unlock_napi(bnapi);
1569 return work_done;
1570 }
1571
1572 #ifdef CONFIG_NET_RX_BUSY_POLL
1573 static int bnxt_busy_poll(struct napi_struct *napi)
1574 {
1575 struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
1576 struct bnxt *bp = bnapi->bp;
1577 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
1578 int rx_work, budget = 4;
1579
1580 if (atomic_read(&bp->intr_sem) != 0)
1581 return LL_FLUSH_FAILED;
1582
1583 if (!bnxt_lock_poll(bnapi))
1584 return LL_FLUSH_BUSY;
1585
1586 rx_work = bnxt_poll_work(bp, bnapi, budget);
1587
1588 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
1589
1590 bnxt_unlock_poll(bnapi);
1591 return rx_work;
1592 }
1593 #endif
1594
1595 static void bnxt_free_tx_skbs(struct bnxt *bp)
1596 {
1597 int i, max_idx;
1598 struct pci_dev *pdev = bp->pdev;
1599
1600 if (!bp->tx_ring)
1601 return;
1602
1603 max_idx = bp->tx_nr_pages * TX_DESC_CNT;
1604 for (i = 0; i < bp->tx_nr_rings; i++) {
1605 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1606 int j;
1607
1608 for (j = 0; j < max_idx;) {
1609 struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
1610 struct sk_buff *skb = tx_buf->skb;
1611 int k, last;
1612
1613 if (!skb) {
1614 j++;
1615 continue;
1616 }
1617
1618 tx_buf->skb = NULL;
1619
1620 if (tx_buf->is_push) {
1621 dev_kfree_skb(skb);
1622 j += 2;
1623 continue;
1624 }
1625
1626 dma_unmap_single(&pdev->dev,
1627 dma_unmap_addr(tx_buf, mapping),
1628 skb_headlen(skb),
1629 PCI_DMA_TODEVICE);
1630
1631 last = tx_buf->nr_frags;
1632 j += 2;
1633 for (k = 0; k < last; k++, j++) {
1634 int ring_idx = j & bp->tx_ring_mask;
1635 skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
1636
1637 tx_buf = &txr->tx_buf_ring[ring_idx];
1638 dma_unmap_page(
1639 &pdev->dev,
1640 dma_unmap_addr(tx_buf, mapping),
1641 skb_frag_size(frag), PCI_DMA_TODEVICE);
1642 }
1643 dev_kfree_skb(skb);
1644 }
1645 netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
1646 }
1647 }
1648
1649 static void bnxt_free_rx_skbs(struct bnxt *bp)
1650 {
1651 int i, max_idx, max_agg_idx;
1652 struct pci_dev *pdev = bp->pdev;
1653
1654 if (!bp->rx_ring)
1655 return;
1656
1657 max_idx = bp->rx_nr_pages * RX_DESC_CNT;
1658 max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
1659 for (i = 0; i < bp->rx_nr_rings; i++) {
1660 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1661 int j;
1662
1663 if (rxr->rx_tpa) {
1664 for (j = 0; j < MAX_TPA; j++) {
1665 struct bnxt_tpa_info *tpa_info =
1666 &rxr->rx_tpa[j];
1667 u8 *data = tpa_info->data;
1668
1669 if (!data)
1670 continue;
1671
1672 dma_unmap_single(
1673 &pdev->dev,
1674 dma_unmap_addr(tpa_info, mapping),
1675 bp->rx_buf_use_size,
1676 PCI_DMA_FROMDEVICE);
1677
1678 tpa_info->data = NULL;
1679
1680 kfree(data);
1681 }
1682 }
1683
1684 for (j = 0; j < max_idx; j++) {
1685 struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
1686 u8 *data = rx_buf->data;
1687
1688 if (!data)
1689 continue;
1690
1691 dma_unmap_single(&pdev->dev,
1692 dma_unmap_addr(rx_buf, mapping),
1693 bp->rx_buf_use_size,
1694 PCI_DMA_FROMDEVICE);
1695
1696 rx_buf->data = NULL;
1697
1698 kfree(data);
1699 }
1700
1701 for (j = 0; j < max_agg_idx; j++) {
1702 struct bnxt_sw_rx_agg_bd *rx_agg_buf =
1703 &rxr->rx_agg_ring[j];
1704 struct page *page = rx_agg_buf->page;
1705
1706 if (!page)
1707 continue;
1708
1709 dma_unmap_page(&pdev->dev,
1710 dma_unmap_addr(rx_agg_buf, mapping),
1711 BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE);
1712
1713 rx_agg_buf->page = NULL;
1714 __clear_bit(j, rxr->rx_agg_bmap);
1715
1716 __free_page(page);
1717 }
1718 if (rxr->rx_page) {
1719 __free_page(rxr->rx_page);
1720 rxr->rx_page = NULL;
1721 }
1722 }
1723 }
1724
1725 static void bnxt_free_skbs(struct bnxt *bp)
1726 {
1727 bnxt_free_tx_skbs(bp);
1728 bnxt_free_rx_skbs(bp);
1729 }
1730
1731 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1732 {
1733 struct pci_dev *pdev = bp->pdev;
1734 int i;
1735
1736 for (i = 0; i < ring->nr_pages; i++) {
1737 if (!ring->pg_arr[i])
1738 continue;
1739
1740 dma_free_coherent(&pdev->dev, ring->page_size,
1741 ring->pg_arr[i], ring->dma_arr[i]);
1742
1743 ring->pg_arr[i] = NULL;
1744 }
1745 if (ring->pg_tbl) {
1746 dma_free_coherent(&pdev->dev, ring->nr_pages * 8,
1747 ring->pg_tbl, ring->pg_tbl_map);
1748 ring->pg_tbl = NULL;
1749 }
1750 if (ring->vmem_size && *ring->vmem) {
1751 vfree(*ring->vmem);
1752 *ring->vmem = NULL;
1753 }
1754 }
1755
1756 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_struct *ring)
1757 {
1758 int i;
1759 struct pci_dev *pdev = bp->pdev;
1760
1761 if (ring->nr_pages > 1) {
1762 ring->pg_tbl = dma_alloc_coherent(&pdev->dev,
1763 ring->nr_pages * 8,
1764 &ring->pg_tbl_map,
1765 GFP_KERNEL);
1766 if (!ring->pg_tbl)
1767 return -ENOMEM;
1768 }
1769
1770 for (i = 0; i < ring->nr_pages; i++) {
1771 ring->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
1772 ring->page_size,
1773 &ring->dma_arr[i],
1774 GFP_KERNEL);
1775 if (!ring->pg_arr[i])
1776 return -ENOMEM;
1777
1778 if (ring->nr_pages > 1)
1779 ring->pg_tbl[i] = cpu_to_le64(ring->dma_arr[i]);
1780 }
1781
1782 if (ring->vmem_size) {
1783 *ring->vmem = vzalloc(ring->vmem_size);
1784 if (!(*ring->vmem))
1785 return -ENOMEM;
1786 }
1787 return 0;
1788 }
1789
1790 static void bnxt_free_rx_rings(struct bnxt *bp)
1791 {
1792 int i;
1793
1794 if (!bp->rx_ring)
1795 return;
1796
1797 for (i = 0; i < bp->rx_nr_rings; i++) {
1798 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1799 struct bnxt_ring_struct *ring;
1800
1801 kfree(rxr->rx_tpa);
1802 rxr->rx_tpa = NULL;
1803
1804 kfree(rxr->rx_agg_bmap);
1805 rxr->rx_agg_bmap = NULL;
1806
1807 ring = &rxr->rx_ring_struct;
1808 bnxt_free_ring(bp, ring);
1809
1810 ring = &rxr->rx_agg_ring_struct;
1811 bnxt_free_ring(bp, ring);
1812 }
1813 }
1814
1815 static int bnxt_alloc_rx_rings(struct bnxt *bp)
1816 {
1817 int i, rc, agg_rings = 0, tpa_rings = 0;
1818
1819 if (!bp->rx_ring)
1820 return -ENOMEM;
1821
1822 if (bp->flags & BNXT_FLAG_AGG_RINGS)
1823 agg_rings = 1;
1824
1825 if (bp->flags & BNXT_FLAG_TPA)
1826 tpa_rings = 1;
1827
1828 for (i = 0; i < bp->rx_nr_rings; i++) {
1829 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
1830 struct bnxt_ring_struct *ring;
1831
1832 ring = &rxr->rx_ring_struct;
1833
1834 rc = bnxt_alloc_ring(bp, ring);
1835 if (rc)
1836 return rc;
1837
1838 if (agg_rings) {
1839 u16 mem_size;
1840
1841 ring = &rxr->rx_agg_ring_struct;
1842 rc = bnxt_alloc_ring(bp, ring);
1843 if (rc)
1844 return rc;
1845
1846 rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
1847 mem_size = rxr->rx_agg_bmap_size / 8;
1848 rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
1849 if (!rxr->rx_agg_bmap)
1850 return -ENOMEM;
1851
1852 if (tpa_rings) {
1853 rxr->rx_tpa = kcalloc(MAX_TPA,
1854 sizeof(struct bnxt_tpa_info),
1855 GFP_KERNEL);
1856 if (!rxr->rx_tpa)
1857 return -ENOMEM;
1858 }
1859 }
1860 }
1861 return 0;
1862 }
1863
1864 static void bnxt_free_tx_rings(struct bnxt *bp)
1865 {
1866 int i;
1867 struct pci_dev *pdev = bp->pdev;
1868
1869 if (!bp->tx_ring)
1870 return;
1871
1872 for (i = 0; i < bp->tx_nr_rings; i++) {
1873 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1874 struct bnxt_ring_struct *ring;
1875
1876 if (txr->tx_push) {
1877 dma_free_coherent(&pdev->dev, bp->tx_push_size,
1878 txr->tx_push, txr->tx_push_mapping);
1879 txr->tx_push = NULL;
1880 }
1881
1882 ring = &txr->tx_ring_struct;
1883
1884 bnxt_free_ring(bp, ring);
1885 }
1886 }
1887
1888 static int bnxt_alloc_tx_rings(struct bnxt *bp)
1889 {
1890 int i, j, rc;
1891 struct pci_dev *pdev = bp->pdev;
1892
1893 bp->tx_push_size = 0;
1894 if (bp->tx_push_thresh) {
1895 int push_size;
1896
1897 push_size = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
1898 bp->tx_push_thresh);
1899
1900 if (push_size > 256) {
1901 push_size = 0;
1902 bp->tx_push_thresh = 0;
1903 }
1904
1905 bp->tx_push_size = push_size;
1906 }
1907
1908 for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
1909 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
1910 struct bnxt_ring_struct *ring;
1911
1912 ring = &txr->tx_ring_struct;
1913
1914 rc = bnxt_alloc_ring(bp, ring);
1915 if (rc)
1916 return rc;
1917
1918 if (bp->tx_push_size) {
1919 dma_addr_t mapping;
1920
1921 /* One pre-allocated DMA buffer to backup
1922 * TX push operation
1923 */
1924 txr->tx_push = dma_alloc_coherent(&pdev->dev,
1925 bp->tx_push_size,
1926 &txr->tx_push_mapping,
1927 GFP_KERNEL);
1928
1929 if (!txr->tx_push)
1930 return -ENOMEM;
1931
1932 mapping = txr->tx_push_mapping +
1933 sizeof(struct tx_push_bd);
1934 txr->data_mapping = cpu_to_le64(mapping);
1935
1936 memset(txr->tx_push, 0, sizeof(struct tx_push_bd));
1937 }
1938 ring->queue_id = bp->q_info[j].queue_id;
1939 if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
1940 j++;
1941 }
1942 return 0;
1943 }
1944
1945 static void bnxt_free_cp_rings(struct bnxt *bp)
1946 {
1947 int i;
1948
1949 if (!bp->bnapi)
1950 return;
1951
1952 for (i = 0; i < bp->cp_nr_rings; i++) {
1953 struct bnxt_napi *bnapi = bp->bnapi[i];
1954 struct bnxt_cp_ring_info *cpr;
1955 struct bnxt_ring_struct *ring;
1956
1957 if (!bnapi)
1958 continue;
1959
1960 cpr = &bnapi->cp_ring;
1961 ring = &cpr->cp_ring_struct;
1962
1963 bnxt_free_ring(bp, ring);
1964 }
1965 }
1966
1967 static int bnxt_alloc_cp_rings(struct bnxt *bp)
1968 {
1969 int i, rc;
1970
1971 for (i = 0; i < bp->cp_nr_rings; i++) {
1972 struct bnxt_napi *bnapi = bp->bnapi[i];
1973 struct bnxt_cp_ring_info *cpr;
1974 struct bnxt_ring_struct *ring;
1975
1976 if (!bnapi)
1977 continue;
1978
1979 cpr = &bnapi->cp_ring;
1980 ring = &cpr->cp_ring_struct;
1981
1982 rc = bnxt_alloc_ring(bp, ring);
1983 if (rc)
1984 return rc;
1985 }
1986 return 0;
1987 }
1988
1989 static void bnxt_init_ring_struct(struct bnxt *bp)
1990 {
1991 int i;
1992
1993 for (i = 0; i < bp->cp_nr_rings; i++) {
1994 struct bnxt_napi *bnapi = bp->bnapi[i];
1995 struct bnxt_cp_ring_info *cpr;
1996 struct bnxt_rx_ring_info *rxr;
1997 struct bnxt_tx_ring_info *txr;
1998 struct bnxt_ring_struct *ring;
1999
2000 if (!bnapi)
2001 continue;
2002
2003 cpr = &bnapi->cp_ring;
2004 ring = &cpr->cp_ring_struct;
2005 ring->nr_pages = bp->cp_nr_pages;
2006 ring->page_size = HW_CMPD_RING_SIZE;
2007 ring->pg_arr = (void **)cpr->cp_desc_ring;
2008 ring->dma_arr = cpr->cp_desc_mapping;
2009 ring->vmem_size = 0;
2010
2011 rxr = bnapi->rx_ring;
2012 if (!rxr)
2013 goto skip_rx;
2014
2015 ring = &rxr->rx_ring_struct;
2016 ring->nr_pages = bp->rx_nr_pages;
2017 ring->page_size = HW_RXBD_RING_SIZE;
2018 ring->pg_arr = (void **)rxr->rx_desc_ring;
2019 ring->dma_arr = rxr->rx_desc_mapping;
2020 ring->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
2021 ring->vmem = (void **)&rxr->rx_buf_ring;
2022
2023 ring = &rxr->rx_agg_ring_struct;
2024 ring->nr_pages = bp->rx_agg_nr_pages;
2025 ring->page_size = HW_RXBD_RING_SIZE;
2026 ring->pg_arr = (void **)rxr->rx_agg_desc_ring;
2027 ring->dma_arr = rxr->rx_agg_desc_mapping;
2028 ring->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
2029 ring->vmem = (void **)&rxr->rx_agg_ring;
2030
2031 skip_rx:
2032 txr = bnapi->tx_ring;
2033 if (!txr)
2034 continue;
2035
2036 ring = &txr->tx_ring_struct;
2037 ring->nr_pages = bp->tx_nr_pages;
2038 ring->page_size = HW_RXBD_RING_SIZE;
2039 ring->pg_arr = (void **)txr->tx_desc_ring;
2040 ring->dma_arr = txr->tx_desc_mapping;
2041 ring->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
2042 ring->vmem = (void **)&txr->tx_buf_ring;
2043 }
2044 }
2045
2046 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
2047 {
2048 int i;
2049 u32 prod;
2050 struct rx_bd **rx_buf_ring;
2051
2052 rx_buf_ring = (struct rx_bd **)ring->pg_arr;
2053 for (i = 0, prod = 0; i < ring->nr_pages; i++) {
2054 int j;
2055 struct rx_bd *rxbd;
2056
2057 rxbd = rx_buf_ring[i];
2058 if (!rxbd)
2059 continue;
2060
2061 for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
2062 rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
2063 rxbd->rx_bd_opaque = prod;
2064 }
2065 }
2066 }
2067
2068 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
2069 {
2070 struct net_device *dev = bp->dev;
2071 struct bnxt_rx_ring_info *rxr;
2072 struct bnxt_ring_struct *ring;
2073 u32 prod, type;
2074 int i;
2075
2076 type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
2077 RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
2078
2079 if (NET_IP_ALIGN == 2)
2080 type |= RX_BD_FLAGS_SOP;
2081
2082 rxr = &bp->rx_ring[ring_nr];
2083 ring = &rxr->rx_ring_struct;
2084 bnxt_init_rxbd_pages(ring, type);
2085
2086 prod = rxr->rx_prod;
2087 for (i = 0; i < bp->rx_ring_size; i++) {
2088 if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
2089 netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
2090 ring_nr, i, bp->rx_ring_size);
2091 break;
2092 }
2093 prod = NEXT_RX(prod);
2094 }
2095 rxr->rx_prod = prod;
2096 ring->fw_ring_id = INVALID_HW_RING_ID;
2097
2098 ring = &rxr->rx_agg_ring_struct;
2099 ring->fw_ring_id = INVALID_HW_RING_ID;
2100
2101 if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
2102 return 0;
2103
2104 type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
2105 RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
2106
2107 bnxt_init_rxbd_pages(ring, type);
2108
2109 prod = rxr->rx_agg_prod;
2110 for (i = 0; i < bp->rx_agg_ring_size; i++) {
2111 if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
2112 netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
2113 ring_nr, i, bp->rx_ring_size);
2114 break;
2115 }
2116 prod = NEXT_RX_AGG(prod);
2117 }
2118 rxr->rx_agg_prod = prod;
2119
2120 if (bp->flags & BNXT_FLAG_TPA) {
2121 if (rxr->rx_tpa) {
2122 u8 *data;
2123 dma_addr_t mapping;
2124
2125 for (i = 0; i < MAX_TPA; i++) {
2126 data = __bnxt_alloc_rx_data(bp, &mapping,
2127 GFP_KERNEL);
2128 if (!data)
2129 return -ENOMEM;
2130
2131 rxr->rx_tpa[i].data = data;
2132 rxr->rx_tpa[i].mapping = mapping;
2133 }
2134 } else {
2135 netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
2136 return -ENOMEM;
2137 }
2138 }
2139
2140 return 0;
2141 }
2142
2143 static int bnxt_init_rx_rings(struct bnxt *bp)
2144 {
2145 int i, rc = 0;
2146
2147 for (i = 0; i < bp->rx_nr_rings; i++) {
2148 rc = bnxt_init_one_rx_ring(bp, i);
2149 if (rc)
2150 break;
2151 }
2152
2153 return rc;
2154 }
2155
2156 static int bnxt_init_tx_rings(struct bnxt *bp)
2157 {
2158 u16 i;
2159
2160 bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
2161 MAX_SKB_FRAGS + 1);
2162
2163 for (i = 0; i < bp->tx_nr_rings; i++) {
2164 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2165 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
2166
2167 ring->fw_ring_id = INVALID_HW_RING_ID;
2168 }
2169
2170 return 0;
2171 }
2172
2173 static void bnxt_free_ring_grps(struct bnxt *bp)
2174 {
2175 kfree(bp->grp_info);
2176 bp->grp_info = NULL;
2177 }
2178
2179 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
2180 {
2181 int i;
2182
2183 if (irq_re_init) {
2184 bp->grp_info = kcalloc(bp->cp_nr_rings,
2185 sizeof(struct bnxt_ring_grp_info),
2186 GFP_KERNEL);
2187 if (!bp->grp_info)
2188 return -ENOMEM;
2189 }
2190 for (i = 0; i < bp->cp_nr_rings; i++) {
2191 if (irq_re_init)
2192 bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
2193 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
2194 bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
2195 bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
2196 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
2197 }
2198 return 0;
2199 }
2200
2201 static void bnxt_free_vnics(struct bnxt *bp)
2202 {
2203 kfree(bp->vnic_info);
2204 bp->vnic_info = NULL;
2205 bp->nr_vnics = 0;
2206 }
2207
2208 static int bnxt_alloc_vnics(struct bnxt *bp)
2209 {
2210 int num_vnics = 1;
2211
2212 #ifdef CONFIG_RFS_ACCEL
2213 if (bp->flags & BNXT_FLAG_RFS)
2214 num_vnics += bp->rx_nr_rings;
2215 #endif
2216
2217 bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
2218 GFP_KERNEL);
2219 if (!bp->vnic_info)
2220 return -ENOMEM;
2221
2222 bp->nr_vnics = num_vnics;
2223 return 0;
2224 }
2225
2226 static void bnxt_init_vnics(struct bnxt *bp)
2227 {
2228 int i;
2229
2230 for (i = 0; i < bp->nr_vnics; i++) {
2231 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2232
2233 vnic->fw_vnic_id = INVALID_HW_RING_ID;
2234 vnic->fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
2235 vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
2236
2237 if (bp->vnic_info[i].rss_hash_key) {
2238 if (i == 0)
2239 prandom_bytes(vnic->rss_hash_key,
2240 HW_HASH_KEY_SIZE);
2241 else
2242 memcpy(vnic->rss_hash_key,
2243 bp->vnic_info[0].rss_hash_key,
2244 HW_HASH_KEY_SIZE);
2245 }
2246 }
2247 }
2248
2249 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
2250 {
2251 int pages;
2252
2253 pages = ring_size / desc_per_pg;
2254
2255 if (!pages)
2256 return 1;
2257
2258 pages++;
2259
2260 while (pages & (pages - 1))
2261 pages++;
2262
2263 return pages;
2264 }
2265
2266 static void bnxt_set_tpa_flags(struct bnxt *bp)
2267 {
2268 bp->flags &= ~BNXT_FLAG_TPA;
2269 if (bp->dev->features & NETIF_F_LRO)
2270 bp->flags |= BNXT_FLAG_LRO;
2271 if ((bp->dev->features & NETIF_F_GRO) && (bp->pdev->revision > 0))
2272 bp->flags |= BNXT_FLAG_GRO;
2273 }
2274
2275 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
2276 * be set on entry.
2277 */
2278 void bnxt_set_ring_params(struct bnxt *bp)
2279 {
2280 u32 ring_size, rx_size, rx_space;
2281 u32 agg_factor = 0, agg_ring_size = 0;
2282
2283 /* 8 for CRC and VLAN */
2284 rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
2285
2286 rx_space = rx_size + NET_SKB_PAD +
2287 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2288
2289 bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
2290 ring_size = bp->rx_ring_size;
2291 bp->rx_agg_ring_size = 0;
2292 bp->rx_agg_nr_pages = 0;
2293
2294 if (bp->flags & BNXT_FLAG_TPA)
2295 agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
2296
2297 bp->flags &= ~BNXT_FLAG_JUMBO;
2298 if (rx_space > PAGE_SIZE) {
2299 u32 jumbo_factor;
2300
2301 bp->flags |= BNXT_FLAG_JUMBO;
2302 jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
2303 if (jumbo_factor > agg_factor)
2304 agg_factor = jumbo_factor;
2305 }
2306 agg_ring_size = ring_size * agg_factor;
2307
2308 if (agg_ring_size) {
2309 bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
2310 RX_DESC_CNT);
2311 if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
2312 u32 tmp = agg_ring_size;
2313
2314 bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
2315 agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
2316 netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
2317 tmp, agg_ring_size);
2318 }
2319 bp->rx_agg_ring_size = agg_ring_size;
2320 bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2321 rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
2322 rx_space = rx_size + NET_SKB_PAD +
2323 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2324 }
2325
2326 bp->rx_buf_use_size = rx_size;
2327 bp->rx_buf_size = rx_space;
2328
2329 bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
2330 bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
2331
2332 ring_size = bp->tx_ring_size;
2333 bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
2334 bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
2335
2336 ring_size = bp->rx_ring_size * (2 + agg_factor) + bp->tx_ring_size;
2337 bp->cp_ring_size = ring_size;
2338
2339 bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
2340 if (bp->cp_nr_pages > MAX_CP_PAGES) {
2341 bp->cp_nr_pages = MAX_CP_PAGES;
2342 bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
2343 netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
2344 ring_size, bp->cp_ring_size);
2345 }
2346 bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
2347 bp->cp_ring_mask = bp->cp_bit - 1;
2348 }
2349
2350 static void bnxt_free_vnic_attributes(struct bnxt *bp)
2351 {
2352 int i;
2353 struct bnxt_vnic_info *vnic;
2354 struct pci_dev *pdev = bp->pdev;
2355
2356 if (!bp->vnic_info)
2357 return;
2358
2359 for (i = 0; i < bp->nr_vnics; i++) {
2360 vnic = &bp->vnic_info[i];
2361
2362 kfree(vnic->fw_grp_ids);
2363 vnic->fw_grp_ids = NULL;
2364
2365 kfree(vnic->uc_list);
2366 vnic->uc_list = NULL;
2367
2368 if (vnic->mc_list) {
2369 dma_free_coherent(&pdev->dev, vnic->mc_list_size,
2370 vnic->mc_list, vnic->mc_list_mapping);
2371 vnic->mc_list = NULL;
2372 }
2373
2374 if (vnic->rss_table) {
2375 dma_free_coherent(&pdev->dev, PAGE_SIZE,
2376 vnic->rss_table,
2377 vnic->rss_table_dma_addr);
2378 vnic->rss_table = NULL;
2379 }
2380
2381 vnic->rss_hash_key = NULL;
2382 vnic->flags = 0;
2383 }
2384 }
2385
2386 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
2387 {
2388 int i, rc = 0, size;
2389 struct bnxt_vnic_info *vnic;
2390 struct pci_dev *pdev = bp->pdev;
2391 int max_rings;
2392
2393 for (i = 0; i < bp->nr_vnics; i++) {
2394 vnic = &bp->vnic_info[i];
2395
2396 if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
2397 int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
2398
2399 if (mem_size > 0) {
2400 vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
2401 if (!vnic->uc_list) {
2402 rc = -ENOMEM;
2403 goto out;
2404 }
2405 }
2406 }
2407
2408 if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
2409 vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
2410 vnic->mc_list =
2411 dma_alloc_coherent(&pdev->dev,
2412 vnic->mc_list_size,
2413 &vnic->mc_list_mapping,
2414 GFP_KERNEL);
2415 if (!vnic->mc_list) {
2416 rc = -ENOMEM;
2417 goto out;
2418 }
2419 }
2420
2421 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
2422 max_rings = bp->rx_nr_rings;
2423 else
2424 max_rings = 1;
2425
2426 vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
2427 if (!vnic->fw_grp_ids) {
2428 rc = -ENOMEM;
2429 goto out;
2430 }
2431
2432 /* Allocate rss table and hash key */
2433 vnic->rss_table = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2434 &vnic->rss_table_dma_addr,
2435 GFP_KERNEL);
2436 if (!vnic->rss_table) {
2437 rc = -ENOMEM;
2438 goto out;
2439 }
2440
2441 size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
2442
2443 vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
2444 vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
2445 }
2446 return 0;
2447
2448 out:
2449 return rc;
2450 }
2451
2452 static void bnxt_free_hwrm_resources(struct bnxt *bp)
2453 {
2454 struct pci_dev *pdev = bp->pdev;
2455
2456 dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
2457 bp->hwrm_cmd_resp_dma_addr);
2458
2459 bp->hwrm_cmd_resp_addr = NULL;
2460 if (bp->hwrm_dbg_resp_addr) {
2461 dma_free_coherent(&pdev->dev, HWRM_DBG_REG_BUF_SIZE,
2462 bp->hwrm_dbg_resp_addr,
2463 bp->hwrm_dbg_resp_dma_addr);
2464
2465 bp->hwrm_dbg_resp_addr = NULL;
2466 }
2467 }
2468
2469 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
2470 {
2471 struct pci_dev *pdev = bp->pdev;
2472
2473 bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
2474 &bp->hwrm_cmd_resp_dma_addr,
2475 GFP_KERNEL);
2476 if (!bp->hwrm_cmd_resp_addr)
2477 return -ENOMEM;
2478 bp->hwrm_dbg_resp_addr = dma_alloc_coherent(&pdev->dev,
2479 HWRM_DBG_REG_BUF_SIZE,
2480 &bp->hwrm_dbg_resp_dma_addr,
2481 GFP_KERNEL);
2482 if (!bp->hwrm_dbg_resp_addr)
2483 netdev_warn(bp->dev, "fail to alloc debug register dma mem\n");
2484
2485 return 0;
2486 }
2487
2488 static void bnxt_free_stats(struct bnxt *bp)
2489 {
2490 u32 size, i;
2491 struct pci_dev *pdev = bp->pdev;
2492
2493 if (bp->hw_rx_port_stats) {
2494 dma_free_coherent(&pdev->dev, bp->hw_port_stats_size,
2495 bp->hw_rx_port_stats,
2496 bp->hw_rx_port_stats_map);
2497 bp->hw_rx_port_stats = NULL;
2498 bp->flags &= ~BNXT_FLAG_PORT_STATS;
2499 }
2500
2501 if (!bp->bnapi)
2502 return;
2503
2504 size = sizeof(struct ctx_hw_stats);
2505
2506 for (i = 0; i < bp->cp_nr_rings; i++) {
2507 struct bnxt_napi *bnapi = bp->bnapi[i];
2508 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2509
2510 if (cpr->hw_stats) {
2511 dma_free_coherent(&pdev->dev, size, cpr->hw_stats,
2512 cpr->hw_stats_map);
2513 cpr->hw_stats = NULL;
2514 }
2515 }
2516 }
2517
2518 static int bnxt_alloc_stats(struct bnxt *bp)
2519 {
2520 u32 size, i;
2521 struct pci_dev *pdev = bp->pdev;
2522
2523 size = sizeof(struct ctx_hw_stats);
2524
2525 for (i = 0; i < bp->cp_nr_rings; i++) {
2526 struct bnxt_napi *bnapi = bp->bnapi[i];
2527 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2528
2529 cpr->hw_stats = dma_alloc_coherent(&pdev->dev, size,
2530 &cpr->hw_stats_map,
2531 GFP_KERNEL);
2532 if (!cpr->hw_stats)
2533 return -ENOMEM;
2534
2535 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
2536 }
2537
2538 if (BNXT_PF(bp)) {
2539 bp->hw_port_stats_size = sizeof(struct rx_port_stats) +
2540 sizeof(struct tx_port_stats) + 1024;
2541
2542 bp->hw_rx_port_stats =
2543 dma_alloc_coherent(&pdev->dev, bp->hw_port_stats_size,
2544 &bp->hw_rx_port_stats_map,
2545 GFP_KERNEL);
2546 if (!bp->hw_rx_port_stats)
2547 return -ENOMEM;
2548
2549 bp->hw_tx_port_stats = (void *)(bp->hw_rx_port_stats + 1) +
2550 512;
2551 bp->hw_tx_port_stats_map = bp->hw_rx_port_stats_map +
2552 sizeof(struct rx_port_stats) + 512;
2553 bp->flags |= BNXT_FLAG_PORT_STATS;
2554 }
2555 return 0;
2556 }
2557
2558 static void bnxt_clear_ring_indices(struct bnxt *bp)
2559 {
2560 int i;
2561
2562 if (!bp->bnapi)
2563 return;
2564
2565 for (i = 0; i < bp->cp_nr_rings; i++) {
2566 struct bnxt_napi *bnapi = bp->bnapi[i];
2567 struct bnxt_cp_ring_info *cpr;
2568 struct bnxt_rx_ring_info *rxr;
2569 struct bnxt_tx_ring_info *txr;
2570
2571 if (!bnapi)
2572 continue;
2573
2574 cpr = &bnapi->cp_ring;
2575 cpr->cp_raw_cons = 0;
2576
2577 txr = bnapi->tx_ring;
2578 if (txr) {
2579 txr->tx_prod = 0;
2580 txr->tx_cons = 0;
2581 }
2582
2583 rxr = bnapi->rx_ring;
2584 if (rxr) {
2585 rxr->rx_prod = 0;
2586 rxr->rx_agg_prod = 0;
2587 rxr->rx_sw_agg_prod = 0;
2588 rxr->rx_next_cons = 0;
2589 }
2590 }
2591 }
2592
2593 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
2594 {
2595 #ifdef CONFIG_RFS_ACCEL
2596 int i;
2597
2598 /* Under rtnl_lock and all our NAPIs have been disabled. It's
2599 * safe to delete the hash table.
2600 */
2601 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
2602 struct hlist_head *head;
2603 struct hlist_node *tmp;
2604 struct bnxt_ntuple_filter *fltr;
2605
2606 head = &bp->ntp_fltr_hash_tbl[i];
2607 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
2608 hlist_del(&fltr->hash);
2609 kfree(fltr);
2610 }
2611 }
2612 if (irq_reinit) {
2613 kfree(bp->ntp_fltr_bmap);
2614 bp->ntp_fltr_bmap = NULL;
2615 }
2616 bp->ntp_fltr_count = 0;
2617 #endif
2618 }
2619
2620 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
2621 {
2622 #ifdef CONFIG_RFS_ACCEL
2623 int i, rc = 0;
2624
2625 if (!(bp->flags & BNXT_FLAG_RFS))
2626 return 0;
2627
2628 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
2629 INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
2630
2631 bp->ntp_fltr_count = 0;
2632 bp->ntp_fltr_bmap = kzalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
2633 GFP_KERNEL);
2634
2635 if (!bp->ntp_fltr_bmap)
2636 rc = -ENOMEM;
2637
2638 return rc;
2639 #else
2640 return 0;
2641 #endif
2642 }
2643
2644 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
2645 {
2646 bnxt_free_vnic_attributes(bp);
2647 bnxt_free_tx_rings(bp);
2648 bnxt_free_rx_rings(bp);
2649 bnxt_free_cp_rings(bp);
2650 bnxt_free_ntp_fltrs(bp, irq_re_init);
2651 if (irq_re_init) {
2652 bnxt_free_stats(bp);
2653 bnxt_free_ring_grps(bp);
2654 bnxt_free_vnics(bp);
2655 kfree(bp->tx_ring);
2656 bp->tx_ring = NULL;
2657 kfree(bp->rx_ring);
2658 bp->rx_ring = NULL;
2659 kfree(bp->bnapi);
2660 bp->bnapi = NULL;
2661 } else {
2662 bnxt_clear_ring_indices(bp);
2663 }
2664 }
2665
2666 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
2667 {
2668 int i, j, rc, size, arr_size;
2669 void *bnapi;
2670
2671 if (irq_re_init) {
2672 /* Allocate bnapi mem pointer array and mem block for
2673 * all queues
2674 */
2675 arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
2676 bp->cp_nr_rings);
2677 size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
2678 bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
2679 if (!bnapi)
2680 return -ENOMEM;
2681
2682 bp->bnapi = bnapi;
2683 bnapi += arr_size;
2684 for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
2685 bp->bnapi[i] = bnapi;
2686 bp->bnapi[i]->index = i;
2687 bp->bnapi[i]->bp = bp;
2688 }
2689
2690 bp->rx_ring = kcalloc(bp->rx_nr_rings,
2691 sizeof(struct bnxt_rx_ring_info),
2692 GFP_KERNEL);
2693 if (!bp->rx_ring)
2694 return -ENOMEM;
2695
2696 for (i = 0; i < bp->rx_nr_rings; i++) {
2697 bp->rx_ring[i].bnapi = bp->bnapi[i];
2698 bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
2699 }
2700
2701 bp->tx_ring = kcalloc(bp->tx_nr_rings,
2702 sizeof(struct bnxt_tx_ring_info),
2703 GFP_KERNEL);
2704 if (!bp->tx_ring)
2705 return -ENOMEM;
2706
2707 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
2708 j = 0;
2709 else
2710 j = bp->rx_nr_rings;
2711
2712 for (i = 0; i < bp->tx_nr_rings; i++, j++) {
2713 bp->tx_ring[i].bnapi = bp->bnapi[j];
2714 bp->bnapi[j]->tx_ring = &bp->tx_ring[i];
2715 }
2716
2717 rc = bnxt_alloc_stats(bp);
2718 if (rc)
2719 goto alloc_mem_err;
2720
2721 rc = bnxt_alloc_ntp_fltrs(bp);
2722 if (rc)
2723 goto alloc_mem_err;
2724
2725 rc = bnxt_alloc_vnics(bp);
2726 if (rc)
2727 goto alloc_mem_err;
2728 }
2729
2730 bnxt_init_ring_struct(bp);
2731
2732 rc = bnxt_alloc_rx_rings(bp);
2733 if (rc)
2734 goto alloc_mem_err;
2735
2736 rc = bnxt_alloc_tx_rings(bp);
2737 if (rc)
2738 goto alloc_mem_err;
2739
2740 rc = bnxt_alloc_cp_rings(bp);
2741 if (rc)
2742 goto alloc_mem_err;
2743
2744 bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
2745 BNXT_VNIC_UCAST_FLAG;
2746 rc = bnxt_alloc_vnic_attributes(bp);
2747 if (rc)
2748 goto alloc_mem_err;
2749 return 0;
2750
2751 alloc_mem_err:
2752 bnxt_free_mem(bp, true);
2753 return rc;
2754 }
2755
2756 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
2757 u16 cmpl_ring, u16 target_id)
2758 {
2759 struct input *req = request;
2760
2761 req->req_type = cpu_to_le16(req_type);
2762 req->cmpl_ring = cpu_to_le16(cmpl_ring);
2763 req->target_id = cpu_to_le16(target_id);
2764 req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
2765 }
2766
2767 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
2768 int timeout, bool silent)
2769 {
2770 int i, intr_process, rc, tmo_count;
2771 struct input *req = msg;
2772 u32 *data = msg;
2773 __le32 *resp_len, *valid;
2774 u16 cp_ring_id, len = 0;
2775 struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
2776
2777 req->seq_id = cpu_to_le16(bp->hwrm_cmd_seq++);
2778 memset(resp, 0, PAGE_SIZE);
2779 cp_ring_id = le16_to_cpu(req->cmpl_ring);
2780 intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
2781
2782 /* Write request msg to hwrm channel */
2783 __iowrite32_copy(bp->bar0, data, msg_len / 4);
2784
2785 for (i = msg_len; i < BNXT_HWRM_MAX_REQ_LEN; i += 4)
2786 writel(0, bp->bar0 + i);
2787
2788 /* currently supports only one outstanding message */
2789 if (intr_process)
2790 bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
2791
2792 /* Ring channel doorbell */
2793 writel(1, bp->bar0 + 0x100);
2794
2795 if (!timeout)
2796 timeout = DFLT_HWRM_CMD_TIMEOUT;
2797
2798 i = 0;
2799 tmo_count = timeout * 40;
2800 if (intr_process) {
2801 /* Wait until hwrm response cmpl interrupt is processed */
2802 while (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID &&
2803 i++ < tmo_count) {
2804 usleep_range(25, 40);
2805 }
2806
2807 if (bp->hwrm_intr_seq_id != HWRM_SEQ_ID_INVALID) {
2808 netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
2809 le16_to_cpu(req->req_type));
2810 return -1;
2811 }
2812 } else {
2813 /* Check if response len is updated */
2814 resp_len = bp->hwrm_cmd_resp_addr + HWRM_RESP_LEN_OFFSET;
2815 for (i = 0; i < tmo_count; i++) {
2816 len = (le32_to_cpu(*resp_len) & HWRM_RESP_LEN_MASK) >>
2817 HWRM_RESP_LEN_SFT;
2818 if (len)
2819 break;
2820 usleep_range(25, 40);
2821 }
2822
2823 if (i >= tmo_count) {
2824 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
2825 timeout, le16_to_cpu(req->req_type),
2826 le16_to_cpu(req->seq_id), len);
2827 return -1;
2828 }
2829
2830 /* Last word of resp contains valid bit */
2831 valid = bp->hwrm_cmd_resp_addr + len - 4;
2832 for (i = 0; i < 5; i++) {
2833 if (le32_to_cpu(*valid) & HWRM_RESP_VALID_MASK)
2834 break;
2835 udelay(1);
2836 }
2837
2838 if (i >= 5) {
2839 netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
2840 timeout, le16_to_cpu(req->req_type),
2841 le16_to_cpu(req->seq_id), len, *valid);
2842 return -1;
2843 }
2844 }
2845
2846 rc = le16_to_cpu(resp->error_code);
2847 if (rc && !silent)
2848 netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
2849 le16_to_cpu(resp->req_type),
2850 le16_to_cpu(resp->seq_id), rc);
2851 return rc;
2852 }
2853
2854 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2855 {
2856 return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
2857 }
2858
2859 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
2860 {
2861 int rc;
2862
2863 mutex_lock(&bp->hwrm_cmd_lock);
2864 rc = _hwrm_send_message(bp, msg, msg_len, timeout);
2865 mutex_unlock(&bp->hwrm_cmd_lock);
2866 return rc;
2867 }
2868
2869 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
2870 int timeout)
2871 {
2872 int rc;
2873
2874 mutex_lock(&bp->hwrm_cmd_lock);
2875 rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
2876 mutex_unlock(&bp->hwrm_cmd_lock);
2877 return rc;
2878 }
2879
2880 static int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp)
2881 {
2882 struct hwrm_func_drv_rgtr_input req = {0};
2883 int i;
2884 DECLARE_BITMAP(async_events_bmap, 256);
2885 u32 *events = (u32 *)async_events_bmap;
2886
2887 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
2888
2889 req.enables =
2890 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
2891 FUNC_DRV_RGTR_REQ_ENABLES_VER |
2892 FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
2893
2894 memset(async_events_bmap, 0, sizeof(async_events_bmap));
2895 for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++)
2896 __set_bit(bnxt_async_events_arr[i], async_events_bmap);
2897
2898 for (i = 0; i < 8; i++)
2899 req.async_event_fwd[i] |= cpu_to_le32(events[i]);
2900
2901 req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
2902 req.ver_maj = DRV_VER_MAJ;
2903 req.ver_min = DRV_VER_MIN;
2904 req.ver_upd = DRV_VER_UPD;
2905
2906 if (BNXT_PF(bp)) {
2907 DECLARE_BITMAP(vf_req_snif_bmap, 256);
2908 u32 *data = (u32 *)vf_req_snif_bmap;
2909
2910 memset(vf_req_snif_bmap, 0, sizeof(vf_req_snif_bmap));
2911 for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++)
2912 __set_bit(bnxt_vf_req_snif[i], vf_req_snif_bmap);
2913
2914 for (i = 0; i < 8; i++)
2915 req.vf_req_fwd[i] = cpu_to_le32(data[i]);
2916
2917 req.enables |=
2918 cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
2919 }
2920
2921 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2922 }
2923
2924 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
2925 {
2926 struct hwrm_func_drv_unrgtr_input req = {0};
2927
2928 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
2929 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2930 }
2931
2932 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
2933 {
2934 u32 rc = 0;
2935 struct hwrm_tunnel_dst_port_free_input req = {0};
2936
2937 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
2938 req.tunnel_type = tunnel_type;
2939
2940 switch (tunnel_type) {
2941 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
2942 req.tunnel_dst_port_id = bp->vxlan_fw_dst_port_id;
2943 break;
2944 case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
2945 req.tunnel_dst_port_id = bp->nge_fw_dst_port_id;
2946 break;
2947 default:
2948 break;
2949 }
2950
2951 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2952 if (rc)
2953 netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
2954 rc);
2955 return rc;
2956 }
2957
2958 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
2959 u8 tunnel_type)
2960 {
2961 u32 rc = 0;
2962 struct hwrm_tunnel_dst_port_alloc_input req = {0};
2963 struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
2964
2965 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
2966
2967 req.tunnel_type = tunnel_type;
2968 req.tunnel_dst_port_val = port;
2969
2970 mutex_lock(&bp->hwrm_cmd_lock);
2971 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
2972 if (rc) {
2973 netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
2974 rc);
2975 goto err_out;
2976 }
2977
2978 if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN)
2979 bp->vxlan_fw_dst_port_id = resp->tunnel_dst_port_id;
2980
2981 else if (tunnel_type & TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE)
2982 bp->nge_fw_dst_port_id = resp->tunnel_dst_port_id;
2983 err_out:
2984 mutex_unlock(&bp->hwrm_cmd_lock);
2985 return rc;
2986 }
2987
2988 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
2989 {
2990 struct hwrm_cfa_l2_set_rx_mask_input req = {0};
2991 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2992
2993 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
2994 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
2995
2996 req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
2997 req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
2998 req.mask = cpu_to_le32(vnic->rx_mask);
2999 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3000 }
3001
3002 #ifdef CONFIG_RFS_ACCEL
3003 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
3004 struct bnxt_ntuple_filter *fltr)
3005 {
3006 struct hwrm_cfa_ntuple_filter_free_input req = {0};
3007
3008 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
3009 req.ntuple_filter_id = fltr->filter_id;
3010 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3011 }
3012
3013 #define BNXT_NTP_FLTR_FLAGS \
3014 (CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID | \
3015 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE | \
3016 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR | \
3017 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE | \
3018 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR | \
3019 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK | \
3020 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR | \
3021 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK | \
3022 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL | \
3023 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT | \
3024 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK | \
3025 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT | \
3026 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK | \
3027 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
3028
3029 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
3030 struct bnxt_ntuple_filter *fltr)
3031 {
3032 int rc = 0;
3033 struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
3034 struct hwrm_cfa_ntuple_filter_alloc_output *resp =
3035 bp->hwrm_cmd_resp_addr;
3036 struct flow_keys *keys = &fltr->fkeys;
3037 struct bnxt_vnic_info *vnic = &bp->vnic_info[fltr->rxq + 1];
3038
3039 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
3040 req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[0];
3041
3042 req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
3043
3044 req.ethertype = htons(ETH_P_IP);
3045 memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
3046 req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
3047 req.ip_protocol = keys->basic.ip_proto;
3048
3049 req.src_ipaddr[0] = keys->addrs.v4addrs.src;
3050 req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3051 req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
3052 req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
3053
3054 req.src_port = keys->ports.src;
3055 req.src_port_mask = cpu_to_be16(0xffff);
3056 req.dst_port = keys->ports.dst;
3057 req.dst_port_mask = cpu_to_be16(0xffff);
3058
3059 req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
3060 mutex_lock(&bp->hwrm_cmd_lock);
3061 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3062 if (!rc)
3063 fltr->filter_id = resp->ntuple_filter_id;
3064 mutex_unlock(&bp->hwrm_cmd_lock);
3065 return rc;
3066 }
3067 #endif
3068
3069 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
3070 u8 *mac_addr)
3071 {
3072 u32 rc = 0;
3073 struct hwrm_cfa_l2_filter_alloc_input req = {0};
3074 struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3075
3076 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
3077 req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX |
3078 CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
3079 req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
3080 req.enables =
3081 cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
3082 CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
3083 CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
3084 memcpy(req.l2_addr, mac_addr, ETH_ALEN);
3085 req.l2_addr_mask[0] = 0xff;
3086 req.l2_addr_mask[1] = 0xff;
3087 req.l2_addr_mask[2] = 0xff;
3088 req.l2_addr_mask[3] = 0xff;
3089 req.l2_addr_mask[4] = 0xff;
3090 req.l2_addr_mask[5] = 0xff;
3091
3092 mutex_lock(&bp->hwrm_cmd_lock);
3093 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3094 if (!rc)
3095 bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
3096 resp->l2_filter_id;
3097 mutex_unlock(&bp->hwrm_cmd_lock);
3098 return rc;
3099 }
3100
3101 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
3102 {
3103 u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
3104 int rc = 0;
3105
3106 /* Any associated ntuple filters will also be cleared by firmware. */
3107 mutex_lock(&bp->hwrm_cmd_lock);
3108 for (i = 0; i < num_of_vnics; i++) {
3109 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3110
3111 for (j = 0; j < vnic->uc_filter_count; j++) {
3112 struct hwrm_cfa_l2_filter_free_input req = {0};
3113
3114 bnxt_hwrm_cmd_hdr_init(bp, &req,
3115 HWRM_CFA_L2_FILTER_FREE, -1, -1);
3116
3117 req.l2_filter_id = vnic->fw_l2_filter_id[j];
3118
3119 rc = _hwrm_send_message(bp, &req, sizeof(req),
3120 HWRM_CMD_TIMEOUT);
3121 }
3122 vnic->uc_filter_count = 0;
3123 }
3124 mutex_unlock(&bp->hwrm_cmd_lock);
3125
3126 return rc;
3127 }
3128
3129 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
3130 {
3131 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3132 struct hwrm_vnic_tpa_cfg_input req = {0};
3133
3134 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
3135
3136 if (tpa_flags) {
3137 u16 mss = bp->dev->mtu - 40;
3138 u32 nsegs, n, segs = 0, flags;
3139
3140 flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
3141 VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
3142 VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
3143 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
3144 VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
3145 if (tpa_flags & BNXT_FLAG_GRO)
3146 flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
3147
3148 req.flags = cpu_to_le32(flags);
3149
3150 req.enables =
3151 cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
3152 VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
3153 VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
3154
3155 /* Number of segs are log2 units, and first packet is not
3156 * included as part of this units.
3157 */
3158 if (mss <= BNXT_RX_PAGE_SIZE) {
3159 n = BNXT_RX_PAGE_SIZE / mss;
3160 nsegs = (MAX_SKB_FRAGS - 1) * n;
3161 } else {
3162 n = mss / BNXT_RX_PAGE_SIZE;
3163 if (mss & (BNXT_RX_PAGE_SIZE - 1))
3164 n++;
3165 nsegs = (MAX_SKB_FRAGS - n) / n;
3166 }
3167
3168 segs = ilog2(nsegs);
3169 req.max_agg_segs = cpu_to_le16(segs);
3170 req.max_aggs = cpu_to_le16(VNIC_TPA_CFG_REQ_MAX_AGGS_MAX);
3171
3172 req.min_agg_len = cpu_to_le32(512);
3173 }
3174 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3175
3176 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3177 }
3178
3179 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
3180 {
3181 u32 i, j, max_rings;
3182 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3183 struct hwrm_vnic_rss_cfg_input req = {0};
3184
3185 if (vnic->fw_rss_cos_lb_ctx == INVALID_HW_RING_ID)
3186 return 0;
3187
3188 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
3189 if (set_rss) {
3190 vnic->hash_type = BNXT_RSS_HASH_TYPE_FLAG_IPV4 |
3191 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV4 |
3192 BNXT_RSS_HASH_TYPE_FLAG_IPV6 |
3193 BNXT_RSS_HASH_TYPE_FLAG_TCP_IPV6;
3194
3195 req.hash_type = cpu_to_le32(vnic->hash_type);
3196
3197 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3198 max_rings = bp->rx_nr_rings;
3199 else
3200 max_rings = 1;
3201
3202 /* Fill the RSS indirection table with ring group ids */
3203 for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++, j++) {
3204 if (j == max_rings)
3205 j = 0;
3206 vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
3207 }
3208
3209 req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
3210 req.hash_key_tbl_addr =
3211 cpu_to_le64(vnic->rss_hash_key_dma_addr);
3212 }
3213 req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3214 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3215 }
3216
3217 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
3218 {
3219 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3220 struct hwrm_vnic_plcmodes_cfg_input req = {0};
3221
3222 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
3223 req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
3224 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
3225 VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
3226 req.enables =
3227 cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
3228 VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
3229 /* thresholds not implemented in firmware yet */
3230 req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
3231 req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
3232 req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
3233 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3234 }
3235
3236 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id)
3237 {
3238 struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
3239
3240 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
3241 req.rss_cos_lb_ctx_id =
3242 cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx);
3243
3244 hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3245 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3246 }
3247
3248 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
3249 {
3250 int i;
3251
3252 for (i = 0; i < bp->nr_vnics; i++) {
3253 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3254
3255 if (vnic->fw_rss_cos_lb_ctx != INVALID_HW_RING_ID)
3256 bnxt_hwrm_vnic_ctx_free_one(bp, i);
3257 }
3258 bp->rsscos_nr_ctxs = 0;
3259 }
3260
3261 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id)
3262 {
3263 int rc;
3264 struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
3265 struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
3266 bp->hwrm_cmd_resp_addr;
3267
3268 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
3269 -1);
3270
3271 mutex_lock(&bp->hwrm_cmd_lock);
3272 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3273 if (!rc)
3274 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx =
3275 le16_to_cpu(resp->rss_cos_lb_ctx_id);
3276 mutex_unlock(&bp->hwrm_cmd_lock);
3277
3278 return rc;
3279 }
3280
3281 static int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
3282 {
3283 unsigned int ring = 0, grp_idx;
3284 struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
3285 struct hwrm_vnic_cfg_input req = {0};
3286 u16 def_vlan = 0;
3287
3288 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
3289 /* Only RSS support for now TBD: COS & LB */
3290 req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP |
3291 VNIC_CFG_REQ_ENABLES_RSS_RULE);
3292 req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx);
3293 req.cos_rule = cpu_to_le16(0xffff);
3294 if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3295 ring = 0;
3296 else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
3297 ring = vnic_id - 1;
3298
3299 grp_idx = bp->rx_ring[ring].bnapi->index;
3300 req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
3301 req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
3302
3303 req.lb_rule = cpu_to_le16(0xffff);
3304 req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + ETH_FCS_LEN +
3305 VLAN_HLEN);
3306
3307 #ifdef CONFIG_BNXT_SRIOV
3308 if (BNXT_VF(bp))
3309 def_vlan = bp->vf.vlan;
3310 #endif
3311 if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
3312 req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
3313
3314 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3315 }
3316
3317 static int bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
3318 {
3319 u32 rc = 0;
3320
3321 if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
3322 struct hwrm_vnic_free_input req = {0};
3323
3324 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
3325 req.vnic_id =
3326 cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
3327
3328 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3329 if (rc)
3330 return rc;
3331 bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
3332 }
3333 return rc;
3334 }
3335
3336 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
3337 {
3338 u16 i;
3339
3340 for (i = 0; i < bp->nr_vnics; i++)
3341 bnxt_hwrm_vnic_free_one(bp, i);
3342 }
3343
3344 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
3345 unsigned int start_rx_ring_idx,
3346 unsigned int nr_rings)
3347 {
3348 int rc = 0;
3349 unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
3350 struct hwrm_vnic_alloc_input req = {0};
3351 struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3352
3353 /* map ring groups to this vnic */
3354 for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
3355 grp_idx = bp->rx_ring[i].bnapi->index;
3356 if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
3357 netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
3358 j, nr_rings);
3359 break;
3360 }
3361 bp->vnic_info[vnic_id].fw_grp_ids[j] =
3362 bp->grp_info[grp_idx].fw_grp_id;
3363 }
3364
3365 bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx = INVALID_HW_RING_ID;
3366 if (vnic_id == 0)
3367 req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
3368
3369 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
3370
3371 mutex_lock(&bp->hwrm_cmd_lock);
3372 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3373 if (!rc)
3374 bp->vnic_info[vnic_id].fw_vnic_id = le32_to_cpu(resp->vnic_id);
3375 mutex_unlock(&bp->hwrm_cmd_lock);
3376 return rc;
3377 }
3378
3379 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
3380 {
3381 u16 i;
3382 u32 rc = 0;
3383
3384 mutex_lock(&bp->hwrm_cmd_lock);
3385 for (i = 0; i < bp->rx_nr_rings; i++) {
3386 struct hwrm_ring_grp_alloc_input req = {0};
3387 struct hwrm_ring_grp_alloc_output *resp =
3388 bp->hwrm_cmd_resp_addr;
3389 unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
3390
3391 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
3392
3393 req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
3394 req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
3395 req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
3396 req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
3397
3398 rc = _hwrm_send_message(bp, &req, sizeof(req),
3399 HWRM_CMD_TIMEOUT);
3400 if (rc)
3401 break;
3402
3403 bp->grp_info[grp_idx].fw_grp_id =
3404 le32_to_cpu(resp->ring_group_id);
3405 }
3406 mutex_unlock(&bp->hwrm_cmd_lock);
3407 return rc;
3408 }
3409
3410 static int bnxt_hwrm_ring_grp_free(struct bnxt *bp)
3411 {
3412 u16 i;
3413 u32 rc = 0;
3414 struct hwrm_ring_grp_free_input req = {0};
3415
3416 if (!bp->grp_info)
3417 return 0;
3418
3419 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
3420
3421 mutex_lock(&bp->hwrm_cmd_lock);
3422 for (i = 0; i < bp->cp_nr_rings; i++) {
3423 if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
3424 continue;
3425 req.ring_group_id =
3426 cpu_to_le32(bp->grp_info[i].fw_grp_id);
3427
3428 rc = _hwrm_send_message(bp, &req, sizeof(req),
3429 HWRM_CMD_TIMEOUT);
3430 if (rc)
3431 break;
3432 bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3433 }
3434 mutex_unlock(&bp->hwrm_cmd_lock);
3435 return rc;
3436 }
3437
3438 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
3439 struct bnxt_ring_struct *ring,
3440 u32 ring_type, u32 map_index,
3441 u32 stats_ctx_id)
3442 {
3443 int rc = 0, err = 0;
3444 struct hwrm_ring_alloc_input req = {0};
3445 struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3446 u16 ring_id;
3447
3448 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
3449
3450 req.enables = 0;
3451 if (ring->nr_pages > 1) {
3452 req.page_tbl_addr = cpu_to_le64(ring->pg_tbl_map);
3453 /* Page size is in log2 units */
3454 req.page_size = BNXT_PAGE_SHIFT;
3455 req.page_tbl_depth = 1;
3456 } else {
3457 req.page_tbl_addr = cpu_to_le64(ring->dma_arr[0]);
3458 }
3459 req.fbo = 0;
3460 /* Association of ring index with doorbell index and MSIX number */
3461 req.logical_id = cpu_to_le16(map_index);
3462
3463 switch (ring_type) {
3464 case HWRM_RING_ALLOC_TX:
3465 req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
3466 /* Association of transmit ring with completion ring */
3467 req.cmpl_ring_id =
3468 cpu_to_le16(bp->grp_info[map_index].cp_fw_ring_id);
3469 req.length = cpu_to_le32(bp->tx_ring_mask + 1);
3470 req.stat_ctx_id = cpu_to_le32(stats_ctx_id);
3471 req.queue_id = cpu_to_le16(ring->queue_id);
3472 break;
3473 case HWRM_RING_ALLOC_RX:
3474 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3475 req.length = cpu_to_le32(bp->rx_ring_mask + 1);
3476 break;
3477 case HWRM_RING_ALLOC_AGG:
3478 req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
3479 req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
3480 break;
3481 case HWRM_RING_ALLOC_CMPL:
3482 req.ring_type = RING_ALLOC_REQ_RING_TYPE_CMPL;
3483 req.length = cpu_to_le32(bp->cp_ring_mask + 1);
3484 if (bp->flags & BNXT_FLAG_USING_MSIX)
3485 req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
3486 break;
3487 default:
3488 netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
3489 ring_type);
3490 return -1;
3491 }
3492
3493 mutex_lock(&bp->hwrm_cmd_lock);
3494 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3495 err = le16_to_cpu(resp->error_code);
3496 ring_id = le16_to_cpu(resp->ring_id);
3497 mutex_unlock(&bp->hwrm_cmd_lock);
3498
3499 if (rc || err) {
3500 switch (ring_type) {
3501 case RING_FREE_REQ_RING_TYPE_CMPL:
3502 netdev_err(bp->dev, "hwrm_ring_alloc cp failed. rc:%x err:%x\n",
3503 rc, err);
3504 return -1;
3505
3506 case RING_FREE_REQ_RING_TYPE_RX:
3507 netdev_err(bp->dev, "hwrm_ring_alloc rx failed. rc:%x err:%x\n",
3508 rc, err);
3509 return -1;
3510
3511 case RING_FREE_REQ_RING_TYPE_TX:
3512 netdev_err(bp->dev, "hwrm_ring_alloc tx failed. rc:%x err:%x\n",
3513 rc, err);
3514 return -1;
3515
3516 default:
3517 netdev_err(bp->dev, "Invalid ring\n");
3518 return -1;
3519 }
3520 }
3521 ring->fw_ring_id = ring_id;
3522 return rc;
3523 }
3524
3525 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
3526 {
3527 int i, rc = 0;
3528
3529 for (i = 0; i < bp->cp_nr_rings; i++) {
3530 struct bnxt_napi *bnapi = bp->bnapi[i];
3531 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3532 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3533
3534 cpr->cp_doorbell = bp->bar1 + i * 0x80;
3535 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_CMPL, i,
3536 INVALID_STATS_CTX_ID);
3537 if (rc)
3538 goto err_out;
3539 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
3540 bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
3541 }
3542
3543 for (i = 0; i < bp->tx_nr_rings; i++) {
3544 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3545 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3546 u32 map_idx = txr->bnapi->index;
3547 u16 fw_stats_ctx = bp->grp_info[map_idx].fw_stats_ctx;
3548
3549 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_TX,
3550 map_idx, fw_stats_ctx);
3551 if (rc)
3552 goto err_out;
3553 txr->tx_doorbell = bp->bar1 + map_idx * 0x80;
3554 }
3555
3556 for (i = 0; i < bp->rx_nr_rings; i++) {
3557 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3558 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3559 u32 map_idx = rxr->bnapi->index;
3560
3561 rc = hwrm_ring_alloc_send_msg(bp, ring, HWRM_RING_ALLOC_RX,
3562 map_idx, INVALID_STATS_CTX_ID);
3563 if (rc)
3564 goto err_out;
3565 rxr->rx_doorbell = bp->bar1 + map_idx * 0x80;
3566 writel(DB_KEY_RX | rxr->rx_prod, rxr->rx_doorbell);
3567 bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
3568 }
3569
3570 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
3571 for (i = 0; i < bp->rx_nr_rings; i++) {
3572 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3573 struct bnxt_ring_struct *ring =
3574 &rxr->rx_agg_ring_struct;
3575 u32 grp_idx = rxr->bnapi->index;
3576 u32 map_idx = grp_idx + bp->rx_nr_rings;
3577
3578 rc = hwrm_ring_alloc_send_msg(bp, ring,
3579 HWRM_RING_ALLOC_AGG,
3580 map_idx,
3581 INVALID_STATS_CTX_ID);
3582 if (rc)
3583 goto err_out;
3584
3585 rxr->rx_agg_doorbell = bp->bar1 + map_idx * 0x80;
3586 writel(DB_KEY_RX | rxr->rx_agg_prod,
3587 rxr->rx_agg_doorbell);
3588 bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
3589 }
3590 }
3591 err_out:
3592 return rc;
3593 }
3594
3595 static int hwrm_ring_free_send_msg(struct bnxt *bp,
3596 struct bnxt_ring_struct *ring,
3597 u32 ring_type, int cmpl_ring_id)
3598 {
3599 int rc;
3600 struct hwrm_ring_free_input req = {0};
3601 struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
3602 u16 error_code;
3603
3604 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
3605 req.ring_type = ring_type;
3606 req.ring_id = cpu_to_le16(ring->fw_ring_id);
3607
3608 mutex_lock(&bp->hwrm_cmd_lock);
3609 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3610 error_code = le16_to_cpu(resp->error_code);
3611 mutex_unlock(&bp->hwrm_cmd_lock);
3612
3613 if (rc || error_code) {
3614 switch (ring_type) {
3615 case RING_FREE_REQ_RING_TYPE_CMPL:
3616 netdev_err(bp->dev, "hwrm_ring_free cp failed. rc:%d\n",
3617 rc);
3618 return rc;
3619 case RING_FREE_REQ_RING_TYPE_RX:
3620 netdev_err(bp->dev, "hwrm_ring_free rx failed. rc:%d\n",
3621 rc);
3622 return rc;
3623 case RING_FREE_REQ_RING_TYPE_TX:
3624 netdev_err(bp->dev, "hwrm_ring_free tx failed. rc:%d\n",
3625 rc);
3626 return rc;
3627 default:
3628 netdev_err(bp->dev, "Invalid ring\n");
3629 return -1;
3630 }
3631 }
3632 return 0;
3633 }
3634
3635 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
3636 {
3637 int i;
3638
3639 if (!bp->bnapi)
3640 return;
3641
3642 for (i = 0; i < bp->tx_nr_rings; i++) {
3643 struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3644 struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3645 u32 grp_idx = txr->bnapi->index;
3646 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3647
3648 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3649 hwrm_ring_free_send_msg(bp, ring,
3650 RING_FREE_REQ_RING_TYPE_TX,
3651 close_path ? cmpl_ring_id :
3652 INVALID_HW_RING_ID);
3653 ring->fw_ring_id = INVALID_HW_RING_ID;
3654 }
3655 }
3656
3657 for (i = 0; i < bp->rx_nr_rings; i++) {
3658 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3659 struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
3660 u32 grp_idx = rxr->bnapi->index;
3661 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3662
3663 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3664 hwrm_ring_free_send_msg(bp, ring,
3665 RING_FREE_REQ_RING_TYPE_RX,
3666 close_path ? cmpl_ring_id :
3667 INVALID_HW_RING_ID);
3668 ring->fw_ring_id = INVALID_HW_RING_ID;
3669 bp->grp_info[grp_idx].rx_fw_ring_id =
3670 INVALID_HW_RING_ID;
3671 }
3672 }
3673
3674 for (i = 0; i < bp->rx_nr_rings; i++) {
3675 struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3676 struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
3677 u32 grp_idx = rxr->bnapi->index;
3678 u32 cmpl_ring_id = bp->grp_info[grp_idx].cp_fw_ring_id;
3679
3680 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3681 hwrm_ring_free_send_msg(bp, ring,
3682 RING_FREE_REQ_RING_TYPE_RX,
3683 close_path ? cmpl_ring_id :
3684 INVALID_HW_RING_ID);
3685 ring->fw_ring_id = INVALID_HW_RING_ID;
3686 bp->grp_info[grp_idx].agg_fw_ring_id =
3687 INVALID_HW_RING_ID;
3688 }
3689 }
3690
3691 for (i = 0; i < bp->cp_nr_rings; i++) {
3692 struct bnxt_napi *bnapi = bp->bnapi[i];
3693 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3694 struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3695
3696 if (ring->fw_ring_id != INVALID_HW_RING_ID) {
3697 hwrm_ring_free_send_msg(bp, ring,
3698 RING_FREE_REQ_RING_TYPE_CMPL,
3699 INVALID_HW_RING_ID);
3700 ring->fw_ring_id = INVALID_HW_RING_ID;
3701 bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3702 }
3703 }
3704 }
3705
3706 static void bnxt_hwrm_set_coal_params(struct bnxt *bp, u32 max_bufs,
3707 u32 buf_tmrs, u16 flags,
3708 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
3709 {
3710 req->flags = cpu_to_le16(flags);
3711 req->num_cmpl_dma_aggr = cpu_to_le16((u16)max_bufs);
3712 req->num_cmpl_dma_aggr_during_int = cpu_to_le16(max_bufs >> 16);
3713 req->cmpl_aggr_dma_tmr = cpu_to_le16((u16)buf_tmrs);
3714 req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(buf_tmrs >> 16);
3715 /* Minimum time between 2 interrupts set to buf_tmr x 2 */
3716 req->int_lat_tmr_min = cpu_to_le16((u16)buf_tmrs * 2);
3717 req->int_lat_tmr_max = cpu_to_le16((u16)buf_tmrs * 4);
3718 req->num_cmpl_aggr_int = cpu_to_le16((u16)max_bufs * 4);
3719 }
3720
3721 int bnxt_hwrm_set_coal(struct bnxt *bp)
3722 {
3723 int i, rc = 0;
3724 struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
3725 req_tx = {0}, *req;
3726 u16 max_buf, max_buf_irq;
3727 u16 buf_tmr, buf_tmr_irq;
3728 u32 flags;
3729
3730 bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
3731 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3732 bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
3733 HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
3734
3735 /* Each rx completion (2 records) should be DMAed immediately.
3736 * DMA 1/4 of the completion buffers at a time.
3737 */
3738 max_buf = min_t(u16, bp->rx_coal_bufs / 4, 2);
3739 /* max_buf must not be zero */
3740 max_buf = clamp_t(u16, max_buf, 1, 63);
3741 max_buf_irq = clamp_t(u16, bp->rx_coal_bufs_irq, 1, 63);
3742 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks);
3743 /* buf timer set to 1/4 of interrupt timer */
3744 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3745 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->rx_coal_ticks_irq);
3746 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3747
3748 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3749
3750 /* RING_IDLE generates more IRQs for lower latency. Enable it only
3751 * if coal_ticks is less than 25 us.
3752 */
3753 if (bp->rx_coal_ticks < 25)
3754 flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
3755
3756 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3757 buf_tmr_irq << 16 | buf_tmr, flags, &req_rx);
3758
3759 /* max_buf must not be zero */
3760 max_buf = clamp_t(u16, bp->tx_coal_bufs, 1, 63);
3761 max_buf_irq = clamp_t(u16, bp->tx_coal_bufs_irq, 1, 63);
3762 buf_tmr = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks);
3763 /* buf timer set to 1/4 of interrupt timer */
3764 buf_tmr = max_t(u16, buf_tmr / 4, 1);
3765 buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(bp->tx_coal_ticks_irq);
3766 buf_tmr_irq = max_t(u16, buf_tmr_irq, 1);
3767
3768 flags = RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
3769 bnxt_hwrm_set_coal_params(bp, max_buf_irq << 16 | max_buf,
3770 buf_tmr_irq << 16 | buf_tmr, flags, &req_tx);
3771
3772 mutex_lock(&bp->hwrm_cmd_lock);
3773 for (i = 0; i < bp->cp_nr_rings; i++) {
3774 struct bnxt_napi *bnapi = bp->bnapi[i];
3775
3776 req = &req_rx;
3777 if (!bnapi->rx_ring)
3778 req = &req_tx;
3779 req->ring_id = cpu_to_le16(bp->grp_info[i].cp_fw_ring_id);
3780
3781 rc = _hwrm_send_message(bp, req, sizeof(*req),
3782 HWRM_CMD_TIMEOUT);
3783 if (rc)
3784 break;
3785 }
3786 mutex_unlock(&bp->hwrm_cmd_lock);
3787 return rc;
3788 }
3789
3790 static int bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
3791 {
3792 int rc = 0, i;
3793 struct hwrm_stat_ctx_free_input req = {0};
3794
3795 if (!bp->bnapi)
3796 return 0;
3797
3798 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
3799
3800 mutex_lock(&bp->hwrm_cmd_lock);
3801 for (i = 0; i < bp->cp_nr_rings; i++) {
3802 struct bnxt_napi *bnapi = bp->bnapi[i];
3803 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3804
3805 if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
3806 req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
3807
3808 rc = _hwrm_send_message(bp, &req, sizeof(req),
3809 HWRM_CMD_TIMEOUT);
3810 if (rc)
3811 break;
3812
3813 cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3814 }
3815 }
3816 mutex_unlock(&bp->hwrm_cmd_lock);
3817 return rc;
3818 }
3819
3820 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
3821 {
3822 int rc = 0, i;
3823 struct hwrm_stat_ctx_alloc_input req = {0};
3824 struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
3825
3826 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
3827
3828 req.update_period_ms = cpu_to_le32(1000);
3829
3830 mutex_lock(&bp->hwrm_cmd_lock);
3831 for (i = 0; i < bp->cp_nr_rings; i++) {
3832 struct bnxt_napi *bnapi = bp->bnapi[i];
3833 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3834
3835 req.stats_dma_addr = cpu_to_le64(cpr->hw_stats_map);
3836
3837 rc = _hwrm_send_message(bp, &req, sizeof(req),
3838 HWRM_CMD_TIMEOUT);
3839 if (rc)
3840 break;
3841
3842 cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
3843
3844 bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
3845 }
3846 mutex_unlock(&bp->hwrm_cmd_lock);
3847 return 0;
3848 }
3849
3850 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
3851 {
3852 struct hwrm_func_qcfg_input req = {0};
3853 struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
3854 int rc;
3855
3856 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
3857 req.fid = cpu_to_le16(0xffff);
3858 mutex_lock(&bp->hwrm_cmd_lock);
3859 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3860 if (rc)
3861 goto func_qcfg_exit;
3862
3863 #ifdef CONFIG_BNXT_SRIOV
3864 if (BNXT_VF(bp)) {
3865 struct bnxt_vf_info *vf = &bp->vf;
3866
3867 vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
3868 }
3869 #endif
3870 switch (resp->port_partition_type) {
3871 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
3872 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
3873 case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
3874 bp->port_partition_type = resp->port_partition_type;
3875 break;
3876 }
3877
3878 func_qcfg_exit:
3879 mutex_unlock(&bp->hwrm_cmd_lock);
3880 return rc;
3881 }
3882
3883 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
3884 {
3885 int rc = 0;
3886 struct hwrm_func_qcaps_input req = {0};
3887 struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
3888
3889 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
3890 req.fid = cpu_to_le16(0xffff);
3891
3892 mutex_lock(&bp->hwrm_cmd_lock);
3893 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3894 if (rc)
3895 goto hwrm_func_qcaps_exit;
3896
3897 if (BNXT_PF(bp)) {
3898 struct bnxt_pf_info *pf = &bp->pf;
3899
3900 pf->fw_fid = le16_to_cpu(resp->fid);
3901 pf->port_id = le16_to_cpu(resp->port_id);
3902 memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
3903 memcpy(bp->dev->dev_addr, pf->mac_addr, ETH_ALEN);
3904 pf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3905 pf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3906 pf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3907 pf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3908 pf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3909 if (!pf->max_hw_ring_grps)
3910 pf->max_hw_ring_grps = pf->max_tx_rings;
3911 pf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3912 pf->max_vnics = le16_to_cpu(resp->max_vnics);
3913 pf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3914 pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
3915 pf->max_vfs = le16_to_cpu(resp->max_vfs);
3916 pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
3917 pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
3918 pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
3919 pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
3920 pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
3921 pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
3922 } else {
3923 #ifdef CONFIG_BNXT_SRIOV
3924 struct bnxt_vf_info *vf = &bp->vf;
3925
3926 vf->fw_fid = le16_to_cpu(resp->fid);
3927 memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
3928 if (is_valid_ether_addr(vf->mac_addr))
3929 /* overwrite netdev dev_adr with admin VF MAC */
3930 memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
3931 else
3932 random_ether_addr(bp->dev->dev_addr);
3933
3934 vf->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
3935 vf->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
3936 vf->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
3937 vf->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
3938 vf->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
3939 if (!vf->max_hw_ring_grps)
3940 vf->max_hw_ring_grps = vf->max_tx_rings;
3941 vf->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
3942 vf->max_vnics = le16_to_cpu(resp->max_vnics);
3943 vf->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
3944 #endif
3945 }
3946
3947 bp->tx_push_thresh = 0;
3948 if (resp->flags &
3949 cpu_to_le32(FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED))
3950 bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
3951
3952 hwrm_func_qcaps_exit:
3953 mutex_unlock(&bp->hwrm_cmd_lock);
3954 return rc;
3955 }
3956
3957 static int bnxt_hwrm_func_reset(struct bnxt *bp)
3958 {
3959 struct hwrm_func_reset_input req = {0};
3960
3961 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
3962 req.enables = 0;
3963
3964 return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
3965 }
3966
3967 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
3968 {
3969 int rc = 0;
3970 struct hwrm_queue_qportcfg_input req = {0};
3971 struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
3972 u8 i, *qptr;
3973
3974 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
3975
3976 mutex_lock(&bp->hwrm_cmd_lock);
3977 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3978 if (rc)
3979 goto qportcfg_exit;
3980
3981 if (!resp->max_configurable_queues) {
3982 rc = -EINVAL;
3983 goto qportcfg_exit;
3984 }
3985 bp->max_tc = resp->max_configurable_queues;
3986 if (bp->max_tc > BNXT_MAX_QUEUE)
3987 bp->max_tc = BNXT_MAX_QUEUE;
3988
3989 qptr = &resp->queue_id0;
3990 for (i = 0; i < bp->max_tc; i++) {
3991 bp->q_info[i].queue_id = *qptr++;
3992 bp->q_info[i].queue_profile = *qptr++;
3993 }
3994
3995 qportcfg_exit:
3996 mutex_unlock(&bp->hwrm_cmd_lock);
3997 return rc;
3998 }
3999
4000 static int bnxt_hwrm_ver_get(struct bnxt *bp)
4001 {
4002 int rc;
4003 struct hwrm_ver_get_input req = {0};
4004 struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
4005
4006 bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
4007 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
4008 req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
4009 req.hwrm_intf_min = HWRM_VERSION_MINOR;
4010 req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
4011 mutex_lock(&bp->hwrm_cmd_lock);
4012 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4013 if (rc)
4014 goto hwrm_ver_get_exit;
4015
4016 memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
4017
4018 bp->hwrm_spec_code = resp->hwrm_intf_maj << 16 |
4019 resp->hwrm_intf_min << 8 | resp->hwrm_intf_upd;
4020 if (resp->hwrm_intf_maj < 1) {
4021 netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
4022 resp->hwrm_intf_maj, resp->hwrm_intf_min,
4023 resp->hwrm_intf_upd);
4024 netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
4025 }
4026 snprintf(bp->fw_ver_str, BC_HWRM_STR_LEN, "%d.%d.%d/%d.%d.%d",
4027 resp->hwrm_fw_maj, resp->hwrm_fw_min, resp->hwrm_fw_bld,
4028 resp->hwrm_intf_maj, resp->hwrm_intf_min, resp->hwrm_intf_upd);
4029
4030 bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
4031 if (!bp->hwrm_cmd_timeout)
4032 bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
4033
4034 if (resp->hwrm_intf_maj >= 1)
4035 bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
4036
4037 hwrm_ver_get_exit:
4038 mutex_unlock(&bp->hwrm_cmd_lock);
4039 return rc;
4040 }
4041
4042 static int bnxt_hwrm_port_qstats(struct bnxt *bp)
4043 {
4044 int rc;
4045 struct bnxt_pf_info *pf = &bp->pf;
4046 struct hwrm_port_qstats_input req = {0};
4047
4048 if (!(bp->flags & BNXT_FLAG_PORT_STATS))
4049 return 0;
4050
4051 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
4052 req.port_id = cpu_to_le16(pf->port_id);
4053 req.tx_stat_host_addr = cpu_to_le64(bp->hw_tx_port_stats_map);
4054 req.rx_stat_host_addr = cpu_to_le64(bp->hw_rx_port_stats_map);
4055 rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4056 return rc;
4057 }
4058
4059 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
4060 {
4061 if (bp->vxlan_port_cnt) {
4062 bnxt_hwrm_tunnel_dst_port_free(
4063 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
4064 }
4065 bp->vxlan_port_cnt = 0;
4066 if (bp->nge_port_cnt) {
4067 bnxt_hwrm_tunnel_dst_port_free(
4068 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
4069 }
4070 bp->nge_port_cnt = 0;
4071 }
4072
4073 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
4074 {
4075 int rc, i;
4076 u32 tpa_flags = 0;
4077
4078 if (set_tpa)
4079 tpa_flags = bp->flags & BNXT_FLAG_TPA;
4080 for (i = 0; i < bp->nr_vnics; i++) {
4081 rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
4082 if (rc) {
4083 netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
4084 rc, i);
4085 return rc;
4086 }
4087 }
4088 return 0;
4089 }
4090
4091 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
4092 {
4093 int i;
4094
4095 for (i = 0; i < bp->nr_vnics; i++)
4096 bnxt_hwrm_vnic_set_rss(bp, i, false);
4097 }
4098
4099 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
4100 bool irq_re_init)
4101 {
4102 if (bp->vnic_info) {
4103 bnxt_hwrm_clear_vnic_filter(bp);
4104 /* clear all RSS setting before free vnic ctx */
4105 bnxt_hwrm_clear_vnic_rss(bp);
4106 bnxt_hwrm_vnic_ctx_free(bp);
4107 /* before free the vnic, undo the vnic tpa settings */
4108 if (bp->flags & BNXT_FLAG_TPA)
4109 bnxt_set_tpa(bp, false);
4110 bnxt_hwrm_vnic_free(bp);
4111 }
4112 bnxt_hwrm_ring_free(bp, close_path);
4113 bnxt_hwrm_ring_grp_free(bp);
4114 if (irq_re_init) {
4115 bnxt_hwrm_stat_ctx_free(bp);
4116 bnxt_hwrm_free_tunnel_ports(bp);
4117 }
4118 }
4119
4120 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
4121 {
4122 int rc;
4123
4124 /* allocate context for vnic */
4125 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id);
4126 if (rc) {
4127 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4128 vnic_id, rc);
4129 goto vnic_setup_err;
4130 }
4131 bp->rsscos_nr_ctxs++;
4132
4133 /* configure default vnic, ring grp */
4134 rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
4135 if (rc) {
4136 netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
4137 vnic_id, rc);
4138 goto vnic_setup_err;
4139 }
4140
4141 /* Enable RSS hashing on vnic */
4142 rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
4143 if (rc) {
4144 netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
4145 vnic_id, rc);
4146 goto vnic_setup_err;
4147 }
4148
4149 if (bp->flags & BNXT_FLAG_AGG_RINGS) {
4150 rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
4151 if (rc) {
4152 netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
4153 vnic_id, rc);
4154 }
4155 }
4156
4157 vnic_setup_err:
4158 return rc;
4159 }
4160
4161 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
4162 {
4163 #ifdef CONFIG_RFS_ACCEL
4164 int i, rc = 0;
4165
4166 for (i = 0; i < bp->rx_nr_rings; i++) {
4167 u16 vnic_id = i + 1;
4168 u16 ring_id = i;
4169
4170 if (vnic_id >= bp->nr_vnics)
4171 break;
4172
4173 bp->vnic_info[vnic_id].flags |= BNXT_VNIC_RFS_FLAG;
4174 rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
4175 if (rc) {
4176 netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
4177 vnic_id, rc);
4178 break;
4179 }
4180 rc = bnxt_setup_vnic(bp, vnic_id);
4181 if (rc)
4182 break;
4183 }
4184 return rc;
4185 #else
4186 return 0;
4187 #endif
4188 }
4189
4190 static int bnxt_cfg_rx_mode(struct bnxt *);
4191 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
4192
4193 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
4194 {
4195 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
4196 int rc = 0;
4197
4198 if (irq_re_init) {
4199 rc = bnxt_hwrm_stat_ctx_alloc(bp);
4200 if (rc) {
4201 netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
4202 rc);
4203 goto err_out;
4204 }
4205 }
4206
4207 rc = bnxt_hwrm_ring_alloc(bp);
4208 if (rc) {
4209 netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
4210 goto err_out;
4211 }
4212
4213 rc = bnxt_hwrm_ring_grp_alloc(bp);
4214 if (rc) {
4215 netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
4216 goto err_out;
4217 }
4218
4219 /* default vnic 0 */
4220 rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, bp->rx_nr_rings);
4221 if (rc) {
4222 netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
4223 goto err_out;
4224 }
4225
4226 rc = bnxt_setup_vnic(bp, 0);
4227 if (rc)
4228 goto err_out;
4229
4230 if (bp->flags & BNXT_FLAG_RFS) {
4231 rc = bnxt_alloc_rfs_vnics(bp);
4232 if (rc)
4233 goto err_out;
4234 }
4235
4236 if (bp->flags & BNXT_FLAG_TPA) {
4237 rc = bnxt_set_tpa(bp, true);
4238 if (rc)
4239 goto err_out;
4240 }
4241
4242 if (BNXT_VF(bp))
4243 bnxt_update_vf_mac(bp);
4244
4245 /* Filter for default vnic 0 */
4246 rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
4247 if (rc) {
4248 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
4249 goto err_out;
4250 }
4251 vnic->uc_filter_count = 1;
4252
4253 vnic->rx_mask = CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
4254
4255 if ((bp->dev->flags & IFF_PROMISC) && BNXT_PF(bp))
4256 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
4257
4258 if (bp->dev->flags & IFF_ALLMULTI) {
4259 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
4260 vnic->mc_list_count = 0;
4261 } else {
4262 u32 mask = 0;
4263
4264 bnxt_mc_list_updated(bp, &mask);
4265 vnic->rx_mask |= mask;
4266 }
4267
4268 rc = bnxt_cfg_rx_mode(bp);
4269 if (rc)
4270 goto err_out;
4271
4272 rc = bnxt_hwrm_set_coal(bp);
4273 if (rc)
4274 netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
4275 rc);
4276
4277 if (BNXT_VF(bp)) {
4278 bnxt_hwrm_func_qcfg(bp);
4279 netdev_update_features(bp->dev);
4280 }
4281
4282 return 0;
4283
4284 err_out:
4285 bnxt_hwrm_resource_free(bp, 0, true);
4286
4287 return rc;
4288 }
4289
4290 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
4291 {
4292 bnxt_hwrm_resource_free(bp, 1, irq_re_init);
4293 return 0;
4294 }
4295
4296 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
4297 {
4298 bnxt_init_rx_rings(bp);
4299 bnxt_init_tx_rings(bp);
4300 bnxt_init_ring_grps(bp, irq_re_init);
4301 bnxt_init_vnics(bp);
4302
4303 return bnxt_init_chip(bp, irq_re_init);
4304 }
4305
4306 static void bnxt_disable_int(struct bnxt *bp)
4307 {
4308 int i;
4309
4310 if (!bp->bnapi)
4311 return;
4312
4313 for (i = 0; i < bp->cp_nr_rings; i++) {
4314 struct bnxt_napi *bnapi = bp->bnapi[i];
4315 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4316
4317 BNXT_CP_DB(cpr->cp_doorbell, cpr->cp_raw_cons);
4318 }
4319 }
4320
4321 static void bnxt_enable_int(struct bnxt *bp)
4322 {
4323 int i;
4324
4325 atomic_set(&bp->intr_sem, 0);
4326 for (i = 0; i < bp->cp_nr_rings; i++) {
4327 struct bnxt_napi *bnapi = bp->bnapi[i];
4328 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4329
4330 BNXT_CP_DB_REARM(cpr->cp_doorbell, cpr->cp_raw_cons);
4331 }
4332 }
4333
4334 static int bnxt_set_real_num_queues(struct bnxt *bp)
4335 {
4336 int rc;
4337 struct net_device *dev = bp->dev;
4338
4339 rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings);
4340 if (rc)
4341 return rc;
4342
4343 rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
4344 if (rc)
4345 return rc;
4346
4347 #ifdef CONFIG_RFS_ACCEL
4348 if (bp->flags & BNXT_FLAG_RFS)
4349 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
4350 #endif
4351
4352 return rc;
4353 }
4354
4355 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
4356 bool shared)
4357 {
4358 int _rx = *rx, _tx = *tx;
4359
4360 if (shared) {
4361 *rx = min_t(int, _rx, max);
4362 *tx = min_t(int, _tx, max);
4363 } else {
4364 if (max < 2)
4365 return -ENOMEM;
4366
4367 while (_rx + _tx > max) {
4368 if (_rx > _tx && _rx > 1)
4369 _rx--;
4370 else if (_tx > 1)
4371 _tx--;
4372 }
4373 *rx = _rx;
4374 *tx = _tx;
4375 }
4376 return 0;
4377 }
4378
4379 static int bnxt_setup_msix(struct bnxt *bp)
4380 {
4381 struct msix_entry *msix_ent;
4382 struct net_device *dev = bp->dev;
4383 int i, total_vecs, rc = 0, min = 1;
4384 const int len = sizeof(bp->irq_tbl[0].name);
4385
4386 bp->flags &= ~BNXT_FLAG_USING_MSIX;
4387 total_vecs = bp->cp_nr_rings;
4388
4389 msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
4390 if (!msix_ent)
4391 return -ENOMEM;
4392
4393 for (i = 0; i < total_vecs; i++) {
4394 msix_ent[i].entry = i;
4395 msix_ent[i].vector = 0;
4396 }
4397
4398 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
4399 min = 2;
4400
4401 total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
4402 if (total_vecs < 0) {
4403 rc = -ENODEV;
4404 goto msix_setup_exit;
4405 }
4406
4407 bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
4408 if (bp->irq_tbl) {
4409 int tcs;
4410
4411 /* Trim rings based upon num of vectors allocated */
4412 rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
4413 total_vecs, min == 1);
4414 if (rc)
4415 goto msix_setup_exit;
4416
4417 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4418 tcs = netdev_get_num_tc(dev);
4419 if (tcs > 1) {
4420 bp->tx_nr_rings_per_tc = bp->tx_nr_rings / tcs;
4421 if (bp->tx_nr_rings_per_tc == 0) {
4422 netdev_reset_tc(dev);
4423 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4424 } else {
4425 int i, off, count;
4426
4427 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tcs;
4428 for (i = 0; i < tcs; i++) {
4429 count = bp->tx_nr_rings_per_tc;
4430 off = i * count;
4431 netdev_set_tc_queue(dev, i, count, off);
4432 }
4433 }
4434 }
4435 bp->cp_nr_rings = total_vecs;
4436
4437 for (i = 0; i < bp->cp_nr_rings; i++) {
4438 char *attr;
4439
4440 bp->irq_tbl[i].vector = msix_ent[i].vector;
4441 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4442 attr = "TxRx";
4443 else if (i < bp->rx_nr_rings)
4444 attr = "rx";
4445 else
4446 attr = "tx";
4447
4448 snprintf(bp->irq_tbl[i].name, len,
4449 "%s-%s-%d", dev->name, attr, i);
4450 bp->irq_tbl[i].handler = bnxt_msix;
4451 }
4452 rc = bnxt_set_real_num_queues(bp);
4453 if (rc)
4454 goto msix_setup_exit;
4455 } else {
4456 rc = -ENOMEM;
4457 goto msix_setup_exit;
4458 }
4459 bp->flags |= BNXT_FLAG_USING_MSIX;
4460 kfree(msix_ent);
4461 return 0;
4462
4463 msix_setup_exit:
4464 netdev_err(bp->dev, "bnxt_setup_msix err: %x\n", rc);
4465 pci_disable_msix(bp->pdev);
4466 kfree(msix_ent);
4467 return rc;
4468 }
4469
4470 static int bnxt_setup_inta(struct bnxt *bp)
4471 {
4472 int rc;
4473 const int len = sizeof(bp->irq_tbl[0].name);
4474
4475 if (netdev_get_num_tc(bp->dev))
4476 netdev_reset_tc(bp->dev);
4477
4478 bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
4479 if (!bp->irq_tbl) {
4480 rc = -ENOMEM;
4481 return rc;
4482 }
4483 bp->rx_nr_rings = 1;
4484 bp->tx_nr_rings = 1;
4485 bp->cp_nr_rings = 1;
4486 bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
4487 bp->flags |= BNXT_FLAG_SHARED_RINGS;
4488 bp->irq_tbl[0].vector = bp->pdev->irq;
4489 snprintf(bp->irq_tbl[0].name, len,
4490 "%s-%s-%d", bp->dev->name, "TxRx", 0);
4491 bp->irq_tbl[0].handler = bnxt_inta;
4492 rc = bnxt_set_real_num_queues(bp);
4493 return rc;
4494 }
4495
4496 static int bnxt_setup_int_mode(struct bnxt *bp)
4497 {
4498 int rc = 0;
4499
4500 if (bp->flags & BNXT_FLAG_MSIX_CAP)
4501 rc = bnxt_setup_msix(bp);
4502
4503 if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
4504 /* fallback to INTA */
4505 rc = bnxt_setup_inta(bp);
4506 }
4507 return rc;
4508 }
4509
4510 static void bnxt_free_irq(struct bnxt *bp)
4511 {
4512 struct bnxt_irq *irq;
4513 int i;
4514
4515 #ifdef CONFIG_RFS_ACCEL
4516 free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
4517 bp->dev->rx_cpu_rmap = NULL;
4518 #endif
4519 if (!bp->irq_tbl)
4520 return;
4521
4522 for (i = 0; i < bp->cp_nr_rings; i++) {
4523 irq = &bp->irq_tbl[i];
4524 if (irq->requested)
4525 free_irq(irq->vector, bp->bnapi[i]);
4526 irq->requested = 0;
4527 }
4528 if (bp->flags & BNXT_FLAG_USING_MSIX)
4529 pci_disable_msix(bp->pdev);
4530 kfree(bp->irq_tbl);
4531 bp->irq_tbl = NULL;
4532 }
4533
4534 static int bnxt_request_irq(struct bnxt *bp)
4535 {
4536 int i, j, rc = 0;
4537 unsigned long flags = 0;
4538 #ifdef CONFIG_RFS_ACCEL
4539 struct cpu_rmap *rmap = bp->dev->rx_cpu_rmap;
4540 #endif
4541
4542 if (!(bp->flags & BNXT_FLAG_USING_MSIX))
4543 flags = IRQF_SHARED;
4544
4545 for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
4546 struct bnxt_irq *irq = &bp->irq_tbl[i];
4547 #ifdef CONFIG_RFS_ACCEL
4548 if (rmap && bp->bnapi[i]->rx_ring) {
4549 rc = irq_cpu_rmap_add(rmap, irq->vector);
4550 if (rc)
4551 netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
4552 j);
4553 j++;
4554 }
4555 #endif
4556 rc = request_irq(irq->vector, irq->handler, flags, irq->name,
4557 bp->bnapi[i]);
4558 if (rc)
4559 break;
4560
4561 irq->requested = 1;
4562 }
4563 return rc;
4564 }
4565
4566 static void bnxt_del_napi(struct bnxt *bp)
4567 {
4568 int i;
4569
4570 if (!bp->bnapi)
4571 return;
4572
4573 for (i = 0; i < bp->cp_nr_rings; i++) {
4574 struct bnxt_napi *bnapi = bp->bnapi[i];
4575
4576 napi_hash_del(&bnapi->napi);
4577 netif_napi_del(&bnapi->napi);
4578 }
4579 }
4580
4581 static void bnxt_init_napi(struct bnxt *bp)
4582 {
4583 int i;
4584 struct bnxt_napi *bnapi;
4585
4586 if (bp->flags & BNXT_FLAG_USING_MSIX) {
4587 for (i = 0; i < bp->cp_nr_rings; i++) {
4588 bnapi = bp->bnapi[i];
4589 netif_napi_add(bp->dev, &bnapi->napi,
4590 bnxt_poll, 64);
4591 }
4592 } else {
4593 bnapi = bp->bnapi[0];
4594 netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
4595 }
4596 }
4597
4598 static void bnxt_disable_napi(struct bnxt *bp)
4599 {
4600 int i;
4601
4602 if (!bp->bnapi)
4603 return;
4604
4605 for (i = 0; i < bp->cp_nr_rings; i++) {
4606 napi_disable(&bp->bnapi[i]->napi);
4607 bnxt_disable_poll(bp->bnapi[i]);
4608 }
4609 }
4610
4611 static void bnxt_enable_napi(struct bnxt *bp)
4612 {
4613 int i;
4614
4615 for (i = 0; i < bp->cp_nr_rings; i++) {
4616 bp->bnapi[i]->in_reset = false;
4617 bnxt_enable_poll(bp->bnapi[i]);
4618 napi_enable(&bp->bnapi[i]->napi);
4619 }
4620 }
4621
4622 static void bnxt_tx_disable(struct bnxt *bp)
4623 {
4624 int i;
4625 struct bnxt_tx_ring_info *txr;
4626 struct netdev_queue *txq;
4627
4628 if (bp->tx_ring) {
4629 for (i = 0; i < bp->tx_nr_rings; i++) {
4630 txr = &bp->tx_ring[i];
4631 txq = netdev_get_tx_queue(bp->dev, i);
4632 __netif_tx_lock(txq, smp_processor_id());
4633 txr->dev_state = BNXT_DEV_STATE_CLOSING;
4634 __netif_tx_unlock(txq);
4635 }
4636 }
4637 /* Stop all TX queues */
4638 netif_tx_disable(bp->dev);
4639 netif_carrier_off(bp->dev);
4640 }
4641
4642 static void bnxt_tx_enable(struct bnxt *bp)
4643 {
4644 int i;
4645 struct bnxt_tx_ring_info *txr;
4646 struct netdev_queue *txq;
4647
4648 for (i = 0; i < bp->tx_nr_rings; i++) {
4649 txr = &bp->tx_ring[i];
4650 txq = netdev_get_tx_queue(bp->dev, i);
4651 txr->dev_state = 0;
4652 }
4653 netif_tx_wake_all_queues(bp->dev);
4654 if (bp->link_info.link_up)
4655 netif_carrier_on(bp->dev);
4656 }
4657
4658 static void bnxt_report_link(struct bnxt *bp)
4659 {
4660 if (bp->link_info.link_up) {
4661 const char *duplex;
4662 const char *flow_ctrl;
4663 u16 speed;
4664
4665 netif_carrier_on(bp->dev);
4666 if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
4667 duplex = "full";
4668 else
4669 duplex = "half";
4670 if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
4671 flow_ctrl = "ON - receive & transmit";
4672 else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
4673 flow_ctrl = "ON - transmit";
4674 else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
4675 flow_ctrl = "ON - receive";
4676 else
4677 flow_ctrl = "none";
4678 speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
4679 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
4680 speed, duplex, flow_ctrl);
4681 if (bp->flags & BNXT_FLAG_EEE_CAP)
4682 netdev_info(bp->dev, "EEE is %s\n",
4683 bp->eee.eee_active ? "active" :
4684 "not active");
4685 } else {
4686 netif_carrier_off(bp->dev);
4687 netdev_err(bp->dev, "NIC Link is Down\n");
4688 }
4689 }
4690
4691 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
4692 {
4693 int rc = 0;
4694 struct hwrm_port_phy_qcaps_input req = {0};
4695 struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
4696
4697 if (bp->hwrm_spec_code < 0x10201)
4698 return 0;
4699
4700 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
4701
4702 mutex_lock(&bp->hwrm_cmd_lock);
4703 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4704 if (rc)
4705 goto hwrm_phy_qcaps_exit;
4706
4707 if (resp->eee_supported & PORT_PHY_QCAPS_RESP_EEE_SUPPORTED) {
4708 struct ethtool_eee *eee = &bp->eee;
4709 u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
4710
4711 bp->flags |= BNXT_FLAG_EEE_CAP;
4712 eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4713 bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
4714 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
4715 bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
4716 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
4717 }
4718
4719 hwrm_phy_qcaps_exit:
4720 mutex_unlock(&bp->hwrm_cmd_lock);
4721 return rc;
4722 }
4723
4724 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
4725 {
4726 int rc = 0;
4727 struct bnxt_link_info *link_info = &bp->link_info;
4728 struct hwrm_port_phy_qcfg_input req = {0};
4729 struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
4730 u8 link_up = link_info->link_up;
4731
4732 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
4733
4734 mutex_lock(&bp->hwrm_cmd_lock);
4735 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4736 if (rc) {
4737 mutex_unlock(&bp->hwrm_cmd_lock);
4738 return rc;
4739 }
4740
4741 memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
4742 link_info->phy_link_status = resp->link;
4743 link_info->duplex = resp->duplex;
4744 link_info->pause = resp->pause;
4745 link_info->auto_mode = resp->auto_mode;
4746 link_info->auto_pause_setting = resp->auto_pause;
4747 link_info->lp_pause = resp->link_partner_adv_pause;
4748 link_info->force_pause_setting = resp->force_pause;
4749 link_info->duplex_setting = resp->duplex;
4750 if (link_info->phy_link_status == BNXT_LINK_LINK)
4751 link_info->link_speed = le16_to_cpu(resp->link_speed);
4752 else
4753 link_info->link_speed = 0;
4754 link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
4755 link_info->support_speeds = le16_to_cpu(resp->support_speeds);
4756 link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
4757 link_info->lp_auto_link_speeds =
4758 le16_to_cpu(resp->link_partner_adv_speeds);
4759 link_info->preemphasis = le32_to_cpu(resp->preemphasis);
4760 link_info->phy_ver[0] = resp->phy_maj;
4761 link_info->phy_ver[1] = resp->phy_min;
4762 link_info->phy_ver[2] = resp->phy_bld;
4763 link_info->media_type = resp->media_type;
4764 link_info->phy_type = resp->phy_type;
4765 link_info->transceiver = resp->xcvr_pkg_type;
4766 link_info->phy_addr = resp->eee_config_phy_addr &
4767 PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
4768 link_info->module_status = resp->module_status;
4769
4770 if (bp->flags & BNXT_FLAG_EEE_CAP) {
4771 struct ethtool_eee *eee = &bp->eee;
4772 u16 fw_speeds;
4773
4774 eee->eee_active = 0;
4775 if (resp->eee_config_phy_addr &
4776 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
4777 eee->eee_active = 1;
4778 fw_speeds = le16_to_cpu(
4779 resp->link_partner_adv_eee_link_speed_mask);
4780 eee->lp_advertised =
4781 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4782 }
4783
4784 /* Pull initial EEE config */
4785 if (!chng_link_state) {
4786 if (resp->eee_config_phy_addr &
4787 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
4788 eee->eee_enabled = 1;
4789
4790 fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
4791 eee->advertised =
4792 _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
4793
4794 if (resp->eee_config_phy_addr &
4795 PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
4796 __le32 tmr;
4797
4798 eee->tx_lpi_enabled = 1;
4799 tmr = resp->xcvr_identifier_type_tx_lpi_timer;
4800 eee->tx_lpi_timer = le32_to_cpu(tmr) &
4801 PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
4802 }
4803 }
4804 }
4805 /* TODO: need to add more logic to report VF link */
4806 if (chng_link_state) {
4807 if (link_info->phy_link_status == BNXT_LINK_LINK)
4808 link_info->link_up = 1;
4809 else
4810 link_info->link_up = 0;
4811 if (link_up != link_info->link_up)
4812 bnxt_report_link(bp);
4813 } else {
4814 /* alwasy link down if not require to update link state */
4815 link_info->link_up = 0;
4816 }
4817 mutex_unlock(&bp->hwrm_cmd_lock);
4818 return 0;
4819 }
4820
4821 static void bnxt_get_port_module_status(struct bnxt *bp)
4822 {
4823 struct bnxt_link_info *link_info = &bp->link_info;
4824 struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
4825 u8 module_status;
4826
4827 if (bnxt_update_link(bp, true))
4828 return;
4829
4830 module_status = link_info->module_status;
4831 switch (module_status) {
4832 case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
4833 case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
4834 case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
4835 netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
4836 bp->pf.port_id);
4837 if (bp->hwrm_spec_code >= 0x10201) {
4838 netdev_warn(bp->dev, "Module part number %s\n",
4839 resp->phy_vendor_partnumber);
4840 }
4841 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
4842 netdev_warn(bp->dev, "TX is disabled\n");
4843 if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
4844 netdev_warn(bp->dev, "SFP+ module is shutdown\n");
4845 }
4846 }
4847
4848 static void
4849 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
4850 {
4851 if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
4852 if (bp->hwrm_spec_code >= 0x10201)
4853 req->auto_pause =
4854 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
4855 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4856 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
4857 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4858 req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
4859 req->enables |=
4860 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4861 } else {
4862 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
4863 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
4864 if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
4865 req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
4866 req->enables |=
4867 cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
4868 if (bp->hwrm_spec_code >= 0x10201) {
4869 req->auto_pause = req->force_pause;
4870 req->enables |= cpu_to_le32(
4871 PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
4872 }
4873 }
4874 }
4875
4876 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
4877 struct hwrm_port_phy_cfg_input *req)
4878 {
4879 u8 autoneg = bp->link_info.autoneg;
4880 u16 fw_link_speed = bp->link_info.req_link_speed;
4881 u32 advertising = bp->link_info.advertising;
4882
4883 if (autoneg & BNXT_AUTONEG_SPEED) {
4884 req->auto_mode |=
4885 PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
4886
4887 req->enables |= cpu_to_le32(
4888 PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
4889 req->auto_link_speed_mask = cpu_to_le16(advertising);
4890
4891 req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
4892 req->flags |=
4893 cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
4894 } else {
4895 req->force_link_speed = cpu_to_le16(fw_link_speed);
4896 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
4897 }
4898
4899 /* tell chimp that the setting takes effect immediately */
4900 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
4901 }
4902
4903 int bnxt_hwrm_set_pause(struct bnxt *bp)
4904 {
4905 struct hwrm_port_phy_cfg_input req = {0};
4906 int rc;
4907
4908 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4909 bnxt_hwrm_set_pause_common(bp, &req);
4910
4911 if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
4912 bp->link_info.force_link_chng)
4913 bnxt_hwrm_set_link_common(bp, &req);
4914
4915 mutex_lock(&bp->hwrm_cmd_lock);
4916 rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4917 if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
4918 /* since changing of pause setting doesn't trigger any link
4919 * change event, the driver needs to update the current pause
4920 * result upon successfully return of the phy_cfg command
4921 */
4922 bp->link_info.pause =
4923 bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
4924 bp->link_info.auto_pause_setting = 0;
4925 if (!bp->link_info.force_link_chng)
4926 bnxt_report_link(bp);
4927 }
4928 bp->link_info.force_link_chng = false;
4929 mutex_unlock(&bp->hwrm_cmd_lock);
4930 return rc;
4931 }
4932
4933 static void bnxt_hwrm_set_eee(struct bnxt *bp,
4934 struct hwrm_port_phy_cfg_input *req)
4935 {
4936 struct ethtool_eee *eee = &bp->eee;
4937
4938 if (eee->eee_enabled) {
4939 u16 eee_speeds;
4940 u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
4941
4942 if (eee->tx_lpi_enabled)
4943 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
4944 else
4945 flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
4946
4947 req->flags |= cpu_to_le32(flags);
4948 eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
4949 req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
4950 req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
4951 } else {
4952 req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
4953 }
4954 }
4955
4956 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
4957 {
4958 struct hwrm_port_phy_cfg_input req = {0};
4959
4960 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4961 if (set_pause)
4962 bnxt_hwrm_set_pause_common(bp, &req);
4963
4964 bnxt_hwrm_set_link_common(bp, &req);
4965
4966 if (set_eee)
4967 bnxt_hwrm_set_eee(bp, &req);
4968 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4969 }
4970
4971 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
4972 {
4973 struct hwrm_port_phy_cfg_input req = {0};
4974
4975 if (!BNXT_SINGLE_PF(bp))
4976 return 0;
4977
4978 if (pci_num_vf(bp->pdev))
4979 return 0;
4980
4981 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
4982 req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DOWN);
4983 return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4984 }
4985
4986 static bool bnxt_eee_config_ok(struct bnxt *bp)
4987 {
4988 struct ethtool_eee *eee = &bp->eee;
4989 struct bnxt_link_info *link_info = &bp->link_info;
4990
4991 if (!(bp->flags & BNXT_FLAG_EEE_CAP))
4992 return true;
4993
4994 if (eee->eee_enabled) {
4995 u32 advertising =
4996 _bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
4997
4998 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
4999 eee->eee_enabled = 0;
5000 return false;
5001 }
5002 if (eee->advertised & ~advertising) {
5003 eee->advertised = advertising & eee->supported;
5004 return false;
5005 }
5006 }
5007 return true;
5008 }
5009
5010 static int bnxt_update_phy_setting(struct bnxt *bp)
5011 {
5012 int rc;
5013 bool update_link = false;
5014 bool update_pause = false;
5015 bool update_eee = false;
5016 struct bnxt_link_info *link_info = &bp->link_info;
5017
5018 rc = bnxt_update_link(bp, true);
5019 if (rc) {
5020 netdev_err(bp->dev, "failed to update link (rc: %x)\n",
5021 rc);
5022 return rc;
5023 }
5024 if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
5025 (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
5026 link_info->req_flow_ctrl)
5027 update_pause = true;
5028 if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
5029 link_info->force_pause_setting != link_info->req_flow_ctrl)
5030 update_pause = true;
5031 if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
5032 if (BNXT_AUTO_MODE(link_info->auto_mode))
5033 update_link = true;
5034 if (link_info->req_link_speed != link_info->force_link_speed)
5035 update_link = true;
5036 if (link_info->req_duplex != link_info->duplex_setting)
5037 update_link = true;
5038 } else {
5039 if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
5040 update_link = true;
5041 if (link_info->advertising != link_info->auto_link_speeds)
5042 update_link = true;
5043 }
5044
5045 if (!bnxt_eee_config_ok(bp))
5046 update_eee = true;
5047
5048 if (update_link)
5049 rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
5050 else if (update_pause)
5051 rc = bnxt_hwrm_set_pause(bp);
5052 if (rc) {
5053 netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
5054 rc);
5055 return rc;
5056 }
5057
5058 return rc;
5059 }
5060
5061 /* Common routine to pre-map certain register block to different GRC window.
5062 * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
5063 * in PF and 3 windows in VF that can be customized to map in different
5064 * register blocks.
5065 */
5066 static void bnxt_preset_reg_win(struct bnxt *bp)
5067 {
5068 if (BNXT_PF(bp)) {
5069 /* CAG registers map to GRC window #4 */
5070 writel(BNXT_CAG_REG_BASE,
5071 bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
5072 }
5073 }
5074
5075 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5076 {
5077 int rc = 0;
5078
5079 bnxt_preset_reg_win(bp);
5080 netif_carrier_off(bp->dev);
5081 if (irq_re_init) {
5082 rc = bnxt_setup_int_mode(bp);
5083 if (rc) {
5084 netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
5085 rc);
5086 return rc;
5087 }
5088 }
5089 if ((bp->flags & BNXT_FLAG_RFS) &&
5090 !(bp->flags & BNXT_FLAG_USING_MSIX)) {
5091 /* disable RFS if falling back to INTA */
5092 bp->dev->hw_features &= ~NETIF_F_NTUPLE;
5093 bp->flags &= ~BNXT_FLAG_RFS;
5094 }
5095
5096 rc = bnxt_alloc_mem(bp, irq_re_init);
5097 if (rc) {
5098 netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
5099 goto open_err_free_mem;
5100 }
5101
5102 if (irq_re_init) {
5103 bnxt_init_napi(bp);
5104 rc = bnxt_request_irq(bp);
5105 if (rc) {
5106 netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
5107 goto open_err;
5108 }
5109 }
5110
5111 bnxt_enable_napi(bp);
5112
5113 rc = bnxt_init_nic(bp, irq_re_init);
5114 if (rc) {
5115 netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
5116 goto open_err;
5117 }
5118
5119 if (link_re_init) {
5120 rc = bnxt_update_phy_setting(bp);
5121 if (rc)
5122 netdev_warn(bp->dev, "failed to update phy settings\n");
5123 }
5124
5125 if (irq_re_init) {
5126 #if defined(CONFIG_VXLAN) || defined(CONFIG_VXLAN_MODULE)
5127 vxlan_get_rx_port(bp->dev);
5128 #endif
5129 if (!bnxt_hwrm_tunnel_dst_port_alloc(
5130 bp, htons(0x17c1),
5131 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE))
5132 bp->nge_port_cnt = 1;
5133 }
5134
5135 set_bit(BNXT_STATE_OPEN, &bp->state);
5136 bnxt_enable_int(bp);
5137 /* Enable TX queues */
5138 bnxt_tx_enable(bp);
5139 mod_timer(&bp->timer, jiffies + bp->current_interval);
5140 /* Poll link status and check for SFP+ module status */
5141 bnxt_get_port_module_status(bp);
5142
5143 return 0;
5144
5145 open_err:
5146 bnxt_disable_napi(bp);
5147 bnxt_del_napi(bp);
5148
5149 open_err_free_mem:
5150 bnxt_free_skbs(bp);
5151 bnxt_free_irq(bp);
5152 bnxt_free_mem(bp, true);
5153 return rc;
5154 }
5155
5156 /* rtnl_lock held */
5157 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5158 {
5159 int rc = 0;
5160
5161 rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
5162 if (rc) {
5163 netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
5164 dev_close(bp->dev);
5165 }
5166 return rc;
5167 }
5168
5169 static int bnxt_open(struct net_device *dev)
5170 {
5171 struct bnxt *bp = netdev_priv(dev);
5172 int rc = 0;
5173
5174 rc = bnxt_hwrm_func_reset(bp);
5175 if (rc) {
5176 netdev_err(bp->dev, "hwrm chip reset failure rc: %x\n",
5177 rc);
5178 rc = -1;
5179 return rc;
5180 }
5181 return __bnxt_open_nic(bp, true, true);
5182 }
5183
5184 static void bnxt_disable_int_sync(struct bnxt *bp)
5185 {
5186 int i;
5187
5188 atomic_inc(&bp->intr_sem);
5189 if (!netif_running(bp->dev))
5190 return;
5191
5192 bnxt_disable_int(bp);
5193 for (i = 0; i < bp->cp_nr_rings; i++)
5194 synchronize_irq(bp->irq_tbl[i].vector);
5195 }
5196
5197 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
5198 {
5199 int rc = 0;
5200
5201 #ifdef CONFIG_BNXT_SRIOV
5202 if (bp->sriov_cfg) {
5203 rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
5204 !bp->sriov_cfg,
5205 BNXT_SRIOV_CFG_WAIT_TMO);
5206 if (rc)
5207 netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
5208 }
5209 #endif
5210 /* Change device state to avoid TX queue wake up's */
5211 bnxt_tx_disable(bp);
5212
5213 clear_bit(BNXT_STATE_OPEN, &bp->state);
5214 smp_mb__after_atomic();
5215 while (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state))
5216 msleep(20);
5217
5218 /* Flush rings before disabling interrupts */
5219 bnxt_shutdown_nic(bp, irq_re_init);
5220
5221 /* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
5222
5223 bnxt_disable_napi(bp);
5224 bnxt_disable_int_sync(bp);
5225 del_timer_sync(&bp->timer);
5226 bnxt_free_skbs(bp);
5227
5228 if (irq_re_init) {
5229 bnxt_free_irq(bp);
5230 bnxt_del_napi(bp);
5231 }
5232 bnxt_free_mem(bp, irq_re_init);
5233 return rc;
5234 }
5235
5236 static int bnxt_close(struct net_device *dev)
5237 {
5238 struct bnxt *bp = netdev_priv(dev);
5239
5240 bnxt_close_nic(bp, true, true);
5241 bnxt_hwrm_shutdown_link(bp);
5242 return 0;
5243 }
5244
5245 /* rtnl_lock held */
5246 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5247 {
5248 switch (cmd) {
5249 case SIOCGMIIPHY:
5250 /* fallthru */
5251 case SIOCGMIIREG: {
5252 if (!netif_running(dev))
5253 return -EAGAIN;
5254
5255 return 0;
5256 }
5257
5258 case SIOCSMIIREG:
5259 if (!netif_running(dev))
5260 return -EAGAIN;
5261
5262 return 0;
5263
5264 default:
5265 /* do nothing */
5266 break;
5267 }
5268 return -EOPNOTSUPP;
5269 }
5270
5271 static struct rtnl_link_stats64 *
5272 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
5273 {
5274 u32 i;
5275 struct bnxt *bp = netdev_priv(dev);
5276
5277 memset(stats, 0, sizeof(struct rtnl_link_stats64));
5278
5279 if (!bp->bnapi)
5280 return stats;
5281
5282 /* TODO check if we need to synchronize with bnxt_close path */
5283 for (i = 0; i < bp->cp_nr_rings; i++) {
5284 struct bnxt_napi *bnapi = bp->bnapi[i];
5285 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5286 struct ctx_hw_stats *hw_stats = cpr->hw_stats;
5287
5288 stats->rx_packets += le64_to_cpu(hw_stats->rx_ucast_pkts);
5289 stats->rx_packets += le64_to_cpu(hw_stats->rx_mcast_pkts);
5290 stats->rx_packets += le64_to_cpu(hw_stats->rx_bcast_pkts);
5291
5292 stats->tx_packets += le64_to_cpu(hw_stats->tx_ucast_pkts);
5293 stats->tx_packets += le64_to_cpu(hw_stats->tx_mcast_pkts);
5294 stats->tx_packets += le64_to_cpu(hw_stats->tx_bcast_pkts);
5295
5296 stats->rx_bytes += le64_to_cpu(hw_stats->rx_ucast_bytes);
5297 stats->rx_bytes += le64_to_cpu(hw_stats->rx_mcast_bytes);
5298 stats->rx_bytes += le64_to_cpu(hw_stats->rx_bcast_bytes);
5299
5300 stats->tx_bytes += le64_to_cpu(hw_stats->tx_ucast_bytes);
5301 stats->tx_bytes += le64_to_cpu(hw_stats->tx_mcast_bytes);
5302 stats->tx_bytes += le64_to_cpu(hw_stats->tx_bcast_bytes);
5303
5304 stats->rx_missed_errors +=
5305 le64_to_cpu(hw_stats->rx_discard_pkts);
5306
5307 stats->multicast += le64_to_cpu(hw_stats->rx_mcast_pkts);
5308
5309 stats->tx_dropped += le64_to_cpu(hw_stats->tx_drop_pkts);
5310 }
5311
5312 if (bp->flags & BNXT_FLAG_PORT_STATS) {
5313 struct rx_port_stats *rx = bp->hw_rx_port_stats;
5314 struct tx_port_stats *tx = bp->hw_tx_port_stats;
5315
5316 stats->rx_crc_errors = le64_to_cpu(rx->rx_fcs_err_frames);
5317 stats->rx_frame_errors = le64_to_cpu(rx->rx_align_err_frames);
5318 stats->rx_length_errors = le64_to_cpu(rx->rx_undrsz_frames) +
5319 le64_to_cpu(rx->rx_ovrsz_frames) +
5320 le64_to_cpu(rx->rx_runt_frames);
5321 stats->rx_errors = le64_to_cpu(rx->rx_false_carrier_frames) +
5322 le64_to_cpu(rx->rx_jbr_frames);
5323 stats->collisions = le64_to_cpu(tx->tx_total_collisions);
5324 stats->tx_fifo_errors = le64_to_cpu(tx->tx_fifo_underruns);
5325 stats->tx_errors = le64_to_cpu(tx->tx_err);
5326 }
5327
5328 return stats;
5329 }
5330
5331 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
5332 {
5333 struct net_device *dev = bp->dev;
5334 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5335 struct netdev_hw_addr *ha;
5336 u8 *haddr;
5337 int mc_count = 0;
5338 bool update = false;
5339 int off = 0;
5340
5341 netdev_for_each_mc_addr(ha, dev) {
5342 if (mc_count >= BNXT_MAX_MC_ADDRS) {
5343 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5344 vnic->mc_list_count = 0;
5345 return false;
5346 }
5347 haddr = ha->addr;
5348 if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
5349 memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
5350 update = true;
5351 }
5352 off += ETH_ALEN;
5353 mc_count++;
5354 }
5355 if (mc_count)
5356 *rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
5357
5358 if (mc_count != vnic->mc_list_count) {
5359 vnic->mc_list_count = mc_count;
5360 update = true;
5361 }
5362 return update;
5363 }
5364
5365 static bool bnxt_uc_list_updated(struct bnxt *bp)
5366 {
5367 struct net_device *dev = bp->dev;
5368 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5369 struct netdev_hw_addr *ha;
5370 int off = 0;
5371
5372 if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
5373 return true;
5374
5375 netdev_for_each_uc_addr(ha, dev) {
5376 if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
5377 return true;
5378
5379 off += ETH_ALEN;
5380 }
5381 return false;
5382 }
5383
5384 static void bnxt_set_rx_mode(struct net_device *dev)
5385 {
5386 struct bnxt *bp = netdev_priv(dev);
5387 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5388 u32 mask = vnic->rx_mask;
5389 bool mc_update = false;
5390 bool uc_update;
5391
5392 if (!netif_running(dev))
5393 return;
5394
5395 mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
5396 CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
5397 CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST);
5398
5399 /* Only allow PF to be in promiscuous mode */
5400 if ((dev->flags & IFF_PROMISC) && BNXT_PF(bp))
5401 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5402
5403 uc_update = bnxt_uc_list_updated(bp);
5404
5405 if (dev->flags & IFF_ALLMULTI) {
5406 mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
5407 vnic->mc_list_count = 0;
5408 } else {
5409 mc_update = bnxt_mc_list_updated(bp, &mask);
5410 }
5411
5412 if (mask != vnic->rx_mask || uc_update || mc_update) {
5413 vnic->rx_mask = mask;
5414
5415 set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
5416 schedule_work(&bp->sp_task);
5417 }
5418 }
5419
5420 static int bnxt_cfg_rx_mode(struct bnxt *bp)
5421 {
5422 struct net_device *dev = bp->dev;
5423 struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5424 struct netdev_hw_addr *ha;
5425 int i, off = 0, rc;
5426 bool uc_update;
5427
5428 netif_addr_lock_bh(dev);
5429 uc_update = bnxt_uc_list_updated(bp);
5430 netif_addr_unlock_bh(dev);
5431
5432 if (!uc_update)
5433 goto skip_uc;
5434
5435 mutex_lock(&bp->hwrm_cmd_lock);
5436 for (i = 1; i < vnic->uc_filter_count; i++) {
5437 struct hwrm_cfa_l2_filter_free_input req = {0};
5438
5439 bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
5440 -1);
5441
5442 req.l2_filter_id = vnic->fw_l2_filter_id[i];
5443
5444 rc = _hwrm_send_message(bp, &req, sizeof(req),
5445 HWRM_CMD_TIMEOUT);
5446 }
5447 mutex_unlock(&bp->hwrm_cmd_lock);
5448
5449 vnic->uc_filter_count = 1;
5450
5451 netif_addr_lock_bh(dev);
5452 if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
5453 vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
5454 } else {
5455 netdev_for_each_uc_addr(ha, dev) {
5456 memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
5457 off += ETH_ALEN;
5458 vnic->uc_filter_count++;
5459 }
5460 }
5461 netif_addr_unlock_bh(dev);
5462
5463 for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
5464 rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
5465 if (rc) {
5466 netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
5467 rc);
5468 vnic->uc_filter_count = i;
5469 return rc;
5470 }
5471 }
5472
5473 skip_uc:
5474 rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
5475 if (rc)
5476 netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %x\n",
5477 rc);
5478
5479 return rc;
5480 }
5481
5482 static bool bnxt_rfs_capable(struct bnxt *bp)
5483 {
5484 #ifdef CONFIG_RFS_ACCEL
5485 struct bnxt_pf_info *pf = &bp->pf;
5486 int vnics;
5487
5488 if (BNXT_VF(bp) || !(bp->flags & BNXT_FLAG_MSIX_CAP))
5489 return false;
5490
5491 vnics = 1 + bp->rx_nr_rings;
5492 if (vnics > pf->max_rsscos_ctxs || vnics > pf->max_vnics)
5493 return false;
5494
5495 return true;
5496 #else
5497 return false;
5498 #endif
5499 }
5500
5501 static netdev_features_t bnxt_fix_features(struct net_device *dev,
5502 netdev_features_t features)
5503 {
5504 struct bnxt *bp = netdev_priv(dev);
5505
5506 if (!bnxt_rfs_capable(bp))
5507 features &= ~NETIF_F_NTUPLE;
5508
5509 /* Both CTAG and STAG VLAN accelaration on the RX side have to be
5510 * turned on or off together.
5511 */
5512 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) !=
5513 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) {
5514 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
5515 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
5516 NETIF_F_HW_VLAN_STAG_RX);
5517 else
5518 features |= NETIF_F_HW_VLAN_CTAG_RX |
5519 NETIF_F_HW_VLAN_STAG_RX;
5520 }
5521 #ifdef CONFIG_BNXT_SRIOV
5522 if (BNXT_VF(bp)) {
5523 if (bp->vf.vlan) {
5524 features &= ~(NETIF_F_HW_VLAN_CTAG_RX |
5525 NETIF_F_HW_VLAN_STAG_RX);
5526 }
5527 }
5528 #endif
5529 return features;
5530 }
5531
5532 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
5533 {
5534 struct bnxt *bp = netdev_priv(dev);
5535 u32 flags = bp->flags;
5536 u32 changes;
5537 int rc = 0;
5538 bool re_init = false;
5539 bool update_tpa = false;
5540
5541 flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
5542 if ((features & NETIF_F_GRO) && (bp->pdev->revision > 0))
5543 flags |= BNXT_FLAG_GRO;
5544 if (features & NETIF_F_LRO)
5545 flags |= BNXT_FLAG_LRO;
5546
5547 if (features & NETIF_F_HW_VLAN_CTAG_RX)
5548 flags |= BNXT_FLAG_STRIP_VLAN;
5549
5550 if (features & NETIF_F_NTUPLE)
5551 flags |= BNXT_FLAG_RFS;
5552
5553 changes = flags ^ bp->flags;
5554 if (changes & BNXT_FLAG_TPA) {
5555 update_tpa = true;
5556 if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
5557 (flags & BNXT_FLAG_TPA) == 0)
5558 re_init = true;
5559 }
5560
5561 if (changes & ~BNXT_FLAG_TPA)
5562 re_init = true;
5563
5564 if (flags != bp->flags) {
5565 u32 old_flags = bp->flags;
5566
5567 bp->flags = flags;
5568
5569 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5570 if (update_tpa)
5571 bnxt_set_ring_params(bp);
5572 return rc;
5573 }
5574
5575 if (re_init) {
5576 bnxt_close_nic(bp, false, false);
5577 if (update_tpa)
5578 bnxt_set_ring_params(bp);
5579
5580 return bnxt_open_nic(bp, false, false);
5581 }
5582 if (update_tpa) {
5583 rc = bnxt_set_tpa(bp,
5584 (flags & BNXT_FLAG_TPA) ?
5585 true : false);
5586 if (rc)
5587 bp->flags = old_flags;
5588 }
5589 }
5590 return rc;
5591 }
5592
5593 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
5594 {
5595 struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
5596 int i = bnapi->index;
5597
5598 if (!txr)
5599 return;
5600
5601 netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
5602 i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
5603 txr->tx_cons);
5604 }
5605
5606 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
5607 {
5608 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
5609 int i = bnapi->index;
5610
5611 if (!rxr)
5612 return;
5613
5614 netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
5615 i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
5616 rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
5617 rxr->rx_sw_agg_prod);
5618 }
5619
5620 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
5621 {
5622 struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5623 int i = bnapi->index;
5624
5625 netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
5626 i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
5627 }
5628
5629 static void bnxt_dbg_dump_states(struct bnxt *bp)
5630 {
5631 int i;
5632 struct bnxt_napi *bnapi;
5633
5634 for (i = 0; i < bp->cp_nr_rings; i++) {
5635 bnapi = bp->bnapi[i];
5636 if (netif_msg_drv(bp)) {
5637 bnxt_dump_tx_sw_state(bnapi);
5638 bnxt_dump_rx_sw_state(bnapi);
5639 bnxt_dump_cp_sw_state(bnapi);
5640 }
5641 }
5642 }
5643
5644 static void bnxt_reset_task(struct bnxt *bp, bool silent)
5645 {
5646 if (!silent)
5647 bnxt_dbg_dump_states(bp);
5648 if (netif_running(bp->dev)) {
5649 bnxt_close_nic(bp, false, false);
5650 bnxt_open_nic(bp, false, false);
5651 }
5652 }
5653
5654 static void bnxt_tx_timeout(struct net_device *dev)
5655 {
5656 struct bnxt *bp = netdev_priv(dev);
5657
5658 netdev_err(bp->dev, "TX timeout detected, starting reset task!\n");
5659 set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
5660 schedule_work(&bp->sp_task);
5661 }
5662
5663 #ifdef CONFIG_NET_POLL_CONTROLLER
5664 static void bnxt_poll_controller(struct net_device *dev)
5665 {
5666 struct bnxt *bp = netdev_priv(dev);
5667 int i;
5668
5669 for (i = 0; i < bp->cp_nr_rings; i++) {
5670 struct bnxt_irq *irq = &bp->irq_tbl[i];
5671
5672 disable_irq(irq->vector);
5673 irq->handler(irq->vector, bp->bnapi[i]);
5674 enable_irq(irq->vector);
5675 }
5676 }
5677 #endif
5678
5679 static void bnxt_timer(unsigned long data)
5680 {
5681 struct bnxt *bp = (struct bnxt *)data;
5682 struct net_device *dev = bp->dev;
5683
5684 if (!netif_running(dev))
5685 return;
5686
5687 if (atomic_read(&bp->intr_sem) != 0)
5688 goto bnxt_restart_timer;
5689
5690 if (bp->link_info.link_up && (bp->flags & BNXT_FLAG_PORT_STATS)) {
5691 set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
5692 schedule_work(&bp->sp_task);
5693 }
5694 bnxt_restart_timer:
5695 mod_timer(&bp->timer, jiffies + bp->current_interval);
5696 }
5697
5698 /* Only called from bnxt_sp_task() */
5699 static void bnxt_reset(struct bnxt *bp, bool silent)
5700 {
5701 /* bnxt_reset_task() calls bnxt_close_nic() which waits
5702 * for BNXT_STATE_IN_SP_TASK to clear.
5703 * If there is a parallel dev_close(), bnxt_close() may be holding
5704 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we
5705 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
5706 */
5707 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5708 rtnl_lock();
5709 if (test_bit(BNXT_STATE_OPEN, &bp->state))
5710 bnxt_reset_task(bp, silent);
5711 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5712 rtnl_unlock();
5713 }
5714
5715 static void bnxt_cfg_ntp_filters(struct bnxt *);
5716
5717 static void bnxt_sp_task(struct work_struct *work)
5718 {
5719 struct bnxt *bp = container_of(work, struct bnxt, sp_task);
5720 int rc;
5721
5722 set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5723 smp_mb__after_atomic();
5724 if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
5725 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5726 return;
5727 }
5728
5729 if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
5730 bnxt_cfg_rx_mode(bp);
5731
5732 if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
5733 bnxt_cfg_ntp_filters(bp);
5734 if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
5735 rc = bnxt_update_link(bp, true);
5736 if (rc)
5737 netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
5738 rc);
5739 }
5740 if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
5741 bnxt_hwrm_exec_fwd_req(bp);
5742 if (test_and_clear_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event)) {
5743 bnxt_hwrm_tunnel_dst_port_alloc(
5744 bp, bp->vxlan_port,
5745 TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5746 }
5747 if (test_and_clear_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event)) {
5748 bnxt_hwrm_tunnel_dst_port_free(
5749 bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
5750 }
5751 if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
5752 bnxt_reset(bp, false);
5753
5754 if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
5755 bnxt_reset(bp, true);
5756
5757 if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event))
5758 bnxt_get_port_module_status(bp);
5759
5760 if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event))
5761 bnxt_hwrm_port_qstats(bp);
5762
5763 smp_mb__before_atomic();
5764 clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
5765 }
5766
5767 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
5768 {
5769 int rc;
5770 struct bnxt *bp = netdev_priv(dev);
5771
5772 SET_NETDEV_DEV(dev, &pdev->dev);
5773
5774 /* enable device (incl. PCI PM wakeup), and bus-mastering */
5775 rc = pci_enable_device(pdev);
5776 if (rc) {
5777 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
5778 goto init_err;
5779 }
5780
5781 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
5782 dev_err(&pdev->dev,
5783 "Cannot find PCI device base address, aborting\n");
5784 rc = -ENODEV;
5785 goto init_err_disable;
5786 }
5787
5788 rc = pci_request_regions(pdev, DRV_MODULE_NAME);
5789 if (rc) {
5790 dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
5791 goto init_err_disable;
5792 }
5793
5794 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
5795 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
5796 dev_err(&pdev->dev, "System does not support DMA, aborting\n");
5797 goto init_err_disable;
5798 }
5799
5800 pci_set_master(pdev);
5801
5802 bp->dev = dev;
5803 bp->pdev = pdev;
5804
5805 bp->bar0 = pci_ioremap_bar(pdev, 0);
5806 if (!bp->bar0) {
5807 dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
5808 rc = -ENOMEM;
5809 goto init_err_release;
5810 }
5811
5812 bp->bar1 = pci_ioremap_bar(pdev, 2);
5813 if (!bp->bar1) {
5814 dev_err(&pdev->dev, "Cannot map doorbell registers, aborting\n");
5815 rc = -ENOMEM;
5816 goto init_err_release;
5817 }
5818
5819 bp->bar2 = pci_ioremap_bar(pdev, 4);
5820 if (!bp->bar2) {
5821 dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
5822 rc = -ENOMEM;
5823 goto init_err_release;
5824 }
5825
5826 pci_enable_pcie_error_reporting(pdev);
5827
5828 INIT_WORK(&bp->sp_task, bnxt_sp_task);
5829
5830 spin_lock_init(&bp->ntp_fltr_lock);
5831
5832 bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
5833 bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
5834
5835 /* tick values in micro seconds */
5836 bp->rx_coal_ticks = 12;
5837 bp->rx_coal_bufs = 30;
5838 bp->rx_coal_ticks_irq = 1;
5839 bp->rx_coal_bufs_irq = 2;
5840
5841 bp->tx_coal_ticks = 25;
5842 bp->tx_coal_bufs = 30;
5843 bp->tx_coal_ticks_irq = 2;
5844 bp->tx_coal_bufs_irq = 2;
5845
5846 init_timer(&bp->timer);
5847 bp->timer.data = (unsigned long)bp;
5848 bp->timer.function = bnxt_timer;
5849 bp->current_interval = BNXT_TIMER_INTERVAL;
5850
5851 clear_bit(BNXT_STATE_OPEN, &bp->state);
5852
5853 return 0;
5854
5855 init_err_release:
5856 if (bp->bar2) {
5857 pci_iounmap(pdev, bp->bar2);
5858 bp->bar2 = NULL;
5859 }
5860
5861 if (bp->bar1) {
5862 pci_iounmap(pdev, bp->bar1);
5863 bp->bar1 = NULL;
5864 }
5865
5866 if (bp->bar0) {
5867 pci_iounmap(pdev, bp->bar0);
5868 bp->bar0 = NULL;
5869 }
5870
5871 pci_release_regions(pdev);
5872
5873 init_err_disable:
5874 pci_disable_device(pdev);
5875
5876 init_err:
5877 return rc;
5878 }
5879
5880 /* rtnl_lock held */
5881 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
5882 {
5883 struct sockaddr *addr = p;
5884 struct bnxt *bp = netdev_priv(dev);
5885 int rc = 0;
5886
5887 if (!is_valid_ether_addr(addr->sa_data))
5888 return -EADDRNOTAVAIL;
5889
5890 rc = bnxt_approve_mac(bp, addr->sa_data);
5891 if (rc)
5892 return rc;
5893
5894 if (ether_addr_equal(addr->sa_data, dev->dev_addr))
5895 return 0;
5896
5897 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
5898 if (netif_running(dev)) {
5899 bnxt_close_nic(bp, false, false);
5900 rc = bnxt_open_nic(bp, false, false);
5901 }
5902
5903 return rc;
5904 }
5905
5906 /* rtnl_lock held */
5907 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
5908 {
5909 struct bnxt *bp = netdev_priv(dev);
5910
5911 if (new_mtu < 60 || new_mtu > 9000)
5912 return -EINVAL;
5913
5914 if (netif_running(dev))
5915 bnxt_close_nic(bp, false, false);
5916
5917 dev->mtu = new_mtu;
5918 bnxt_set_ring_params(bp);
5919
5920 if (netif_running(dev))
5921 return bnxt_open_nic(bp, false, false);
5922
5923 return 0;
5924 }
5925
5926 static int bnxt_setup_tc(struct net_device *dev, u32 handle, __be16 proto,
5927 struct tc_to_netdev *ntc)
5928 {
5929 struct bnxt *bp = netdev_priv(dev);
5930 u8 tc;
5931
5932 if (ntc->type != TC_SETUP_MQPRIO)
5933 return -EINVAL;
5934
5935 tc = ntc->tc;
5936
5937 if (tc > bp->max_tc) {
5938 netdev_err(dev, "too many traffic classes requested: %d Max supported is %d\n",
5939 tc, bp->max_tc);
5940 return -EINVAL;
5941 }
5942
5943 if (netdev_get_num_tc(dev) == tc)
5944 return 0;
5945
5946 if (tc) {
5947 int max_rx_rings, max_tx_rings, rc;
5948 bool sh = false;
5949
5950 if (bp->flags & BNXT_FLAG_SHARED_RINGS)
5951 sh = true;
5952
5953 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
5954 if (rc || bp->tx_nr_rings_per_tc * tc > max_tx_rings)
5955 return -ENOMEM;
5956 }
5957
5958 /* Needs to close the device and do hw resource re-allocations */
5959 if (netif_running(bp->dev))
5960 bnxt_close_nic(bp, true, false);
5961
5962 if (tc) {
5963 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
5964 netdev_set_num_tc(dev, tc);
5965 } else {
5966 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
5967 netdev_reset_tc(dev);
5968 }
5969 bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
5970 bp->num_stat_ctxs = bp->cp_nr_rings;
5971
5972 if (netif_running(bp->dev))
5973 return bnxt_open_nic(bp, true, false);
5974
5975 return 0;
5976 }
5977
5978 #ifdef CONFIG_RFS_ACCEL
5979 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
5980 struct bnxt_ntuple_filter *f2)
5981 {
5982 struct flow_keys *keys1 = &f1->fkeys;
5983 struct flow_keys *keys2 = &f2->fkeys;
5984
5985 if (keys1->addrs.v4addrs.src == keys2->addrs.v4addrs.src &&
5986 keys1->addrs.v4addrs.dst == keys2->addrs.v4addrs.dst &&
5987 keys1->ports.ports == keys2->ports.ports &&
5988 keys1->basic.ip_proto == keys2->basic.ip_proto &&
5989 keys1->basic.n_proto == keys2->basic.n_proto &&
5990 ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr))
5991 return true;
5992
5993 return false;
5994 }
5995
5996 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
5997 u16 rxq_index, u32 flow_id)
5998 {
5999 struct bnxt *bp = netdev_priv(dev);
6000 struct bnxt_ntuple_filter *fltr, *new_fltr;
6001 struct flow_keys *fkeys;
6002 struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
6003 int rc = 0, idx, bit_id;
6004 struct hlist_head *head;
6005
6006 if (skb->encapsulation)
6007 return -EPROTONOSUPPORT;
6008
6009 new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
6010 if (!new_fltr)
6011 return -ENOMEM;
6012
6013 fkeys = &new_fltr->fkeys;
6014 if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
6015 rc = -EPROTONOSUPPORT;
6016 goto err_free;
6017 }
6018
6019 if ((fkeys->basic.n_proto != htons(ETH_P_IP)) ||
6020 ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
6021 (fkeys->basic.ip_proto != IPPROTO_UDP))) {
6022 rc = -EPROTONOSUPPORT;
6023 goto err_free;
6024 }
6025
6026 memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
6027
6028 idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
6029 head = &bp->ntp_fltr_hash_tbl[idx];
6030 rcu_read_lock();
6031 hlist_for_each_entry_rcu(fltr, head, hash) {
6032 if (bnxt_fltr_match(fltr, new_fltr)) {
6033 rcu_read_unlock();
6034 rc = 0;
6035 goto err_free;
6036 }
6037 }
6038 rcu_read_unlock();
6039
6040 spin_lock_bh(&bp->ntp_fltr_lock);
6041 bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
6042 BNXT_NTP_FLTR_MAX_FLTR, 0);
6043 if (bit_id < 0) {
6044 spin_unlock_bh(&bp->ntp_fltr_lock);
6045 rc = -ENOMEM;
6046 goto err_free;
6047 }
6048
6049 new_fltr->sw_id = (u16)bit_id;
6050 new_fltr->flow_id = flow_id;
6051 new_fltr->rxq = rxq_index;
6052 hlist_add_head_rcu(&new_fltr->hash, head);
6053 bp->ntp_fltr_count++;
6054 spin_unlock_bh(&bp->ntp_fltr_lock);
6055
6056 set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
6057 schedule_work(&bp->sp_task);
6058
6059 return new_fltr->sw_id;
6060
6061 err_free:
6062 kfree(new_fltr);
6063 return rc;
6064 }
6065
6066 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
6067 {
6068 int i;
6069
6070 for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
6071 struct hlist_head *head;
6072 struct hlist_node *tmp;
6073 struct bnxt_ntuple_filter *fltr;
6074 int rc;
6075
6076 head = &bp->ntp_fltr_hash_tbl[i];
6077 hlist_for_each_entry_safe(fltr, tmp, head, hash) {
6078 bool del = false;
6079
6080 if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
6081 if (rps_may_expire_flow(bp->dev, fltr->rxq,
6082 fltr->flow_id,
6083 fltr->sw_id)) {
6084 bnxt_hwrm_cfa_ntuple_filter_free(bp,
6085 fltr);
6086 del = true;
6087 }
6088 } else {
6089 rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
6090 fltr);
6091 if (rc)
6092 del = true;
6093 else
6094 set_bit(BNXT_FLTR_VALID, &fltr->state);
6095 }
6096
6097 if (del) {
6098 spin_lock_bh(&bp->ntp_fltr_lock);
6099 hlist_del_rcu(&fltr->hash);
6100 bp->ntp_fltr_count--;
6101 spin_unlock_bh(&bp->ntp_fltr_lock);
6102 synchronize_rcu();
6103 clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
6104 kfree(fltr);
6105 }
6106 }
6107 }
6108 if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
6109 netdev_info(bp->dev, "Receive PF driver unload event!");
6110 }
6111
6112 #else
6113
6114 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
6115 {
6116 }
6117
6118 #endif /* CONFIG_RFS_ACCEL */
6119
6120 static void bnxt_add_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6121 __be16 port)
6122 {
6123 struct bnxt *bp = netdev_priv(dev);
6124
6125 if (!netif_running(dev))
6126 return;
6127
6128 if (sa_family != AF_INET6 && sa_family != AF_INET)
6129 return;
6130
6131 if (bp->vxlan_port_cnt && bp->vxlan_port != port)
6132 return;
6133
6134 bp->vxlan_port_cnt++;
6135 if (bp->vxlan_port_cnt == 1) {
6136 bp->vxlan_port = port;
6137 set_bit(BNXT_VXLAN_ADD_PORT_SP_EVENT, &bp->sp_event);
6138 schedule_work(&bp->sp_task);
6139 }
6140 }
6141
6142 static void bnxt_del_vxlan_port(struct net_device *dev, sa_family_t sa_family,
6143 __be16 port)
6144 {
6145 struct bnxt *bp = netdev_priv(dev);
6146
6147 if (!netif_running(dev))
6148 return;
6149
6150 if (sa_family != AF_INET6 && sa_family != AF_INET)
6151 return;
6152
6153 if (bp->vxlan_port_cnt && bp->vxlan_port == port) {
6154 bp->vxlan_port_cnt--;
6155
6156 if (bp->vxlan_port_cnt == 0) {
6157 set_bit(BNXT_VXLAN_DEL_PORT_SP_EVENT, &bp->sp_event);
6158 schedule_work(&bp->sp_task);
6159 }
6160 }
6161 }
6162
6163 static const struct net_device_ops bnxt_netdev_ops = {
6164 .ndo_open = bnxt_open,
6165 .ndo_start_xmit = bnxt_start_xmit,
6166 .ndo_stop = bnxt_close,
6167 .ndo_get_stats64 = bnxt_get_stats64,
6168 .ndo_set_rx_mode = bnxt_set_rx_mode,
6169 .ndo_do_ioctl = bnxt_ioctl,
6170 .ndo_validate_addr = eth_validate_addr,
6171 .ndo_set_mac_address = bnxt_change_mac_addr,
6172 .ndo_change_mtu = bnxt_change_mtu,
6173 .ndo_fix_features = bnxt_fix_features,
6174 .ndo_set_features = bnxt_set_features,
6175 .ndo_tx_timeout = bnxt_tx_timeout,
6176 #ifdef CONFIG_BNXT_SRIOV
6177 .ndo_get_vf_config = bnxt_get_vf_config,
6178 .ndo_set_vf_mac = bnxt_set_vf_mac,
6179 .ndo_set_vf_vlan = bnxt_set_vf_vlan,
6180 .ndo_set_vf_rate = bnxt_set_vf_bw,
6181 .ndo_set_vf_link_state = bnxt_set_vf_link_state,
6182 .ndo_set_vf_spoofchk = bnxt_set_vf_spoofchk,
6183 #endif
6184 #ifdef CONFIG_NET_POLL_CONTROLLER
6185 .ndo_poll_controller = bnxt_poll_controller,
6186 #endif
6187 .ndo_setup_tc = bnxt_setup_tc,
6188 #ifdef CONFIG_RFS_ACCEL
6189 .ndo_rx_flow_steer = bnxt_rx_flow_steer,
6190 #endif
6191 .ndo_add_vxlan_port = bnxt_add_vxlan_port,
6192 .ndo_del_vxlan_port = bnxt_del_vxlan_port,
6193 #ifdef CONFIG_NET_RX_BUSY_POLL
6194 .ndo_busy_poll = bnxt_busy_poll,
6195 #endif
6196 };
6197
6198 static void bnxt_remove_one(struct pci_dev *pdev)
6199 {
6200 struct net_device *dev = pci_get_drvdata(pdev);
6201 struct bnxt *bp = netdev_priv(dev);
6202
6203 if (BNXT_PF(bp))
6204 bnxt_sriov_disable(bp);
6205
6206 pci_disable_pcie_error_reporting(pdev);
6207 unregister_netdev(dev);
6208 cancel_work_sync(&bp->sp_task);
6209 bp->sp_event = 0;
6210
6211 bnxt_hwrm_func_drv_unrgtr(bp);
6212 bnxt_free_hwrm_resources(bp);
6213 pci_iounmap(pdev, bp->bar2);
6214 pci_iounmap(pdev, bp->bar1);
6215 pci_iounmap(pdev, bp->bar0);
6216 free_netdev(dev);
6217
6218 pci_release_regions(pdev);
6219 pci_disable_device(pdev);
6220 }
6221
6222 static int bnxt_probe_phy(struct bnxt *bp)
6223 {
6224 int rc = 0;
6225 struct bnxt_link_info *link_info = &bp->link_info;
6226
6227 rc = bnxt_hwrm_phy_qcaps(bp);
6228 if (rc) {
6229 netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
6230 rc);
6231 return rc;
6232 }
6233
6234 rc = bnxt_update_link(bp, false);
6235 if (rc) {
6236 netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
6237 rc);
6238 return rc;
6239 }
6240
6241 /*initialize the ethool setting copy with NVM settings */
6242 if (BNXT_AUTO_MODE(link_info->auto_mode)) {
6243 link_info->autoneg = BNXT_AUTONEG_SPEED;
6244 if (bp->hwrm_spec_code >= 0x10201) {
6245 if (link_info->auto_pause_setting &
6246 PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
6247 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6248 } else {
6249 link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
6250 }
6251 link_info->advertising = link_info->auto_link_speeds;
6252 } else {
6253 link_info->req_link_speed = link_info->force_link_speed;
6254 link_info->req_duplex = link_info->duplex_setting;
6255 }
6256 if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
6257 link_info->req_flow_ctrl =
6258 link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
6259 else
6260 link_info->req_flow_ctrl = link_info->force_pause_setting;
6261 return rc;
6262 }
6263
6264 static int bnxt_get_max_irq(struct pci_dev *pdev)
6265 {
6266 u16 ctrl;
6267
6268 if (!pdev->msix_cap)
6269 return 1;
6270
6271 pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
6272 return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
6273 }
6274
6275 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
6276 int *max_cp)
6277 {
6278 int max_ring_grps = 0;
6279
6280 #ifdef CONFIG_BNXT_SRIOV
6281 if (!BNXT_PF(bp)) {
6282 *max_tx = bp->vf.max_tx_rings;
6283 *max_rx = bp->vf.max_rx_rings;
6284 *max_cp = min_t(int, bp->vf.max_irqs, bp->vf.max_cp_rings);
6285 *max_cp = min_t(int, *max_cp, bp->vf.max_stat_ctxs);
6286 max_ring_grps = bp->vf.max_hw_ring_grps;
6287 } else
6288 #endif
6289 {
6290 *max_tx = bp->pf.max_tx_rings;
6291 *max_rx = bp->pf.max_rx_rings;
6292 *max_cp = min_t(int, bp->pf.max_irqs, bp->pf.max_cp_rings);
6293 *max_cp = min_t(int, *max_cp, bp->pf.max_stat_ctxs);
6294 max_ring_grps = bp->pf.max_hw_ring_grps;
6295 }
6296
6297 if (bp->flags & BNXT_FLAG_AGG_RINGS)
6298 *max_rx >>= 1;
6299 *max_rx = min_t(int, *max_rx, max_ring_grps);
6300 }
6301
6302 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
6303 {
6304 int rx, tx, cp;
6305
6306 _bnxt_get_max_rings(bp, &rx, &tx, &cp);
6307 if (!rx || !tx || !cp)
6308 return -ENOMEM;
6309
6310 *max_rx = rx;
6311 *max_tx = tx;
6312 return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
6313 }
6314
6315 static int bnxt_set_dflt_rings(struct bnxt *bp)
6316 {
6317 int dflt_rings, max_rx_rings, max_tx_rings, rc;
6318 bool sh = true;
6319
6320 if (sh)
6321 bp->flags |= BNXT_FLAG_SHARED_RINGS;
6322 dflt_rings = netif_get_num_default_rss_queues();
6323 rc = bnxt_get_max_rings(bp, &max_rx_rings, &max_tx_rings, sh);
6324 if (rc)
6325 return rc;
6326 bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
6327 bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
6328 bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
6329 bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
6330 bp->tx_nr_rings + bp->rx_nr_rings;
6331 bp->num_stat_ctxs = bp->cp_nr_rings;
6332 return rc;
6333 }
6334
6335 static void bnxt_parse_log_pcie_link(struct bnxt *bp)
6336 {
6337 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN;
6338 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN;
6339
6340 if (pcie_get_minimum_link(bp->pdev, &speed, &width) ||
6341 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN)
6342 netdev_info(bp->dev, "Failed to determine PCIe Link Info\n");
6343 else
6344 netdev_info(bp->dev, "PCIe: Speed %s Width x%d\n",
6345 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" :
6346 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" :
6347 speed == PCIE_SPEED_8_0GT ? "8.0GT/s" :
6348 "Unknown", width);
6349 }
6350
6351 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6352 {
6353 static int version_printed;
6354 struct net_device *dev;
6355 struct bnxt *bp;
6356 int rc, max_irqs;
6357
6358 if (version_printed++ == 0)
6359 pr_info("%s", version);
6360
6361 max_irqs = bnxt_get_max_irq(pdev);
6362 dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
6363 if (!dev)
6364 return -ENOMEM;
6365
6366 bp = netdev_priv(dev);
6367
6368 if (bnxt_vf_pciid(ent->driver_data))
6369 bp->flags |= BNXT_FLAG_VF;
6370
6371 if (pdev->msix_cap)
6372 bp->flags |= BNXT_FLAG_MSIX_CAP;
6373
6374 rc = bnxt_init_board(pdev, dev);
6375 if (rc < 0)
6376 goto init_err_free;
6377
6378 dev->netdev_ops = &bnxt_netdev_ops;
6379 dev->watchdog_timeo = BNXT_TX_TIMEOUT;
6380 dev->ethtool_ops = &bnxt_ethtool_ops;
6381
6382 pci_set_drvdata(pdev, dev);
6383
6384 dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6385 NETIF_F_TSO | NETIF_F_TSO6 |
6386 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6387 NETIF_F_GSO_IPXIP4 |
6388 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6389 NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
6390 NETIF_F_RXCSUM | NETIF_F_LRO | NETIF_F_GRO;
6391
6392 dev->hw_enc_features =
6393 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
6394 NETIF_F_TSO | NETIF_F_TSO6 |
6395 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
6396 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
6397 NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
6398 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
6399 NETIF_F_GSO_GRE_CSUM;
6400 dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
6401 dev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |
6402 NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX;
6403 dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
6404 dev->priv_flags |= IFF_UNICAST_FLT;
6405
6406 #ifdef CONFIG_BNXT_SRIOV
6407 init_waitqueue_head(&bp->sriov_cfg_wait);
6408 #endif
6409 rc = bnxt_alloc_hwrm_resources(bp);
6410 if (rc)
6411 goto init_err;
6412
6413 mutex_init(&bp->hwrm_cmd_lock);
6414 bnxt_hwrm_ver_get(bp);
6415
6416 rc = bnxt_hwrm_func_drv_rgtr(bp);
6417 if (rc)
6418 goto init_err;
6419
6420 /* Get the MAX capabilities for this function */
6421 rc = bnxt_hwrm_func_qcaps(bp);
6422 if (rc) {
6423 netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
6424 rc);
6425 rc = -1;
6426 goto init_err;
6427 }
6428
6429 rc = bnxt_hwrm_queue_qportcfg(bp);
6430 if (rc) {
6431 netdev_err(bp->dev, "hwrm query qportcfg failure rc: %x\n",
6432 rc);
6433 rc = -1;
6434 goto init_err;
6435 }
6436
6437 bnxt_hwrm_func_qcfg(bp);
6438
6439 bnxt_set_tpa_flags(bp);
6440 bnxt_set_ring_params(bp);
6441 if (BNXT_PF(bp))
6442 bp->pf.max_irqs = max_irqs;
6443 #if defined(CONFIG_BNXT_SRIOV)
6444 else
6445 bp->vf.max_irqs = max_irqs;
6446 #endif
6447 bnxt_set_dflt_rings(bp);
6448
6449 if (BNXT_PF(bp)) {
6450 dev->hw_features |= NETIF_F_NTUPLE;
6451 if (bnxt_rfs_capable(bp)) {
6452 bp->flags |= BNXT_FLAG_RFS;
6453 dev->features |= NETIF_F_NTUPLE;
6454 }
6455 }
6456
6457 if (dev->hw_features & NETIF_F_HW_VLAN_CTAG_RX)
6458 bp->flags |= BNXT_FLAG_STRIP_VLAN;
6459
6460 rc = bnxt_probe_phy(bp);
6461 if (rc)
6462 goto init_err;
6463
6464 rc = register_netdev(dev);
6465 if (rc)
6466 goto init_err;
6467
6468 netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
6469 board_info[ent->driver_data].name,
6470 (long)pci_resource_start(pdev, 0), dev->dev_addr);
6471
6472 bnxt_parse_log_pcie_link(bp);
6473
6474 return 0;
6475
6476 init_err:
6477 pci_iounmap(pdev, bp->bar0);
6478 pci_release_regions(pdev);
6479 pci_disable_device(pdev);
6480
6481 init_err_free:
6482 free_netdev(dev);
6483 return rc;
6484 }
6485
6486 /**
6487 * bnxt_io_error_detected - called when PCI error is detected
6488 * @pdev: Pointer to PCI device
6489 * @state: The current pci connection state
6490 *
6491 * This function is called after a PCI bus error affecting
6492 * this device has been detected.
6493 */
6494 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
6495 pci_channel_state_t state)
6496 {
6497 struct net_device *netdev = pci_get_drvdata(pdev);
6498
6499 netdev_info(netdev, "PCI I/O error detected\n");
6500
6501 rtnl_lock();
6502 netif_device_detach(netdev);
6503
6504 if (state == pci_channel_io_perm_failure) {
6505 rtnl_unlock();
6506 return PCI_ERS_RESULT_DISCONNECT;
6507 }
6508
6509 if (netif_running(netdev))
6510 bnxt_close(netdev);
6511
6512 pci_disable_device(pdev);
6513 rtnl_unlock();
6514
6515 /* Request a slot slot reset. */
6516 return PCI_ERS_RESULT_NEED_RESET;
6517 }
6518
6519 /**
6520 * bnxt_io_slot_reset - called after the pci bus has been reset.
6521 * @pdev: Pointer to PCI device
6522 *
6523 * Restart the card from scratch, as if from a cold-boot.
6524 * At this point, the card has exprienced a hard reset,
6525 * followed by fixups by BIOS, and has its config space
6526 * set up identically to what it was at cold boot.
6527 */
6528 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
6529 {
6530 struct net_device *netdev = pci_get_drvdata(pdev);
6531 struct bnxt *bp = netdev_priv(netdev);
6532 int err = 0;
6533 pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
6534
6535 netdev_info(bp->dev, "PCI Slot Reset\n");
6536
6537 rtnl_lock();
6538
6539 if (pci_enable_device(pdev)) {
6540 dev_err(&pdev->dev,
6541 "Cannot re-enable PCI device after reset.\n");
6542 } else {
6543 pci_set_master(pdev);
6544
6545 if (netif_running(netdev))
6546 err = bnxt_open(netdev);
6547
6548 if (!err)
6549 result = PCI_ERS_RESULT_RECOVERED;
6550 }
6551
6552 if (result != PCI_ERS_RESULT_RECOVERED && netif_running(netdev))
6553 dev_close(netdev);
6554
6555 rtnl_unlock();
6556
6557 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6558 if (err) {
6559 dev_err(&pdev->dev,
6560 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
6561 err); /* non-fatal, continue */
6562 }
6563
6564 return PCI_ERS_RESULT_RECOVERED;
6565 }
6566
6567 /**
6568 * bnxt_io_resume - called when traffic can start flowing again.
6569 * @pdev: Pointer to PCI device
6570 *
6571 * This callback is called when the error recovery driver tells
6572 * us that its OK to resume normal operation.
6573 */
6574 static void bnxt_io_resume(struct pci_dev *pdev)
6575 {
6576 struct net_device *netdev = pci_get_drvdata(pdev);
6577
6578 rtnl_lock();
6579
6580 netif_device_attach(netdev);
6581
6582 rtnl_unlock();
6583 }
6584
6585 static const struct pci_error_handlers bnxt_err_handler = {
6586 .error_detected = bnxt_io_error_detected,
6587 .slot_reset = bnxt_io_slot_reset,
6588 .resume = bnxt_io_resume
6589 };
6590
6591 static struct pci_driver bnxt_pci_driver = {
6592 .name = DRV_MODULE_NAME,
6593 .id_table = bnxt_pci_tbl,
6594 .probe = bnxt_init_one,
6595 .remove = bnxt_remove_one,
6596 .err_handler = &bnxt_err_handler,
6597 #if defined(CONFIG_BNXT_SRIOV)
6598 .sriov_configure = bnxt_sriov_configure,
6599 #endif
6600 };
6601
6602 module_pci_driver(bnxt_pci_driver);
Linux kernel - Deliverables
This page took 0.187943 seconds and 5 git commands to generate.