projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'sched-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[deliverable/linux.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
1 /*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <net/addrconf.h>
64 #include <net/bonding.h>
65 #include <net/addrconf.h>
66 #include <asm/uaccess.h>
67
68 #include "cxgb4.h"
69 #include "t4_regs.h"
70 #include "t4_values.h"
71 #include "t4_msg.h"
72 #include "t4fw_api.h"
73 #include "t4fw_version.h"
74 #include "cxgb4_dcb.h"
75 #include "cxgb4_debugfs.h"
76 #include "clip_tbl.h"
77 #include "l2t.h"
78
79 #ifdef DRV_VERSION
80 #undef DRV_VERSION
81 #endif
82 #define DRV_VERSION "2.0.0-ko"
83 #define DRV_DESC "Chelsio T4/T5 Network Driver"
84
85 enum {
86 MAX_TXQ_ENTRIES = 16384,
87 MAX_CTRL_TXQ_ENTRIES = 1024,
88 MAX_RSPQ_ENTRIES = 16384,
89 MAX_RX_BUFFERS = 16384,
90 MIN_TXQ_ENTRIES = 32,
91 MIN_CTRL_TXQ_ENTRIES = 32,
92 MIN_RSPQ_ENTRIES = 128,
93 MIN_FL_ENTRIES = 16
94 };
95
96 /* Host shadow copy of ingress filter entry. This is in host native format
97 * and doesn't match the ordering or bit order, etc. of the hardware of the
98 * firmware command. The use of bit-field structure elements is purely to
99 * remind ourselves of the field size limitations and save memory in the case
100 * where the filter table is large.
101 */
102 struct filter_entry {
103 /* Administrative fields for filter.
104 */
105 u32 valid:1; /* filter allocated and valid */
106 u32 locked:1; /* filter is administratively locked */
107
108 u32 pending:1; /* filter action is pending firmware reply */
109 u32 smtidx:8; /* Source MAC Table index for smac */
110 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
111
112 /* The filter itself. Most of this is a straight copy of information
113 * provided by the extended ioctl(). Some fields are translated to
114 * internal forms -- for instance the Ingress Queue ID passed in from
115 * the ioctl() is translated into the Absolute Ingress Queue ID.
116 */
117 struct ch_filter_specification fs;
118 };
119
120 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
121 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
122 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
123
124 /* Macros needed to support the PCI Device ID Table ...
125 */
126 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
127 static struct pci_device_id cxgb4_pci_tbl[] = {
128 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
129
130 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
131 * called for both.
132 */
133 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
134
135 #define CH_PCI_ID_TABLE_ENTRY(devid) \
136 {PCI_VDEVICE(CHELSIO, (devid)), 4}
137
138 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
139 { 0, } \
140 }
141
142 #include "t4_pci_id_tbl.h"
143
144 #define FW4_FNAME "cxgb4/t4fw.bin"
145 #define FW5_FNAME "cxgb4/t5fw.bin"
146 #define FW4_CFNAME "cxgb4/t4-config.txt"
147 #define FW5_CFNAME "cxgb4/t5-config.txt"
148
149 MODULE_DESCRIPTION(DRV_DESC);
150 MODULE_AUTHOR("Chelsio Communications");
151 MODULE_LICENSE("Dual BSD/GPL");
152 MODULE_VERSION(DRV_VERSION);
153 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
154 MODULE_FIRMWARE(FW4_FNAME);
155 MODULE_FIRMWARE(FW5_FNAME);
156
157 /*
158 * Normally we're willing to become the firmware's Master PF but will be happy
159 * if another PF has already become the Master and initialized the adapter.
160 * Setting "force_init" will cause this driver to forcibly establish itself as
161 * the Master PF and initialize the adapter.
162 */
163 static uint force_init;
164
165 module_param(force_init, uint, 0644);
166 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
167
168 /*
169 * Normally if the firmware we connect to has Configuration File support, we
170 * use that and only fall back to the old Driver-based initialization if the
171 * Configuration File fails for some reason. If force_old_init is set, then
172 * we'll always use the old Driver-based initialization sequence.
173 */
174 static uint force_old_init;
175
176 module_param(force_old_init, uint, 0644);
177 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
178 " parameter");
179
180 static int dflt_msg_enable = DFLT_MSG_ENABLE;
181
182 module_param(dflt_msg_enable, int, 0644);
183 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
184
185 /*
186 * The driver uses the best interrupt scheme available on a platform in the
187 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
188 * of these schemes the driver may consider as follows:
189 *
190 * msi = 2: choose from among all three options
191 * msi = 1: only consider MSI and INTx interrupts
192 * msi = 0: force INTx interrupts
193 */
194 static int msi = 2;
195
196 module_param(msi, int, 0644);
197 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
198
199 /*
200 * Queue interrupt hold-off timer values. Queues default to the first of these
201 * upon creation.
202 */
203 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
204
205 module_param_array(intr_holdoff, uint, NULL, 0644);
206 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
207 "0..4 in microseconds, deprecated parameter");
208
209 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
210
211 module_param_array(intr_cnt, uint, NULL, 0644);
212 MODULE_PARM_DESC(intr_cnt,
213 "thresholds 1..3 for queue interrupt packet counters, "
214 "deprecated parameter");
215
216 /*
217 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
218 * offset by 2 bytes in order to have the IP headers line up on 4-byte
219 * boundaries. This is a requirement for many architectures which will throw
220 * a machine check fault if an attempt is made to access one of the 4-byte IP
221 * header fields on a non-4-byte boundary. And it's a major performance issue
222 * even on some architectures which allow it like some implementations of the
223 * x86 ISA. However, some architectures don't mind this and for some very
224 * edge-case performance sensitive applications (like forwarding large volumes
225 * of small packets), setting this DMA offset to 0 will decrease the number of
226 * PCI-E Bus transfers enough to measurably affect performance.
227 */
228 static int rx_dma_offset = 2;
229
230 static bool vf_acls;
231
232 #ifdef CONFIG_PCI_IOV
233 module_param(vf_acls, bool, 0644);
234 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
235 "deprecated parameter");
236
237 /* Configure the number of PCI-E Virtual Function which are to be instantiated
238 * on SR-IOV Capable Physical Functions.
239 */
240 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
241
242 module_param_array(num_vf, uint, NULL, 0644);
243 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
244 #endif
245
246 /* TX Queue select used to determine what algorithm to use for selecting TX
247 * queue. Select between the kernel provided function (select_queue=0) or user
248 * cxgb_select_queue function (select_queue=1)
249 *
250 * Default: select_queue=0
251 */
252 static int select_queue;
253 module_param(select_queue, int, 0644);
254 MODULE_PARM_DESC(select_queue,
255 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
256
257 static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
258
259 module_param(tp_vlan_pri_map, uint, 0644);
260 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
261 "deprecated parameter");
262
263 static struct dentry *cxgb4_debugfs_root;
264
265 static LIST_HEAD(adapter_list);
266 static DEFINE_MUTEX(uld_mutex);
267 /* Adapter list to be accessed from atomic context */
268 static LIST_HEAD(adap_rcu_list);
269 static DEFINE_SPINLOCK(adap_rcu_lock);
270 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
271 static const char *uld_str[] = { "RDMA", "iSCSI" };
272
273 static void link_report(struct net_device *dev)
274 {
275 if (!netif_carrier_ok(dev))
276 netdev_info(dev, "link down\n");
277 else {
278 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
279
280 const char *s = "10Mbps";
281 const struct port_info *p = netdev_priv(dev);
282
283 switch (p->link_cfg.speed) {
284 case 10000:
285 s = "10Gbps";
286 break;
287 case 1000:
288 s = "1000Mbps";
289 break;
290 case 100:
291 s = "100Mbps";
292 break;
293 case 40000:
294 s = "40Gbps";
295 break;
296 }
297
298 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
299 fc[p->link_cfg.fc]);
300 }
301 }
302
303 #ifdef CONFIG_CHELSIO_T4_DCB
304 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */
305 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
306 {
307 struct port_info *pi = netdev_priv(dev);
308 struct adapter *adap = pi->adapter;
309 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
310 int i;
311
312 /* We use a simple mapping of Port TX Queue Index to DCB
313 * Priority when we're enabling DCB.
314 */
315 for (i = 0; i < pi->nqsets; i++, txq++) {
316 u32 name, value;
317 int err;
318
319 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
320 FW_PARAMS_PARAM_X_V(
321 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
322 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
323 value = enable ? i : 0xffffffff;
324
325 /* Since we can be called while atomic (from "interrupt
326 * level") we need to issue the Set Parameters Commannd
327 * without sleeping (timeout < 0).
328 */
329 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
330 &name, &value);
331
332 if (err)
333 dev_err(adap->pdev_dev,
334 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
335 enable ? "set" : "unset", pi->port_id, i, -err);
336 else
337 txq->dcb_prio = value;
338 }
339 }
340 #endif /* CONFIG_CHELSIO_T4_DCB */
341
342 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
343 {
344 struct net_device *dev = adapter->port[port_id];
345
346 /* Skip changes from disabled ports. */
347 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
348 if (link_stat)
349 netif_carrier_on(dev);
350 else {
351 #ifdef CONFIG_CHELSIO_T4_DCB
352 cxgb4_dcb_state_init(dev);
353 dcb_tx_queue_prio_enable(dev, false);
354 #endif /* CONFIG_CHELSIO_T4_DCB */
355 netif_carrier_off(dev);
356 }
357
358 link_report(dev);
359 }
360 }
361
362 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
363 {
364 static const char *mod_str[] = {
365 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
366 };
367
368 const struct net_device *dev = adap->port[port_id];
369 const struct port_info *pi = netdev_priv(dev);
370
371 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
372 netdev_info(dev, "port module unplugged\n");
373 else if (pi->mod_type < ARRAY_SIZE(mod_str))
374 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
375 }
376
377 /*
378 * Configure the exact and hash address filters to handle a port's multicast
379 * and secondary unicast MAC addresses.
380 */
381 static int set_addr_filters(const struct net_device *dev, bool sleep)
382 {
383 u64 mhash = 0;
384 u64 uhash = 0;
385 bool free = true;
386 u16 filt_idx[7];
387 const u8 *addr[7];
388 int ret, naddr = 0;
389 const struct netdev_hw_addr *ha;
390 int uc_cnt = netdev_uc_count(dev);
391 int mc_cnt = netdev_mc_count(dev);
392 const struct port_info *pi = netdev_priv(dev);
393 unsigned int mb = pi->adapter->fn;
394
395 /* first do the secondary unicast addresses */
396 netdev_for_each_uc_addr(ha, dev) {
397 addr[naddr++] = ha->addr;
398 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
399 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
400 naddr, addr, filt_idx, &uhash, sleep);
401 if (ret < 0)
402 return ret;
403
404 free = false;
405 naddr = 0;
406 }
407 }
408
409 /* next set up the multicast addresses */
410 netdev_for_each_mc_addr(ha, dev) {
411 addr[naddr++] = ha->addr;
412 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
413 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
414 naddr, addr, filt_idx, &mhash, sleep);
415 if (ret < 0)
416 return ret;
417
418 free = false;
419 naddr = 0;
420 }
421 }
422
423 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
424 uhash | mhash, sleep);
425 }
426
427 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
428 module_param(dbfifo_int_thresh, int, 0644);
429 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
430
431 /*
432 * usecs to sleep while draining the dbfifo
433 */
434 static int dbfifo_drain_delay = 1000;
435 module_param(dbfifo_drain_delay, int, 0644);
436 MODULE_PARM_DESC(dbfifo_drain_delay,
437 "usecs to sleep while draining the dbfifo");
438
439 /*
440 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
441 * If @mtu is -1 it is left unchanged.
442 */
443 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
444 {
445 int ret;
446 struct port_info *pi = netdev_priv(dev);
447
448 ret = set_addr_filters(dev, sleep_ok);
449 if (ret == 0)
450 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
451 (dev->flags & IFF_PROMISC) ? 1 : 0,
452 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
453 sleep_ok);
454 return ret;
455 }
456
457 /**
458 * link_start - enable a port
459 * @dev: the port to enable
460 *
461 * Performs the MAC and PHY actions needed to enable a port.
462 */
463 static int link_start(struct net_device *dev)
464 {
465 int ret;
466 struct port_info *pi = netdev_priv(dev);
467 unsigned int mb = pi->adapter->fn;
468
469 /*
470 * We do not set address filters and promiscuity here, the stack does
471 * that step explicitly.
472 */
473 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
474 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
475 if (ret == 0) {
476 ret = t4_change_mac(pi->adapter, mb, pi->viid,
477 pi->xact_addr_filt, dev->dev_addr, true,
478 true);
479 if (ret >= 0) {
480 pi->xact_addr_filt = ret;
481 ret = 0;
482 }
483 }
484 if (ret == 0)
485 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
486 &pi->link_cfg);
487 if (ret == 0) {
488 local_bh_disable();
489 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
490 true, CXGB4_DCB_ENABLED);
491 local_bh_enable();
492 }
493
494 return ret;
495 }
496
497 int cxgb4_dcb_enabled(const struct net_device *dev)
498 {
499 #ifdef CONFIG_CHELSIO_T4_DCB
500 struct port_info *pi = netdev_priv(dev);
501
502 if (!pi->dcb.enabled)
503 return 0;
504
505 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
506 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
507 #else
508 return 0;
509 #endif
510 }
511 EXPORT_SYMBOL(cxgb4_dcb_enabled);
512
513 #ifdef CONFIG_CHELSIO_T4_DCB
514 /* Handle a Data Center Bridging update message from the firmware. */
515 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
516 {
517 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
518 struct net_device *dev = adap->port[port];
519 int old_dcb_enabled = cxgb4_dcb_enabled(dev);
520 int new_dcb_enabled;
521
522 cxgb4_dcb_handle_fw_update(adap, pcmd);
523 new_dcb_enabled = cxgb4_dcb_enabled(dev);
524
525 /* If the DCB has become enabled or disabled on the port then we're
526 * going to need to set up/tear down DCB Priority parameters for the
527 * TX Queues associated with the port.
528 */
529 if (new_dcb_enabled != old_dcb_enabled)
530 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
531 }
532 #endif /* CONFIG_CHELSIO_T4_DCB */
533
534 /* Clear a filter and release any of its resources that we own. This also
535 * clears the filter's "pending" status.
536 */
537 static void clear_filter(struct adapter *adap, struct filter_entry *f)
538 {
539 /* If the new or old filter have loopback rewriteing rules then we'll
540 * need to free any existing Layer Two Table (L2T) entries of the old
541 * filter rule. The firmware will handle freeing up any Source MAC
542 * Table (SMT) entries used for rewriting Source MAC Addresses in
543 * loopback rules.
544 */
545 if (f->l2t)
546 cxgb4_l2t_release(f->l2t);
547
548 /* The zeroing of the filter rule below clears the filter valid,
549 * pending, locked flags, l2t pointer, etc. so it's all we need for
550 * this operation.
551 */
552 memset(f, 0, sizeof(*f));
553 }
554
555 /* Handle a filter write/deletion reply.
556 */
557 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
558 {
559 unsigned int idx = GET_TID(rpl);
560 unsigned int nidx = idx - adap->tids.ftid_base;
561 unsigned int ret;
562 struct filter_entry *f;
563
564 if (idx >= adap->tids.ftid_base && nidx <
565 (adap->tids.nftids + adap->tids.nsftids)) {
566 idx = nidx;
567 ret = TCB_COOKIE_G(rpl->cookie);
568 f = &adap->tids.ftid_tab[idx];
569
570 if (ret == FW_FILTER_WR_FLT_DELETED) {
571 /* Clear the filter when we get confirmation from the
572 * hardware that the filter has been deleted.
573 */
574 clear_filter(adap, f);
575 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
576 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
577 idx);
578 clear_filter(adap, f);
579 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
580 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
581 f->pending = 0; /* asynchronous setup completed */
582 f->valid = 1;
583 } else {
584 /* Something went wrong. Issue a warning about the
585 * problem and clear everything out.
586 */
587 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
588 idx, ret);
589 clear_filter(adap, f);
590 }
591 }
592 }
593
594 /* Response queue handler for the FW event queue.
595 */
596 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
597 const struct pkt_gl *gl)
598 {
599 u8 opcode = ((const struct rss_header *)rsp)->opcode;
600
601 rsp++; /* skip RSS header */
602
603 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
604 */
605 if (unlikely(opcode == CPL_FW4_MSG &&
606 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
607 rsp++;
608 opcode = ((const struct rss_header *)rsp)->opcode;
609 rsp++;
610 if (opcode != CPL_SGE_EGR_UPDATE) {
611 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
612 , opcode);
613 goto out;
614 }
615 }
616
617 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
618 const struct cpl_sge_egr_update *p = (void *)rsp;
619 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
620 struct sge_txq *txq;
621
622 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
623 txq->restarts++;
624 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
625 struct sge_eth_txq *eq;
626
627 eq = container_of(txq, struct sge_eth_txq, q);
628 netif_tx_wake_queue(eq->txq);
629 } else {
630 struct sge_ofld_txq *oq;
631
632 oq = container_of(txq, struct sge_ofld_txq, q);
633 tasklet_schedule(&oq->qresume_tsk);
634 }
635 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
636 const struct cpl_fw6_msg *p = (void *)rsp;
637
638 #ifdef CONFIG_CHELSIO_T4_DCB
639 const struct fw_port_cmd *pcmd = (const void *)p->data;
640 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
641 unsigned int action =
642 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
643
644 if (cmd == FW_PORT_CMD &&
645 action == FW_PORT_ACTION_GET_PORT_INFO) {
646 int port = FW_PORT_CMD_PORTID_G(
647 be32_to_cpu(pcmd->op_to_portid));
648 struct net_device *dev = q->adap->port[port];
649 int state_input = ((pcmd->u.info.dcbxdis_pkd &
650 FW_PORT_CMD_DCBXDIS_F)
651 ? CXGB4_DCB_INPUT_FW_DISABLED
652 : CXGB4_DCB_INPUT_FW_ENABLED);
653
654 cxgb4_dcb_state_fsm(dev, state_input);
655 }
656
657 if (cmd == FW_PORT_CMD &&
658 action == FW_PORT_ACTION_L2_DCB_CFG)
659 dcb_rpl(q->adap, pcmd);
660 else
661 #endif
662 if (p->type == 0)
663 t4_handle_fw_rpl(q->adap, p->data);
664 } else if (opcode == CPL_L2T_WRITE_RPL) {
665 const struct cpl_l2t_write_rpl *p = (void *)rsp;
666
667 do_l2t_write_rpl(q->adap, p);
668 } else if (opcode == CPL_SET_TCB_RPL) {
669 const struct cpl_set_tcb_rpl *p = (void *)rsp;
670
671 filter_rpl(q->adap, p);
672 } else
673 dev_err(q->adap->pdev_dev,
674 "unexpected CPL %#x on FW event queue\n", opcode);
675 out:
676 return 0;
677 }
678
679 /**
680 * uldrx_handler - response queue handler for ULD queues
681 * @q: the response queue that received the packet
682 * @rsp: the response queue descriptor holding the offload message
683 * @gl: the gather list of packet fragments
684 *
685 * Deliver an ingress offload packet to a ULD. All processing is done by
686 * the ULD, we just maintain statistics.
687 */
688 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
689 const struct pkt_gl *gl)
690 {
691 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
692
693 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
694 */
695 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
696 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
697 rsp += 2;
698
699 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
700 rxq->stats.nomem++;
701 return -1;
702 }
703 if (gl == NULL)
704 rxq->stats.imm++;
705 else if (gl == CXGB4_MSG_AN)
706 rxq->stats.an++;
707 else
708 rxq->stats.pkts++;
709 return 0;
710 }
711
712 static void disable_msi(struct adapter *adapter)
713 {
714 if (adapter->flags & USING_MSIX) {
715 pci_disable_msix(adapter->pdev);
716 adapter->flags &= ~USING_MSIX;
717 } else if (adapter->flags & USING_MSI) {
718 pci_disable_msi(adapter->pdev);
719 adapter->flags &= ~USING_MSI;
720 }
721 }
722
723 /*
724 * Interrupt handler for non-data events used with MSI-X.
725 */
726 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
727 {
728 struct adapter *adap = cookie;
729 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
730
731 if (v & PFSW_F) {
732 adap->swintr = 1;
733 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
734 }
735 t4_slow_intr_handler(adap);
736 return IRQ_HANDLED;
737 }
738
739 /*
740 * Name the MSI-X interrupts.
741 */
742 static void name_msix_vecs(struct adapter *adap)
743 {
744 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
745
746 /* non-data interrupts */
747 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
748
749 /* FW events */
750 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
751 adap->port[0]->name);
752
753 /* Ethernet queues */
754 for_each_port(adap, j) {
755 struct net_device *d = adap->port[j];
756 const struct port_info *pi = netdev_priv(d);
757
758 for (i = 0; i < pi->nqsets; i++, msi_idx++)
759 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
760 d->name, i);
761 }
762
763 /* offload queues */
764 for_each_ofldrxq(&adap->sge, i)
765 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
766 adap->port[0]->name, i);
767
768 for_each_rdmarxq(&adap->sge, i)
769 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
770 adap->port[0]->name, i);
771
772 for_each_rdmaciq(&adap->sge, i)
773 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
774 adap->port[0]->name, i);
775 }
776
777 static int request_msix_queue_irqs(struct adapter *adap)
778 {
779 struct sge *s = &adap->sge;
780 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
781 int msi_index = 2;
782
783 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
784 adap->msix_info[1].desc, &s->fw_evtq);
785 if (err)
786 return err;
787
788 for_each_ethrxq(s, ethqidx) {
789 err = request_irq(adap->msix_info[msi_index].vec,
790 t4_sge_intr_msix, 0,
791 adap->msix_info[msi_index].desc,
792 &s->ethrxq[ethqidx].rspq);
793 if (err)
794 goto unwind;
795 msi_index++;
796 }
797 for_each_ofldrxq(s, ofldqidx) {
798 err = request_irq(adap->msix_info[msi_index].vec,
799 t4_sge_intr_msix, 0,
800 adap->msix_info[msi_index].desc,
801 &s->ofldrxq[ofldqidx].rspq);
802 if (err)
803 goto unwind;
804 msi_index++;
805 }
806 for_each_rdmarxq(s, rdmaqidx) {
807 err = request_irq(adap->msix_info[msi_index].vec,
808 t4_sge_intr_msix, 0,
809 adap->msix_info[msi_index].desc,
810 &s->rdmarxq[rdmaqidx].rspq);
811 if (err)
812 goto unwind;
813 msi_index++;
814 }
815 for_each_rdmaciq(s, rdmaciqqidx) {
816 err = request_irq(adap->msix_info[msi_index].vec,
817 t4_sge_intr_msix, 0,
818 adap->msix_info[msi_index].desc,
819 &s->rdmaciq[rdmaciqqidx].rspq);
820 if (err)
821 goto unwind;
822 msi_index++;
823 }
824 return 0;
825
826 unwind:
827 while (--rdmaciqqidx >= 0)
828 free_irq(adap->msix_info[--msi_index].vec,
829 &s->rdmaciq[rdmaciqqidx].rspq);
830 while (--rdmaqidx >= 0)
831 free_irq(adap->msix_info[--msi_index].vec,
832 &s->rdmarxq[rdmaqidx].rspq);
833 while (--ofldqidx >= 0)
834 free_irq(adap->msix_info[--msi_index].vec,
835 &s->ofldrxq[ofldqidx].rspq);
836 while (--ethqidx >= 0)
837 free_irq(adap->msix_info[--msi_index].vec,
838 &s->ethrxq[ethqidx].rspq);
839 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
840 return err;
841 }
842
843 static void free_msix_queue_irqs(struct adapter *adap)
844 {
845 int i, msi_index = 2;
846 struct sge *s = &adap->sge;
847
848 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
849 for_each_ethrxq(s, i)
850 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
851 for_each_ofldrxq(s, i)
852 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
853 for_each_rdmarxq(s, i)
854 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
855 for_each_rdmaciq(s, i)
856 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
857 }
858
859 /**
860 * write_rss - write the RSS table for a given port
861 * @pi: the port
862 * @queues: array of queue indices for RSS
863 *
864 * Sets up the portion of the HW RSS table for the port's VI to distribute
865 * packets to the Rx queues in @queues.
866 */
867 static int write_rss(const struct port_info *pi, const u16 *queues)
868 {
869 u16 *rss;
870 int i, err;
871 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
872
873 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
874 if (!rss)
875 return -ENOMEM;
876
877 /* map the queue indices to queue ids */
878 for (i = 0; i < pi->rss_size; i++, queues++)
879 rss[i] = q[*queues].rspq.abs_id;
880
881 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
882 pi->rss_size, rss, pi->rss_size);
883 kfree(rss);
884 return err;
885 }
886
887 /**
888 * setup_rss - configure RSS
889 * @adap: the adapter
890 *
891 * Sets up RSS for each port.
892 */
893 static int setup_rss(struct adapter *adap)
894 {
895 int i, err;
896
897 for_each_port(adap, i) {
898 const struct port_info *pi = adap2pinfo(adap, i);
899
900 err = write_rss(pi, pi->rss);
901 if (err)
902 return err;
903 }
904 return 0;
905 }
906
907 /*
908 * Return the channel of the ingress queue with the given qid.
909 */
910 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
911 {
912 qid -= p->ingr_start;
913 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
914 }
915
916 /*
917 * Wait until all NAPI handlers are descheduled.
918 */
919 static void quiesce_rx(struct adapter *adap)
920 {
921 int i;
922
923 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
924 struct sge_rspq *q = adap->sge.ingr_map[i];
925
926 if (q && q->handler) {
927 napi_disable(&q->napi);
928 local_bh_disable();
929 while (!cxgb_poll_lock_napi(q))
930 mdelay(1);
931 local_bh_enable();
932 }
933
934 }
935 }
936
937 /*
938 * Enable NAPI scheduling and interrupt generation for all Rx queues.
939 */
940 static void enable_rx(struct adapter *adap)
941 {
942 int i;
943
944 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
945 struct sge_rspq *q = adap->sge.ingr_map[i];
946
947 if (!q)
948 continue;
949 if (q->handler) {
950 cxgb_busy_poll_init_lock(q);
951 napi_enable(&q->napi);
952 }
953 /* 0-increment GTS to start the timer and enable interrupts */
954 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
955 SEINTARM_V(q->intr_params) |
956 INGRESSQID_V(q->cntxt_id));
957 }
958 }
959
960 /**
961 * setup_sge_queues - configure SGE Tx/Rx/response queues
962 * @adap: the adapter
963 *
964 * Determines how many sets of SGE queues to use and initializes them.
965 * We support multiple queue sets per port if we have MSI-X, otherwise
966 * just one queue set per port.
967 */
968 static int setup_sge_queues(struct adapter *adap)
969 {
970 int err, msi_idx, i, j;
971 struct sge *s = &adap->sge;
972
973 bitmap_zero(s->starving_fl, MAX_EGRQ);
974 bitmap_zero(s->txq_maperr, MAX_EGRQ);
975
976 if (adap->flags & USING_MSIX)
977 msi_idx = 1; /* vector 0 is for non-queue interrupts */
978 else {
979 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
980 NULL, NULL);
981 if (err)
982 return err;
983 msi_idx = -((int)s->intrq.abs_id + 1);
984 }
985
986 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
987 msi_idx, NULL, fwevtq_handler);
988 if (err) {
989 freeout: t4_free_sge_resources(adap);
990 return err;
991 }
992
993 for_each_port(adap, i) {
994 struct net_device *dev = adap->port[i];
995 struct port_info *pi = netdev_priv(dev);
996 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
997 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
998
999 for (j = 0; j < pi->nqsets; j++, q++) {
1000 if (msi_idx > 0)
1001 msi_idx++;
1002 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1003 msi_idx, &q->fl,
1004 t4_ethrx_handler);
1005 if (err)
1006 goto freeout;
1007 q->rspq.idx = j;
1008 memset(&q->stats, 0, sizeof(q->stats));
1009 }
1010 for (j = 0; j < pi->nqsets; j++, t++) {
1011 err = t4_sge_alloc_eth_txq(adap, t, dev,
1012 netdev_get_tx_queue(dev, j),
1013 s->fw_evtq.cntxt_id);
1014 if (err)
1015 goto freeout;
1016 }
1017 }
1018
1019 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1020 for_each_ofldrxq(s, i) {
1021 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1022 struct net_device *dev = adap->port[i / j];
1023
1024 if (msi_idx > 0)
1025 msi_idx++;
1026 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1027 q->fl.size ? &q->fl : NULL,
1028 uldrx_handler);
1029 if (err)
1030 goto freeout;
1031 memset(&q->stats, 0, sizeof(q->stats));
1032 s->ofld_rxq[i] = q->rspq.abs_id;
1033 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1034 s->fw_evtq.cntxt_id);
1035 if (err)
1036 goto freeout;
1037 }
1038
1039 for_each_rdmarxq(s, i) {
1040 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1041
1042 if (msi_idx > 0)
1043 msi_idx++;
1044 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1045 msi_idx, q->fl.size ? &q->fl : NULL,
1046 uldrx_handler);
1047 if (err)
1048 goto freeout;
1049 memset(&q->stats, 0, sizeof(q->stats));
1050 s->rdma_rxq[i] = q->rspq.abs_id;
1051 }
1052
1053 for_each_rdmaciq(s, i) {
1054 struct sge_ofld_rxq *q = &s->rdmaciq[i];
1055
1056 if (msi_idx > 0)
1057 msi_idx++;
1058 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1059 msi_idx, q->fl.size ? &q->fl : NULL,
1060 uldrx_handler);
1061 if (err)
1062 goto freeout;
1063 memset(&q->stats, 0, sizeof(q->stats));
1064 s->rdma_ciq[i] = q->rspq.abs_id;
1065 }
1066
1067 for_each_port(adap, i) {
1068 /*
1069 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1070 * have RDMA queues, and that's the right value.
1071 */
1072 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1073 s->fw_evtq.cntxt_id,
1074 s->rdmarxq[i].rspq.cntxt_id);
1075 if (err)
1076 goto freeout;
1077 }
1078
1079 t4_write_reg(adap, is_t4(adap->params.chip) ?
1080 MPS_TRC_RSS_CONTROL_A :
1081 MPS_T5_TRC_RSS_CONTROL_A,
1082 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1083 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
1084 return 0;
1085 }
1086
1087 /*
1088 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1089 * The allocated memory is cleared.
1090 */
1091 void *t4_alloc_mem(size_t size)
1092 {
1093 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1094
1095 if (!p)
1096 p = vzalloc(size);
1097 return p;
1098 }
1099
1100 /*
1101 * Free memory allocated through alloc_mem().
1102 */
1103 void t4_free_mem(void *addr)
1104 {
1105 if (is_vmalloc_addr(addr))
1106 vfree(addr);
1107 else
1108 kfree(addr);
1109 }
1110
1111 /* Send a Work Request to write the filter at a specified index. We construct
1112 * a Firmware Filter Work Request to have the work done and put the indicated
1113 * filter into "pending" mode which will prevent any further actions against
1114 * it till we get a reply from the firmware on the completion status of the
1115 * request.
1116 */
1117 static int set_filter_wr(struct adapter *adapter, int fidx)
1118 {
1119 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1120 struct sk_buff *skb;
1121 struct fw_filter_wr *fwr;
1122 unsigned int ftid;
1123
1124 /* If the new filter requires loopback Destination MAC and/or VLAN
1125 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1126 * the filter.
1127 */
1128 if (f->fs.newdmac || f->fs.newvlan) {
1129 /* allocate L2T entry for new filter */
1130 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1131 if (f->l2t == NULL)
1132 return -EAGAIN;
1133 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1134 f->fs.eport, f->fs.dmac)) {
1135 cxgb4_l2t_release(f->l2t);
1136 f->l2t = NULL;
1137 return -ENOMEM;
1138 }
1139 }
1140
1141 ftid = adapter->tids.ftid_base + fidx;
1142
1143 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1144 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1145 memset(fwr, 0, sizeof(*fwr));
1146
1147 /* It would be nice to put most of the following in t4_hw.c but most
1148 * of the work is translating the cxgbtool ch_filter_specification
1149 * into the Work Request and the definition of that structure is
1150 * currently in cxgbtool.h which isn't appropriate to pull into the
1151 * common code. We may eventually try to come up with a more neutral
1152 * filter specification structure but for now it's easiest to simply
1153 * put this fairly direct code in line ...
1154 */
1155 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1156 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
1157 fwr->tid_to_iq =
1158 htonl(FW_FILTER_WR_TID_V(ftid) |
1159 FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1160 FW_FILTER_WR_NOREPLY_V(0) |
1161 FW_FILTER_WR_IQ_V(f->fs.iq));
1162 fwr->del_filter_to_l2tix =
1163 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1164 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1165 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1166 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1167 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1168 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1169 FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1170 FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1171 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
1172 f->fs.newvlan == VLAN_REWRITE) |
1173 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
1174 f->fs.newvlan == VLAN_REWRITE) |
1175 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1176 FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1177 FW_FILTER_WR_PRIO_V(f->fs.prio) |
1178 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
1179 fwr->ethtype = htons(f->fs.val.ethtype);
1180 fwr->ethtypem = htons(f->fs.mask.ethtype);
1181 fwr->frag_to_ovlan_vldm =
1182 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1183 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1184 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1185 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1186 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1187 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
1188 fwr->smac_sel = 0;
1189 fwr->rx_chan_rx_rpl_iq =
1190 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1191 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
1192 fwr->maci_to_matchtypem =
1193 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1194 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1195 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1196 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1197 FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1198 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1199 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1200 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
1201 fwr->ptcl = f->fs.val.proto;
1202 fwr->ptclm = f->fs.mask.proto;
1203 fwr->ttyp = f->fs.val.tos;
1204 fwr->ttypm = f->fs.mask.tos;
1205 fwr->ivlan = htons(f->fs.val.ivlan);
1206 fwr->ivlanm = htons(f->fs.mask.ivlan);
1207 fwr->ovlan = htons(f->fs.val.ovlan);
1208 fwr->ovlanm = htons(f->fs.mask.ovlan);
1209 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1210 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1211 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1212 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1213 fwr->lp = htons(f->fs.val.lport);
1214 fwr->lpm = htons(f->fs.mask.lport);
1215 fwr->fp = htons(f->fs.val.fport);
1216 fwr->fpm = htons(f->fs.mask.fport);
1217 if (f->fs.newsmac)
1218 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1219
1220 /* Mark the filter as "pending" and ship off the Filter Work Request.
1221 * When we get the Work Request Reply we'll clear the pending status.
1222 */
1223 f->pending = 1;
1224 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1225 t4_ofld_send(adapter, skb);
1226 return 0;
1227 }
1228
1229 /* Delete the filter at a specified index.
1230 */
1231 static int del_filter_wr(struct adapter *adapter, int fidx)
1232 {
1233 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1234 struct sk_buff *skb;
1235 struct fw_filter_wr *fwr;
1236 unsigned int len, ftid;
1237
1238 len = sizeof(*fwr);
1239 ftid = adapter->tids.ftid_base + fidx;
1240
1241 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1242 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1243 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1244
1245 /* Mark the filter as "pending" and ship off the Filter Work Request.
1246 * When we get the Work Request Reply we'll clear the pending status.
1247 */
1248 f->pending = 1;
1249 t4_mgmt_tx(adapter, skb);
1250 return 0;
1251 }
1252
1253 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1254 void *accel_priv, select_queue_fallback_t fallback)
1255 {
1256 int txq;
1257
1258 #ifdef CONFIG_CHELSIO_T4_DCB
1259 /* If a Data Center Bridging has been successfully negotiated on this
1260 * link then we'll use the skb's priority to map it to a TX Queue.
1261 * The skb's priority is determined via the VLAN Tag Priority Code
1262 * Point field.
1263 */
1264 if (cxgb4_dcb_enabled(dev)) {
1265 u16 vlan_tci;
1266 int err;
1267
1268 err = vlan_get_tag(skb, &vlan_tci);
1269 if (unlikely(err)) {
1270 if (net_ratelimit())
1271 netdev_warn(dev,
1272 "TX Packet without VLAN Tag on DCB Link\n");
1273 txq = 0;
1274 } else {
1275 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1276 }
1277 return txq;
1278 }
1279 #endif /* CONFIG_CHELSIO_T4_DCB */
1280
1281 if (select_queue) {
1282 txq = (skb_rx_queue_recorded(skb)
1283 ? skb_get_rx_queue(skb)
1284 : smp_processor_id());
1285
1286 while (unlikely(txq >= dev->real_num_tx_queues))
1287 txq -= dev->real_num_tx_queues;
1288
1289 return txq;
1290 }
1291
1292 return fallback(dev, skb) % dev->real_num_tx_queues;
1293 }
1294
1295 static inline int is_offload(const struct adapter *adap)
1296 {
1297 return adap->params.offload;
1298 }
1299
1300 /*
1301 * Implementation of ethtool operations.
1302 */
1303
1304 static u32 get_msglevel(struct net_device *dev)
1305 {
1306 return netdev2adap(dev)->msg_enable;
1307 }
1308
1309 static void set_msglevel(struct net_device *dev, u32 val)
1310 {
1311 netdev2adap(dev)->msg_enable = val;
1312 }
1313
1314 static char stats_strings[][ETH_GSTRING_LEN] = {
1315 "TxOctetsOK ",
1316 "TxFramesOK ",
1317 "TxBroadcastFrames ",
1318 "TxMulticastFrames ",
1319 "TxUnicastFrames ",
1320 "TxErrorFrames ",
1321
1322 "TxFrames64 ",
1323 "TxFrames65To127 ",
1324 "TxFrames128To255 ",
1325 "TxFrames256To511 ",
1326 "TxFrames512To1023 ",
1327 "TxFrames1024To1518 ",
1328 "TxFrames1519ToMax ",
1329
1330 "TxFramesDropped ",
1331 "TxPauseFrames ",
1332 "TxPPP0Frames ",
1333 "TxPPP1Frames ",
1334 "TxPPP2Frames ",
1335 "TxPPP3Frames ",
1336 "TxPPP4Frames ",
1337 "TxPPP5Frames ",
1338 "TxPPP6Frames ",
1339 "TxPPP7Frames ",
1340
1341 "RxOctetsOK ",
1342 "RxFramesOK ",
1343 "RxBroadcastFrames ",
1344 "RxMulticastFrames ",
1345 "RxUnicastFrames ",
1346
1347 "RxFramesTooLong ",
1348 "RxJabberErrors ",
1349 "RxFCSErrors ",
1350 "RxLengthErrors ",
1351 "RxSymbolErrors ",
1352 "RxRuntFrames ",
1353
1354 "RxFrames64 ",
1355 "RxFrames65To127 ",
1356 "RxFrames128To255 ",
1357 "RxFrames256To511 ",
1358 "RxFrames512To1023 ",
1359 "RxFrames1024To1518 ",
1360 "RxFrames1519ToMax ",
1361
1362 "RxPauseFrames ",
1363 "RxPPP0Frames ",
1364 "RxPPP1Frames ",
1365 "RxPPP2Frames ",
1366 "RxPPP3Frames ",
1367 "RxPPP4Frames ",
1368 "RxPPP5Frames ",
1369 "RxPPP6Frames ",
1370 "RxPPP7Frames ",
1371
1372 "RxBG0FramesDropped ",
1373 "RxBG1FramesDropped ",
1374 "RxBG2FramesDropped ",
1375 "RxBG3FramesDropped ",
1376 "RxBG0FramesTrunc ",
1377 "RxBG1FramesTrunc ",
1378 "RxBG2FramesTrunc ",
1379 "RxBG3FramesTrunc ",
1380
1381 "TSO ",
1382 "TxCsumOffload ",
1383 "RxCsumGood ",
1384 "VLANextractions ",
1385 "VLANinsertions ",
1386 "GROpackets ",
1387 "GROmerged ",
1388 "WriteCoalSuccess ",
1389 "WriteCoalFail ",
1390 };
1391
1392 static int get_sset_count(struct net_device *dev, int sset)
1393 {
1394 switch (sset) {
1395 case ETH_SS_STATS:
1396 return ARRAY_SIZE(stats_strings);
1397 default:
1398 return -EOPNOTSUPP;
1399 }
1400 }
1401
1402 #define T4_REGMAP_SIZE (160 * 1024)
1403 #define T5_REGMAP_SIZE (332 * 1024)
1404
1405 static int get_regs_len(struct net_device *dev)
1406 {
1407 struct adapter *adap = netdev2adap(dev);
1408 if (is_t4(adap->params.chip))
1409 return T4_REGMAP_SIZE;
1410 else
1411 return T5_REGMAP_SIZE;
1412 }
1413
1414 static int get_eeprom_len(struct net_device *dev)
1415 {
1416 return EEPROMSIZE;
1417 }
1418
1419 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1420 {
1421 struct adapter *adapter = netdev2adap(dev);
1422 u32 exprom_vers;
1423
1424 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1425 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1426 strlcpy(info->bus_info, pci_name(adapter->pdev),
1427 sizeof(info->bus_info));
1428
1429 if (adapter->params.fw_vers)
1430 snprintf(info->fw_version, sizeof(info->fw_version),
1431 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1432 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1433 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1434 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1435 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1436 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1437 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1438 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1439 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
1440
1441 if (!t4_get_exprom_version(adapter, &exprom_vers))
1442 snprintf(info->erom_version, sizeof(info->erom_version),
1443 "%u.%u.%u.%u",
1444 FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1445 FW_HDR_FW_VER_MINOR_G(exprom_vers),
1446 FW_HDR_FW_VER_MICRO_G(exprom_vers),
1447 FW_HDR_FW_VER_BUILD_G(exprom_vers));
1448 }
1449
1450 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1451 {
1452 if (stringset == ETH_SS_STATS)
1453 memcpy(data, stats_strings, sizeof(stats_strings));
1454 }
1455
1456 /*
1457 * port stats maintained per queue of the port. They should be in the same
1458 * order as in stats_strings above.
1459 */
1460 struct queue_port_stats {
1461 u64 tso;
1462 u64 tx_csum;
1463 u64 rx_csum;
1464 u64 vlan_ex;
1465 u64 vlan_ins;
1466 u64 gro_pkts;
1467 u64 gro_merged;
1468 };
1469
1470 static void collect_sge_port_stats(const struct adapter *adap,
1471 const struct port_info *p, struct queue_port_stats *s)
1472 {
1473 int i;
1474 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1475 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1476
1477 memset(s, 0, sizeof(*s));
1478 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1479 s->tso += tx->tso;
1480 s->tx_csum += tx->tx_cso;
1481 s->rx_csum += rx->stats.rx_cso;
1482 s->vlan_ex += rx->stats.vlan_ex;
1483 s->vlan_ins += tx->vlan_ins;
1484 s->gro_pkts += rx->stats.lro_pkts;
1485 s->gro_merged += rx->stats.lro_merged;
1486 }
1487 }
1488
1489 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1490 u64 *data)
1491 {
1492 struct port_info *pi = netdev_priv(dev);
1493 struct adapter *adapter = pi->adapter;
1494 u32 val1, val2;
1495
1496 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1497
1498 data += sizeof(struct port_stats) / sizeof(u64);
1499 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1500 data += sizeof(struct queue_port_stats) / sizeof(u64);
1501 if (!is_t4(adapter->params.chip)) {
1502 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1503 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1504 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
1505 *data = val1 - val2;
1506 data++;
1507 *data = val2;
1508 data++;
1509 } else {
1510 memset(data, 0, 2 * sizeof(u64));
1511 *data += 2;
1512 }
1513 }
1514
1515 /*
1516 * Return a version number to identify the type of adapter. The scheme is:
1517 * - bits 0..9: chip version
1518 * - bits 10..15: chip revision
1519 * - bits 16..23: register dump version
1520 */
1521 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1522 {
1523 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1524 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1525 }
1526
1527 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1528 unsigned int end)
1529 {
1530 u32 *p = buf + start;
1531
1532 for ( ; start <= end; start += sizeof(u32))
1533 *p++ = t4_read_reg(ap, start);
1534 }
1535
1536 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1537 void *buf)
1538 {
1539 static const unsigned int t4_reg_ranges[] = {
1540 0x1008, 0x1108,
1541 0x1180, 0x11b4,
1542 0x11fc, 0x123c,
1543 0x1300, 0x173c,
1544 0x1800, 0x18fc,
1545 0x3000, 0x30d8,
1546 0x30e0, 0x5924,
1547 0x5960, 0x59d4,
1548 0x5a00, 0x5af8,
1549 0x6000, 0x6098,
1550 0x6100, 0x6150,
1551 0x6200, 0x6208,
1552 0x6240, 0x6248,
1553 0x6280, 0x6338,
1554 0x6370, 0x638c,
1555 0x6400, 0x643c,
1556 0x6500, 0x6524,
1557 0x6a00, 0x6a38,
1558 0x6a60, 0x6a78,
1559 0x6b00, 0x6b84,
1560 0x6bf0, 0x6c84,
1561 0x6cf0, 0x6d84,
1562 0x6df0, 0x6e84,
1563 0x6ef0, 0x6f84,
1564 0x6ff0, 0x7084,
1565 0x70f0, 0x7184,
1566 0x71f0, 0x7284,
1567 0x72f0, 0x7384,
1568 0x73f0, 0x7450,
1569 0x7500, 0x7530,
1570 0x7600, 0x761c,
1571 0x7680, 0x76cc,
1572 0x7700, 0x7798,
1573 0x77c0, 0x77fc,
1574 0x7900, 0x79fc,
1575 0x7b00, 0x7c38,
1576 0x7d00, 0x7efc,
1577 0x8dc0, 0x8e1c,
1578 0x8e30, 0x8e78,
1579 0x8ea0, 0x8f6c,
1580 0x8fc0, 0x9074,
1581 0x90fc, 0x90fc,
1582 0x9400, 0x9458,
1583 0x9600, 0x96bc,
1584 0x9800, 0x9808,
1585 0x9820, 0x983c,
1586 0x9850, 0x9864,
1587 0x9c00, 0x9c6c,
1588 0x9c80, 0x9cec,
1589 0x9d00, 0x9d6c,
1590 0x9d80, 0x9dec,
1591 0x9e00, 0x9e6c,
1592 0x9e80, 0x9eec,
1593 0x9f00, 0x9f6c,
1594 0x9f80, 0x9fec,
1595 0xd004, 0xd03c,
1596 0xdfc0, 0xdfe0,
1597 0xe000, 0xea7c,
1598 0xf000, 0x11110,
1599 0x11118, 0x11190,
1600 0x19040, 0x1906c,
1601 0x19078, 0x19080,
1602 0x1908c, 0x19124,
1603 0x19150, 0x191b0,
1604 0x191d0, 0x191e8,
1605 0x19238, 0x1924c,
1606 0x193f8, 0x19474,
1607 0x19490, 0x194f8,
1608 0x19800, 0x19f30,
1609 0x1a000, 0x1a06c,
1610 0x1a0b0, 0x1a120,
1611 0x1a128, 0x1a138,
1612 0x1a190, 0x1a1c4,
1613 0x1a1fc, 0x1a1fc,
1614 0x1e040, 0x1e04c,
1615 0x1e284, 0x1e28c,
1616 0x1e2c0, 0x1e2c0,
1617 0x1e2e0, 0x1e2e0,
1618 0x1e300, 0x1e384,
1619 0x1e3c0, 0x1e3c8,
1620 0x1e440, 0x1e44c,
1621 0x1e684, 0x1e68c,
1622 0x1e6c0, 0x1e6c0,
1623 0x1e6e0, 0x1e6e0,
1624 0x1e700, 0x1e784,
1625 0x1e7c0, 0x1e7c8,
1626 0x1e840, 0x1e84c,
1627 0x1ea84, 0x1ea8c,
1628 0x1eac0, 0x1eac0,
1629 0x1eae0, 0x1eae0,
1630 0x1eb00, 0x1eb84,
1631 0x1ebc0, 0x1ebc8,
1632 0x1ec40, 0x1ec4c,
1633 0x1ee84, 0x1ee8c,
1634 0x1eec0, 0x1eec0,
1635 0x1eee0, 0x1eee0,
1636 0x1ef00, 0x1ef84,
1637 0x1efc0, 0x1efc8,
1638 0x1f040, 0x1f04c,
1639 0x1f284, 0x1f28c,
1640 0x1f2c0, 0x1f2c0,
1641 0x1f2e0, 0x1f2e0,
1642 0x1f300, 0x1f384,
1643 0x1f3c0, 0x1f3c8,
1644 0x1f440, 0x1f44c,
1645 0x1f684, 0x1f68c,
1646 0x1f6c0, 0x1f6c0,
1647 0x1f6e0, 0x1f6e0,
1648 0x1f700, 0x1f784,
1649 0x1f7c0, 0x1f7c8,
1650 0x1f840, 0x1f84c,
1651 0x1fa84, 0x1fa8c,
1652 0x1fac0, 0x1fac0,
1653 0x1fae0, 0x1fae0,
1654 0x1fb00, 0x1fb84,
1655 0x1fbc0, 0x1fbc8,
1656 0x1fc40, 0x1fc4c,
1657 0x1fe84, 0x1fe8c,
1658 0x1fec0, 0x1fec0,
1659 0x1fee0, 0x1fee0,
1660 0x1ff00, 0x1ff84,
1661 0x1ffc0, 0x1ffc8,
1662 0x20000, 0x2002c,
1663 0x20100, 0x2013c,
1664 0x20190, 0x201c8,
1665 0x20200, 0x20318,
1666 0x20400, 0x20528,
1667 0x20540, 0x20614,
1668 0x21000, 0x21040,
1669 0x2104c, 0x21060,
1670 0x210c0, 0x210ec,
1671 0x21200, 0x21268,
1672 0x21270, 0x21284,
1673 0x212fc, 0x21388,
1674 0x21400, 0x21404,
1675 0x21500, 0x21518,
1676 0x2152c, 0x2153c,
1677 0x21550, 0x21554,
1678 0x21600, 0x21600,
1679 0x21608, 0x21628,
1680 0x21630, 0x2163c,
1681 0x21700, 0x2171c,
1682 0x21780, 0x2178c,
1683 0x21800, 0x21c38,
1684 0x21c80, 0x21d7c,
1685 0x21e00, 0x21e04,
1686 0x22000, 0x2202c,
1687 0x22100, 0x2213c,
1688 0x22190, 0x221c8,
1689 0x22200, 0x22318,
1690 0x22400, 0x22528,
1691 0x22540, 0x22614,
1692 0x23000, 0x23040,
1693 0x2304c, 0x23060,
1694 0x230c0, 0x230ec,
1695 0x23200, 0x23268,
1696 0x23270, 0x23284,
1697 0x232fc, 0x23388,
1698 0x23400, 0x23404,
1699 0x23500, 0x23518,
1700 0x2352c, 0x2353c,
1701 0x23550, 0x23554,
1702 0x23600, 0x23600,
1703 0x23608, 0x23628,
1704 0x23630, 0x2363c,
1705 0x23700, 0x2371c,
1706 0x23780, 0x2378c,
1707 0x23800, 0x23c38,
1708 0x23c80, 0x23d7c,
1709 0x23e00, 0x23e04,
1710 0x24000, 0x2402c,
1711 0x24100, 0x2413c,
1712 0x24190, 0x241c8,
1713 0x24200, 0x24318,
1714 0x24400, 0x24528,
1715 0x24540, 0x24614,
1716 0x25000, 0x25040,
1717 0x2504c, 0x25060,
1718 0x250c0, 0x250ec,
1719 0x25200, 0x25268,
1720 0x25270, 0x25284,
1721 0x252fc, 0x25388,
1722 0x25400, 0x25404,
1723 0x25500, 0x25518,
1724 0x2552c, 0x2553c,
1725 0x25550, 0x25554,
1726 0x25600, 0x25600,
1727 0x25608, 0x25628,
1728 0x25630, 0x2563c,
1729 0x25700, 0x2571c,
1730 0x25780, 0x2578c,
1731 0x25800, 0x25c38,
1732 0x25c80, 0x25d7c,
1733 0x25e00, 0x25e04,
1734 0x26000, 0x2602c,
1735 0x26100, 0x2613c,
1736 0x26190, 0x261c8,
1737 0x26200, 0x26318,
1738 0x26400, 0x26528,
1739 0x26540, 0x26614,
1740 0x27000, 0x27040,
1741 0x2704c, 0x27060,
1742 0x270c0, 0x270ec,
1743 0x27200, 0x27268,
1744 0x27270, 0x27284,
1745 0x272fc, 0x27388,
1746 0x27400, 0x27404,
1747 0x27500, 0x27518,
1748 0x2752c, 0x2753c,
1749 0x27550, 0x27554,
1750 0x27600, 0x27600,
1751 0x27608, 0x27628,
1752 0x27630, 0x2763c,
1753 0x27700, 0x2771c,
1754 0x27780, 0x2778c,
1755 0x27800, 0x27c38,
1756 0x27c80, 0x27d7c,
1757 0x27e00, 0x27e04
1758 };
1759
1760 static const unsigned int t5_reg_ranges[] = {
1761 0x1008, 0x1148,
1762 0x1180, 0x11b4,
1763 0x11fc, 0x123c,
1764 0x1280, 0x173c,
1765 0x1800, 0x18fc,
1766 0x3000, 0x3028,
1767 0x3060, 0x30d8,
1768 0x30e0, 0x30fc,
1769 0x3140, 0x357c,
1770 0x35a8, 0x35cc,
1771 0x35ec, 0x35ec,
1772 0x3600, 0x5624,
1773 0x56cc, 0x575c,
1774 0x580c, 0x5814,
1775 0x5890, 0x58bc,
1776 0x5940, 0x59dc,
1777 0x59fc, 0x5a18,
1778 0x5a60, 0x5a9c,
1779 0x5b9c, 0x5bfc,
1780 0x6000, 0x6040,
1781 0x6058, 0x614c,
1782 0x7700, 0x7798,
1783 0x77c0, 0x78fc,
1784 0x7b00, 0x7c54,
1785 0x7d00, 0x7efc,
1786 0x8dc0, 0x8de0,
1787 0x8df8, 0x8e84,
1788 0x8ea0, 0x8f84,
1789 0x8fc0, 0x90f8,
1790 0x9400, 0x9470,
1791 0x9600, 0x96f4,
1792 0x9800, 0x9808,
1793 0x9820, 0x983c,
1794 0x9850, 0x9864,
1795 0x9c00, 0x9c6c,
1796 0x9c80, 0x9cec,
1797 0x9d00, 0x9d6c,
1798 0x9d80, 0x9dec,
1799 0x9e00, 0x9e6c,
1800 0x9e80, 0x9eec,
1801 0x9f00, 0x9f6c,
1802 0x9f80, 0xa020,
1803 0xd004, 0xd03c,
1804 0xdfc0, 0xdfe0,
1805 0xe000, 0x11088,
1806 0x1109c, 0x11110,
1807 0x11118, 0x1117c,
1808 0x11190, 0x11204,
1809 0x19040, 0x1906c,
1810 0x19078, 0x19080,
1811 0x1908c, 0x19124,
1812 0x19150, 0x191b0,
1813 0x191d0, 0x191e8,
1814 0x19238, 0x19290,
1815 0x193f8, 0x19474,
1816 0x19490, 0x194cc,
1817 0x194f0, 0x194f8,
1818 0x19c00, 0x19c60,
1819 0x19c94, 0x19e10,
1820 0x19e50, 0x19f34,
1821 0x19f40, 0x19f50,
1822 0x19f90, 0x19fe4,
1823 0x1a000, 0x1a06c,
1824 0x1a0b0, 0x1a120,
1825 0x1a128, 0x1a138,
1826 0x1a190, 0x1a1c4,
1827 0x1a1fc, 0x1a1fc,
1828 0x1e008, 0x1e00c,
1829 0x1e040, 0x1e04c,
1830 0x1e284, 0x1e290,
1831 0x1e2c0, 0x1e2c0,
1832 0x1e2e0, 0x1e2e0,
1833 0x1e300, 0x1e384,
1834 0x1e3c0, 0x1e3c8,
1835 0x1e408, 0x1e40c,
1836 0x1e440, 0x1e44c,
1837 0x1e684, 0x1e690,
1838 0x1e6c0, 0x1e6c0,
1839 0x1e6e0, 0x1e6e0,
1840 0x1e700, 0x1e784,
1841 0x1e7c0, 0x1e7c8,
1842 0x1e808, 0x1e80c,
1843 0x1e840, 0x1e84c,
1844 0x1ea84, 0x1ea90,
1845 0x1eac0, 0x1eac0,
1846 0x1eae0, 0x1eae0,
1847 0x1eb00, 0x1eb84,
1848 0x1ebc0, 0x1ebc8,
1849 0x1ec08, 0x1ec0c,
1850 0x1ec40, 0x1ec4c,
1851 0x1ee84, 0x1ee90,
1852 0x1eec0, 0x1eec0,
1853 0x1eee0, 0x1eee0,
1854 0x1ef00, 0x1ef84,
1855 0x1efc0, 0x1efc8,
1856 0x1f008, 0x1f00c,
1857 0x1f040, 0x1f04c,
1858 0x1f284, 0x1f290,
1859 0x1f2c0, 0x1f2c0,
1860 0x1f2e0, 0x1f2e0,
1861 0x1f300, 0x1f384,
1862 0x1f3c0, 0x1f3c8,
1863 0x1f408, 0x1f40c,
1864 0x1f440, 0x1f44c,
1865 0x1f684, 0x1f690,
1866 0x1f6c0, 0x1f6c0,
1867 0x1f6e0, 0x1f6e0,
1868 0x1f700, 0x1f784,
1869 0x1f7c0, 0x1f7c8,
1870 0x1f808, 0x1f80c,
1871 0x1f840, 0x1f84c,
1872 0x1fa84, 0x1fa90,
1873 0x1fac0, 0x1fac0,
1874 0x1fae0, 0x1fae0,
1875 0x1fb00, 0x1fb84,
1876 0x1fbc0, 0x1fbc8,
1877 0x1fc08, 0x1fc0c,
1878 0x1fc40, 0x1fc4c,
1879 0x1fe84, 0x1fe90,
1880 0x1fec0, 0x1fec0,
1881 0x1fee0, 0x1fee0,
1882 0x1ff00, 0x1ff84,
1883 0x1ffc0, 0x1ffc8,
1884 0x30000, 0x30030,
1885 0x30100, 0x30144,
1886 0x30190, 0x301d0,
1887 0x30200, 0x30318,
1888 0x30400, 0x3052c,
1889 0x30540, 0x3061c,
1890 0x30800, 0x30834,
1891 0x308c0, 0x30908,
1892 0x30910, 0x309ac,
1893 0x30a00, 0x30a04,
1894 0x30a0c, 0x30a2c,
1895 0x30a44, 0x30a50,
1896 0x30a74, 0x30c24,
1897 0x30d08, 0x30d14,
1898 0x30d1c, 0x30d20,
1899 0x30d3c, 0x30d50,
1900 0x31200, 0x3120c,
1901 0x31220, 0x31220,
1902 0x31240, 0x31240,
1903 0x31600, 0x31600,
1904 0x31608, 0x3160c,
1905 0x31a00, 0x31a1c,
1906 0x31e04, 0x31e20,
1907 0x31e38, 0x31e3c,
1908 0x31e80, 0x31e80,
1909 0x31e88, 0x31ea8,
1910 0x31eb0, 0x31eb4,
1911 0x31ec8, 0x31ed4,
1912 0x31fb8, 0x32004,
1913 0x32208, 0x3223c,
1914 0x32600, 0x32630,
1915 0x32a00, 0x32abc,
1916 0x32b00, 0x32b70,
1917 0x33000, 0x33048,
1918 0x33060, 0x3309c,
1919 0x330f0, 0x33148,
1920 0x33160, 0x3319c,
1921 0x331f0, 0x332e4,
1922 0x332f8, 0x333e4,
1923 0x333f8, 0x33448,
1924 0x33460, 0x3349c,
1925 0x334f0, 0x33548,
1926 0x33560, 0x3359c,
1927 0x335f0, 0x336e4,
1928 0x336f8, 0x337e4,
1929 0x337f8, 0x337fc,
1930 0x33814, 0x33814,
1931 0x3382c, 0x3382c,
1932 0x33880, 0x3388c,
1933 0x338e8, 0x338ec,
1934 0x33900, 0x33948,
1935 0x33960, 0x3399c,
1936 0x339f0, 0x33ae4,
1937 0x33af8, 0x33b10,
1938 0x33b28, 0x33b28,
1939 0x33b3c, 0x33b50,
1940 0x33bf0, 0x33c10,
1941 0x33c28, 0x33c28,
1942 0x33c3c, 0x33c50,
1943 0x33cf0, 0x33cfc,
1944 0x34000, 0x34030,
1945 0x34100, 0x34144,
1946 0x34190, 0x341d0,
1947 0x34200, 0x34318,
1948 0x34400, 0x3452c,
1949 0x34540, 0x3461c,
1950 0x34800, 0x34834,
1951 0x348c0, 0x34908,
1952 0x34910, 0x349ac,
1953 0x34a00, 0x34a04,
1954 0x34a0c, 0x34a2c,
1955 0x34a44, 0x34a50,
1956 0x34a74, 0x34c24,
1957 0x34d08, 0x34d14,
1958 0x34d1c, 0x34d20,
1959 0x34d3c, 0x34d50,
1960 0x35200, 0x3520c,
1961 0x35220, 0x35220,
1962 0x35240, 0x35240,
1963 0x35600, 0x35600,
1964 0x35608, 0x3560c,
1965 0x35a00, 0x35a1c,
1966 0x35e04, 0x35e20,
1967 0x35e38, 0x35e3c,
1968 0x35e80, 0x35e80,
1969 0x35e88, 0x35ea8,
1970 0x35eb0, 0x35eb4,
1971 0x35ec8, 0x35ed4,
1972 0x35fb8, 0x36004,
1973 0x36208, 0x3623c,
1974 0x36600, 0x36630,
1975 0x36a00, 0x36abc,
1976 0x36b00, 0x36b70,
1977 0x37000, 0x37048,
1978 0x37060, 0x3709c,
1979 0x370f0, 0x37148,
1980 0x37160, 0x3719c,
1981 0x371f0, 0x372e4,
1982 0x372f8, 0x373e4,
1983 0x373f8, 0x37448,
1984 0x37460, 0x3749c,
1985 0x374f0, 0x37548,
1986 0x37560, 0x3759c,
1987 0x375f0, 0x376e4,
1988 0x376f8, 0x377e4,
1989 0x377f8, 0x377fc,
1990 0x37814, 0x37814,
1991 0x3782c, 0x3782c,
1992 0x37880, 0x3788c,
1993 0x378e8, 0x378ec,
1994 0x37900, 0x37948,
1995 0x37960, 0x3799c,
1996 0x379f0, 0x37ae4,
1997 0x37af8, 0x37b10,
1998 0x37b28, 0x37b28,
1999 0x37b3c, 0x37b50,
2000 0x37bf0, 0x37c10,
2001 0x37c28, 0x37c28,
2002 0x37c3c, 0x37c50,
2003 0x37cf0, 0x37cfc,
2004 0x38000, 0x38030,
2005 0x38100, 0x38144,
2006 0x38190, 0x381d0,
2007 0x38200, 0x38318,
2008 0x38400, 0x3852c,
2009 0x38540, 0x3861c,
2010 0x38800, 0x38834,
2011 0x388c0, 0x38908,
2012 0x38910, 0x389ac,
2013 0x38a00, 0x38a04,
2014 0x38a0c, 0x38a2c,
2015 0x38a44, 0x38a50,
2016 0x38a74, 0x38c24,
2017 0x38d08, 0x38d14,
2018 0x38d1c, 0x38d20,
2019 0x38d3c, 0x38d50,
2020 0x39200, 0x3920c,
2021 0x39220, 0x39220,
2022 0x39240, 0x39240,
2023 0x39600, 0x39600,
2024 0x39608, 0x3960c,
2025 0x39a00, 0x39a1c,
2026 0x39e04, 0x39e20,
2027 0x39e38, 0x39e3c,
2028 0x39e80, 0x39e80,
2029 0x39e88, 0x39ea8,
2030 0x39eb0, 0x39eb4,
2031 0x39ec8, 0x39ed4,
2032 0x39fb8, 0x3a004,
2033 0x3a208, 0x3a23c,
2034 0x3a600, 0x3a630,
2035 0x3aa00, 0x3aabc,
2036 0x3ab00, 0x3ab70,
2037 0x3b000, 0x3b048,
2038 0x3b060, 0x3b09c,
2039 0x3b0f0, 0x3b148,
2040 0x3b160, 0x3b19c,
2041 0x3b1f0, 0x3b2e4,
2042 0x3b2f8, 0x3b3e4,
2043 0x3b3f8, 0x3b448,
2044 0x3b460, 0x3b49c,
2045 0x3b4f0, 0x3b548,
2046 0x3b560, 0x3b59c,
2047 0x3b5f0, 0x3b6e4,
2048 0x3b6f8, 0x3b7e4,
2049 0x3b7f8, 0x3b7fc,
2050 0x3b814, 0x3b814,
2051 0x3b82c, 0x3b82c,
2052 0x3b880, 0x3b88c,
2053 0x3b8e8, 0x3b8ec,
2054 0x3b900, 0x3b948,
2055 0x3b960, 0x3b99c,
2056 0x3b9f0, 0x3bae4,
2057 0x3baf8, 0x3bb10,
2058 0x3bb28, 0x3bb28,
2059 0x3bb3c, 0x3bb50,
2060 0x3bbf0, 0x3bc10,
2061 0x3bc28, 0x3bc28,
2062 0x3bc3c, 0x3bc50,
2063 0x3bcf0, 0x3bcfc,
2064 0x3c000, 0x3c030,
2065 0x3c100, 0x3c144,
2066 0x3c190, 0x3c1d0,
2067 0x3c200, 0x3c318,
2068 0x3c400, 0x3c52c,
2069 0x3c540, 0x3c61c,
2070 0x3c800, 0x3c834,
2071 0x3c8c0, 0x3c908,
2072 0x3c910, 0x3c9ac,
2073 0x3ca00, 0x3ca04,
2074 0x3ca0c, 0x3ca2c,
2075 0x3ca44, 0x3ca50,
2076 0x3ca74, 0x3cc24,
2077 0x3cd08, 0x3cd14,
2078 0x3cd1c, 0x3cd20,
2079 0x3cd3c, 0x3cd50,
2080 0x3d200, 0x3d20c,
2081 0x3d220, 0x3d220,
2082 0x3d240, 0x3d240,
2083 0x3d600, 0x3d600,
2084 0x3d608, 0x3d60c,
2085 0x3da00, 0x3da1c,
2086 0x3de04, 0x3de20,
2087 0x3de38, 0x3de3c,
2088 0x3de80, 0x3de80,
2089 0x3de88, 0x3dea8,
2090 0x3deb0, 0x3deb4,
2091 0x3dec8, 0x3ded4,
2092 0x3dfb8, 0x3e004,
2093 0x3e208, 0x3e23c,
2094 0x3e600, 0x3e630,
2095 0x3ea00, 0x3eabc,
2096 0x3eb00, 0x3eb70,
2097 0x3f000, 0x3f048,
2098 0x3f060, 0x3f09c,
2099 0x3f0f0, 0x3f148,
2100 0x3f160, 0x3f19c,
2101 0x3f1f0, 0x3f2e4,
2102 0x3f2f8, 0x3f3e4,
2103 0x3f3f8, 0x3f448,
2104 0x3f460, 0x3f49c,
2105 0x3f4f0, 0x3f548,
2106 0x3f560, 0x3f59c,
2107 0x3f5f0, 0x3f6e4,
2108 0x3f6f8, 0x3f7e4,
2109 0x3f7f8, 0x3f7fc,
2110 0x3f814, 0x3f814,
2111 0x3f82c, 0x3f82c,
2112 0x3f880, 0x3f88c,
2113 0x3f8e8, 0x3f8ec,
2114 0x3f900, 0x3f948,
2115 0x3f960, 0x3f99c,
2116 0x3f9f0, 0x3fae4,
2117 0x3faf8, 0x3fb10,
2118 0x3fb28, 0x3fb28,
2119 0x3fb3c, 0x3fb50,
2120 0x3fbf0, 0x3fc10,
2121 0x3fc28, 0x3fc28,
2122 0x3fc3c, 0x3fc50,
2123 0x3fcf0, 0x3fcfc,
2124 0x40000, 0x4000c,
2125 0x40040, 0x40068,
2126 0x40080, 0x40144,
2127 0x40180, 0x4018c,
2128 0x40200, 0x40298,
2129 0x402ac, 0x4033c,
2130 0x403f8, 0x403fc,
2131 0x41304, 0x413c4,
2132 0x41400, 0x4141c,
2133 0x41480, 0x414d0,
2134 0x44000, 0x44078,
2135 0x440c0, 0x44278,
2136 0x442c0, 0x44478,
2137 0x444c0, 0x44678,
2138 0x446c0, 0x44878,
2139 0x448c0, 0x449fc,
2140 0x45000, 0x45068,
2141 0x45080, 0x45084,
2142 0x450a0, 0x450b0,
2143 0x45200, 0x45268,
2144 0x45280, 0x45284,
2145 0x452a0, 0x452b0,
2146 0x460c0, 0x460e4,
2147 0x47000, 0x4708c,
2148 0x47200, 0x47250,
2149 0x47400, 0x47420,
2150 0x47600, 0x47618,
2151 0x47800, 0x47814,
2152 0x48000, 0x4800c,
2153 0x48040, 0x48068,
2154 0x48080, 0x48144,
2155 0x48180, 0x4818c,
2156 0x48200, 0x48298,
2157 0x482ac, 0x4833c,
2158 0x483f8, 0x483fc,
2159 0x49304, 0x493c4,
2160 0x49400, 0x4941c,
2161 0x49480, 0x494d0,
2162 0x4c000, 0x4c078,
2163 0x4c0c0, 0x4c278,
2164 0x4c2c0, 0x4c478,
2165 0x4c4c0, 0x4c678,
2166 0x4c6c0, 0x4c878,
2167 0x4c8c0, 0x4c9fc,
2168 0x4d000, 0x4d068,
2169 0x4d080, 0x4d084,
2170 0x4d0a0, 0x4d0b0,
2171 0x4d200, 0x4d268,
2172 0x4d280, 0x4d284,
2173 0x4d2a0, 0x4d2b0,
2174 0x4e0c0, 0x4e0e4,
2175 0x4f000, 0x4f08c,
2176 0x4f200, 0x4f250,
2177 0x4f400, 0x4f420,
2178 0x4f600, 0x4f618,
2179 0x4f800, 0x4f814,
2180 0x50000, 0x500cc,
2181 0x50400, 0x50400,
2182 0x50800, 0x508cc,
2183 0x50c00, 0x50c00,
2184 0x51000, 0x5101c,
2185 0x51300, 0x51308,
2186 };
2187
2188 int i;
2189 struct adapter *ap = netdev2adap(dev);
2190 static const unsigned int *reg_ranges;
2191 int arr_size = 0, buf_size = 0;
2192
2193 if (is_t4(ap->params.chip)) {
2194 reg_ranges = &t4_reg_ranges[0];
2195 arr_size = ARRAY_SIZE(t4_reg_ranges);
2196 buf_size = T4_REGMAP_SIZE;
2197 } else {
2198 reg_ranges = &t5_reg_ranges[0];
2199 arr_size = ARRAY_SIZE(t5_reg_ranges);
2200 buf_size = T5_REGMAP_SIZE;
2201 }
2202
2203 regs->version = mk_adap_vers(ap);
2204
2205 memset(buf, 0, buf_size);
2206 for (i = 0; i < arr_size; i += 2)
2207 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2208 }
2209
2210 static int restart_autoneg(struct net_device *dev)
2211 {
2212 struct port_info *p = netdev_priv(dev);
2213
2214 if (!netif_running(dev))
2215 return -EAGAIN;
2216 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2217 return -EINVAL;
2218 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2219 return 0;
2220 }
2221
2222 static int identify_port(struct net_device *dev,
2223 enum ethtool_phys_id_state state)
2224 {
2225 unsigned int val;
2226 struct adapter *adap = netdev2adap(dev);
2227
2228 if (state == ETHTOOL_ID_ACTIVE)
2229 val = 0xffff;
2230 else if (state == ETHTOOL_ID_INACTIVE)
2231 val = 0;
2232 else
2233 return -EINVAL;
2234
2235 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2236 }
2237
2238 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
2239 {
2240 unsigned int v = 0;
2241
2242 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2243 type == FW_PORT_TYPE_BT_XAUI) {
2244 v |= SUPPORTED_TP;
2245 if (caps & FW_PORT_CAP_SPEED_100M)
2246 v |= SUPPORTED_100baseT_Full;
2247 if (caps & FW_PORT_CAP_SPEED_1G)
2248 v |= SUPPORTED_1000baseT_Full;
2249 if (caps & FW_PORT_CAP_SPEED_10G)
2250 v |= SUPPORTED_10000baseT_Full;
2251 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2252 v |= SUPPORTED_Backplane;
2253 if (caps & FW_PORT_CAP_SPEED_1G)
2254 v |= SUPPORTED_1000baseKX_Full;
2255 if (caps & FW_PORT_CAP_SPEED_10G)
2256 v |= SUPPORTED_10000baseKX4_Full;
2257 } else if (type == FW_PORT_TYPE_KR)
2258 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2259 else if (type == FW_PORT_TYPE_BP_AP)
2260 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2261 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2262 else if (type == FW_PORT_TYPE_BP4_AP)
2263 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2264 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2265 SUPPORTED_10000baseKX4_Full;
2266 else if (type == FW_PORT_TYPE_FIBER_XFI ||
2267 type == FW_PORT_TYPE_FIBER_XAUI ||
2268 type == FW_PORT_TYPE_SFP ||
2269 type == FW_PORT_TYPE_QSFP_10G ||
2270 type == FW_PORT_TYPE_QSA) {
2271 v |= SUPPORTED_FIBRE;
2272 if (caps & FW_PORT_CAP_SPEED_1G)
2273 v |= SUPPORTED_1000baseT_Full;
2274 if (caps & FW_PORT_CAP_SPEED_10G)
2275 v |= SUPPORTED_10000baseT_Full;
2276 } else if (type == FW_PORT_TYPE_BP40_BA ||
2277 type == FW_PORT_TYPE_QSFP) {
2278 v |= SUPPORTED_40000baseSR4_Full;
2279 v |= SUPPORTED_FIBRE;
2280 }
2281
2282 if (caps & FW_PORT_CAP_ANEG)
2283 v |= SUPPORTED_Autoneg;
2284 return v;
2285 }
2286
2287 static unsigned int to_fw_linkcaps(unsigned int caps)
2288 {
2289 unsigned int v = 0;
2290
2291 if (caps & ADVERTISED_100baseT_Full)
2292 v |= FW_PORT_CAP_SPEED_100M;
2293 if (caps & ADVERTISED_1000baseT_Full)
2294 v |= FW_PORT_CAP_SPEED_1G;
2295 if (caps & ADVERTISED_10000baseT_Full)
2296 v |= FW_PORT_CAP_SPEED_10G;
2297 if (caps & ADVERTISED_40000baseSR4_Full)
2298 v |= FW_PORT_CAP_SPEED_40G;
2299 return v;
2300 }
2301
2302 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2303 {
2304 const struct port_info *p = netdev_priv(dev);
2305
2306 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2307 p->port_type == FW_PORT_TYPE_BT_XFI ||
2308 p->port_type == FW_PORT_TYPE_BT_XAUI)
2309 cmd->port = PORT_TP;
2310 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2311 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2312 cmd->port = PORT_FIBRE;
2313 else if (p->port_type == FW_PORT_TYPE_SFP ||
2314 p->port_type == FW_PORT_TYPE_QSFP_10G ||
2315 p->port_type == FW_PORT_TYPE_QSA ||
2316 p->port_type == FW_PORT_TYPE_QSFP) {
2317 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2318 p->mod_type == FW_PORT_MOD_TYPE_SR ||
2319 p->mod_type == FW_PORT_MOD_TYPE_ER ||
2320 p->mod_type == FW_PORT_MOD_TYPE_LRM)
2321 cmd->port = PORT_FIBRE;
2322 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2323 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2324 cmd->port = PORT_DA;
2325 else
2326 cmd->port = PORT_OTHER;
2327 } else
2328 cmd->port = PORT_OTHER;
2329
2330 if (p->mdio_addr >= 0) {
2331 cmd->phy_address = p->mdio_addr;
2332 cmd->transceiver = XCVR_EXTERNAL;
2333 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2334 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2335 } else {
2336 cmd->phy_address = 0; /* not really, but no better option */
2337 cmd->transceiver = XCVR_INTERNAL;
2338 cmd->mdio_support = 0;
2339 }
2340
2341 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2342 cmd->advertising = from_fw_linkcaps(p->port_type,
2343 p->link_cfg.advertising);
2344 ethtool_cmd_speed_set(cmd,
2345 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2346 cmd->duplex = DUPLEX_FULL;
2347 cmd->autoneg = p->link_cfg.autoneg;
2348 cmd->maxtxpkt = 0;
2349 cmd->maxrxpkt = 0;
2350 return 0;
2351 }
2352
2353 static unsigned int speed_to_caps(int speed)
2354 {
2355 if (speed == 100)
2356 return FW_PORT_CAP_SPEED_100M;
2357 if (speed == 1000)
2358 return FW_PORT_CAP_SPEED_1G;
2359 if (speed == 10000)
2360 return FW_PORT_CAP_SPEED_10G;
2361 if (speed == 40000)
2362 return FW_PORT_CAP_SPEED_40G;
2363 return 0;
2364 }
2365
2366 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2367 {
2368 unsigned int cap;
2369 struct port_info *p = netdev_priv(dev);
2370 struct link_config *lc = &p->link_cfg;
2371 u32 speed = ethtool_cmd_speed(cmd);
2372
2373 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2374 return -EINVAL;
2375
2376 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2377 /*
2378 * PHY offers a single speed. See if that's what's
2379 * being requested.
2380 */
2381 if (cmd->autoneg == AUTONEG_DISABLE &&
2382 (lc->supported & speed_to_caps(speed)))
2383 return 0;
2384 return -EINVAL;
2385 }
2386
2387 if (cmd->autoneg == AUTONEG_DISABLE) {
2388 cap = speed_to_caps(speed);
2389
2390 if (!(lc->supported & cap) ||
2391 (speed == 1000) ||
2392 (speed == 10000) ||
2393 (speed == 40000))
2394 return -EINVAL;
2395 lc->requested_speed = cap;
2396 lc->advertising = 0;
2397 } else {
2398 cap = to_fw_linkcaps(cmd->advertising);
2399 if (!(lc->supported & cap))
2400 return -EINVAL;
2401 lc->requested_speed = 0;
2402 lc->advertising = cap | FW_PORT_CAP_ANEG;
2403 }
2404 lc->autoneg = cmd->autoneg;
2405
2406 if (netif_running(dev))
2407 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2408 lc);
2409 return 0;
2410 }
2411
2412 static void get_pauseparam(struct net_device *dev,
2413 struct ethtool_pauseparam *epause)
2414 {
2415 struct port_info *p = netdev_priv(dev);
2416
2417 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2418 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2419 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2420 }
2421
2422 static int set_pauseparam(struct net_device *dev,
2423 struct ethtool_pauseparam *epause)
2424 {
2425 struct port_info *p = netdev_priv(dev);
2426 struct link_config *lc = &p->link_cfg;
2427
2428 if (epause->autoneg == AUTONEG_DISABLE)
2429 lc->requested_fc = 0;
2430 else if (lc->supported & FW_PORT_CAP_ANEG)
2431 lc->requested_fc = PAUSE_AUTONEG;
2432 else
2433 return -EINVAL;
2434
2435 if (epause->rx_pause)
2436 lc->requested_fc |= PAUSE_RX;
2437 if (epause->tx_pause)
2438 lc->requested_fc |= PAUSE_TX;
2439 if (netif_running(dev))
2440 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2441 lc);
2442 return 0;
2443 }
2444
2445 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2446 {
2447 const struct port_info *pi = netdev_priv(dev);
2448 const struct sge *s = &pi->adapter->sge;
2449
2450 e->rx_max_pending = MAX_RX_BUFFERS;
2451 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2452 e->rx_jumbo_max_pending = 0;
2453 e->tx_max_pending = MAX_TXQ_ENTRIES;
2454
2455 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2456 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2457 e->rx_jumbo_pending = 0;
2458 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2459 }
2460
2461 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2462 {
2463 int i;
2464 const struct port_info *pi = netdev_priv(dev);
2465 struct adapter *adapter = pi->adapter;
2466 struct sge *s = &adapter->sge;
2467
2468 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2469 e->tx_pending > MAX_TXQ_ENTRIES ||
2470 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2471 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2472 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2473 return -EINVAL;
2474
2475 if (adapter->flags & FULL_INIT_DONE)
2476 return -EBUSY;
2477
2478 for (i = 0; i < pi->nqsets; ++i) {
2479 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2480 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2481 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2482 }
2483 return 0;
2484 }
2485
2486 static int closest_timer(const struct sge *s, int time)
2487 {
2488 int i, delta, match = 0, min_delta = INT_MAX;
2489
2490 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2491 delta = time - s->timer_val[i];
2492 if (delta < 0)
2493 delta = -delta;
2494 if (delta < min_delta) {
2495 min_delta = delta;
2496 match = i;
2497 }
2498 }
2499 return match;
2500 }
2501
2502 static int closest_thres(const struct sge *s, int thres)
2503 {
2504 int i, delta, match = 0, min_delta = INT_MAX;
2505
2506 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2507 delta = thres - s->counter_val[i];
2508 if (delta < 0)
2509 delta = -delta;
2510 if (delta < min_delta) {
2511 min_delta = delta;
2512 match = i;
2513 }
2514 }
2515 return match;
2516 }
2517
2518 /*
2519 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2520 */
2521 unsigned int qtimer_val(const struct adapter *adap,
2522 const struct sge_rspq *q)
2523 {
2524 unsigned int idx = q->intr_params >> 1;
2525
2526 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2527 }
2528
2529 /**
2530 * set_rspq_intr_params - set a queue's interrupt holdoff parameters
2531 * @q: the Rx queue
2532 * @us: the hold-off time in us, or 0 to disable timer
2533 * @cnt: the hold-off packet count, or 0 to disable counter
2534 *
2535 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2536 * one of the two needs to be enabled for the queue to generate interrupts.
2537 */
2538 static int set_rspq_intr_params(struct sge_rspq *q,
2539 unsigned int us, unsigned int cnt)
2540 {
2541 struct adapter *adap = q->adap;
2542
2543 if ((us | cnt) == 0)
2544 cnt = 1;
2545
2546 if (cnt) {
2547 int err;
2548 u32 v, new_idx;
2549
2550 new_idx = closest_thres(&adap->sge, cnt);
2551 if (q->desc && q->pktcnt_idx != new_idx) {
2552 /* the queue has already been created, update it */
2553 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2554 FW_PARAMS_PARAM_X_V(
2555 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2556 FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
2557 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2558 &new_idx);
2559 if (err)
2560 return err;
2561 }
2562 q->pktcnt_idx = new_idx;
2563 }
2564
2565 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2566 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2567 return 0;
2568 }
2569
2570 /**
2571 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2572 * @dev: the network device
2573 * @us: the hold-off time in us, or 0 to disable timer
2574 * @cnt: the hold-off packet count, or 0 to disable counter
2575 *
2576 * Set the RX interrupt hold-off parameters for a network device.
2577 */
2578 static int set_rx_intr_params(struct net_device *dev,
2579 unsigned int us, unsigned int cnt)
2580 {
2581 int i, err;
2582 struct port_info *pi = netdev_priv(dev);
2583 struct adapter *adap = pi->adapter;
2584 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2585
2586 for (i = 0; i < pi->nqsets; i++, q++) {
2587 err = set_rspq_intr_params(&q->rspq, us, cnt);
2588 if (err)
2589 return err;
2590 }
2591 return 0;
2592 }
2593
2594 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2595 {
2596 int i;
2597 struct port_info *pi = netdev_priv(dev);
2598 struct adapter *adap = pi->adapter;
2599 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2600
2601 for (i = 0; i < pi->nqsets; i++, q++)
2602 q->rspq.adaptive_rx = adaptive_rx;
2603
2604 return 0;
2605 }
2606
2607 static int get_adaptive_rx_setting(struct net_device *dev)
2608 {
2609 struct port_info *pi = netdev_priv(dev);
2610 struct adapter *adap = pi->adapter;
2611 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2612
2613 return q->rspq.adaptive_rx;
2614 }
2615
2616 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2617 {
2618 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
2619 return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2620 c->rx_max_coalesced_frames);
2621 }
2622
2623 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2624 {
2625 const struct port_info *pi = netdev_priv(dev);
2626 const struct adapter *adap = pi->adapter;
2627 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2628
2629 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2630 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2631 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2632 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
2633 return 0;
2634 }
2635
2636 /**
2637 * eeprom_ptov - translate a physical EEPROM address to virtual
2638 * @phys_addr: the physical EEPROM address
2639 * @fn: the PCI function number
2640 * @sz: size of function-specific area
2641 *
2642 * Translate a physical EEPROM address to virtual. The first 1K is
2643 * accessed through virtual addresses starting at 31K, the rest is
2644 * accessed through virtual addresses starting at 0.
2645 *
2646 * The mapping is as follows:
2647 * [0..1K) -> [31K..32K)
2648 * [1K..1K+A) -> [31K-A..31K)
2649 * [1K+A..ES) -> [0..ES-A-1K)
2650 *
2651 * where A = @fn * @sz, and ES = EEPROM size.
2652 */
2653 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2654 {
2655 fn *= sz;
2656 if (phys_addr < 1024)
2657 return phys_addr + (31 << 10);
2658 if (phys_addr < 1024 + fn)
2659 return 31744 - fn + phys_addr - 1024;
2660 if (phys_addr < EEPROMSIZE)
2661 return phys_addr - 1024 - fn;
2662 return -EINVAL;
2663 }
2664
2665 /*
2666 * The next two routines implement eeprom read/write from physical addresses.
2667 */
2668 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2669 {
2670 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2671
2672 if (vaddr >= 0)
2673 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2674 return vaddr < 0 ? vaddr : 0;
2675 }
2676
2677 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2678 {
2679 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2680
2681 if (vaddr >= 0)
2682 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2683 return vaddr < 0 ? vaddr : 0;
2684 }
2685
2686 #define EEPROM_MAGIC 0x38E2F10C
2687
2688 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2689 u8 *data)
2690 {
2691 int i, err = 0;
2692 struct adapter *adapter = netdev2adap(dev);
2693
2694 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2695 if (!buf)
2696 return -ENOMEM;
2697
2698 e->magic = EEPROM_MAGIC;
2699 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2700 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2701
2702 if (!err)
2703 memcpy(data, buf + e->offset, e->len);
2704 kfree(buf);
2705 return err;
2706 }
2707
2708 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2709 u8 *data)
2710 {
2711 u8 *buf;
2712 int err = 0;
2713 u32 aligned_offset, aligned_len, *p;
2714 struct adapter *adapter = netdev2adap(dev);
2715
2716 if (eeprom->magic != EEPROM_MAGIC)
2717 return -EINVAL;
2718
2719 aligned_offset = eeprom->offset & ~3;
2720 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2721
2722 if (adapter->fn > 0) {
2723 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2724
2725 if (aligned_offset < start ||
2726 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2727 return -EPERM;
2728 }
2729
2730 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2731 /*
2732 * RMW possibly needed for first or last words.
2733 */
2734 buf = kmalloc(aligned_len, GFP_KERNEL);
2735 if (!buf)
2736 return -ENOMEM;
2737 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2738 if (!err && aligned_len > 4)
2739 err = eeprom_rd_phys(adapter,
2740 aligned_offset + aligned_len - 4,
2741 (u32 *)&buf[aligned_len - 4]);
2742 if (err)
2743 goto out;
2744 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2745 } else
2746 buf = data;
2747
2748 err = t4_seeprom_wp(adapter, false);
2749 if (err)
2750 goto out;
2751
2752 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2753 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2754 aligned_offset += 4;
2755 }
2756
2757 if (!err)
2758 err = t4_seeprom_wp(adapter, true);
2759 out:
2760 if (buf != data)
2761 kfree(buf);
2762 return err;
2763 }
2764
2765 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2766 {
2767 int ret;
2768 const struct firmware *fw;
2769 struct adapter *adap = netdev2adap(netdev);
2770 unsigned int mbox = PCIE_FW_MASTER_M + 1;
2771
2772 ef->data[sizeof(ef->data) - 1] = '\0';
2773 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2774 if (ret < 0)
2775 return ret;
2776
2777 /* If the adapter has been fully initialized then we'll go ahead and
2778 * try to get the firmware's cooperation in upgrading to the new
2779 * firmware image otherwise we'll try to do the entire job from the
2780 * host ... and we always "force" the operation in this path.
2781 */
2782 if (adap->flags & FULL_INIT_DONE)
2783 mbox = adap->mbox;
2784
2785 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
2786 release_firmware(fw);
2787 if (!ret)
2788 dev_info(adap->pdev_dev, "loaded firmware %s,"
2789 " reload cxgb4 driver\n", ef->data);
2790 return ret;
2791 }
2792
2793 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2794 #define BCAST_CRC 0xa0ccc1a6
2795
2796 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2797 {
2798 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2799 wol->wolopts = netdev2adap(dev)->wol;
2800 memset(&wol->sopass, 0, sizeof(wol->sopass));
2801 }
2802
2803 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2804 {
2805 int err = 0;
2806 struct port_info *pi = netdev_priv(dev);
2807
2808 if (wol->wolopts & ~WOL_SUPPORTED)
2809 return -EINVAL;
2810 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2811 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2812 if (wol->wolopts & WAKE_BCAST) {
2813 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2814 ~0ULL, 0, false);
2815 if (!err)
2816 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2817 ~6ULL, ~0ULL, BCAST_CRC, true);
2818 } else
2819 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2820 return err;
2821 }
2822
2823 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2824 {
2825 const struct port_info *pi = netdev_priv(dev);
2826 netdev_features_t changed = dev->features ^ features;
2827 int err;
2828
2829 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2830 return 0;
2831
2832 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2833 -1, -1, -1,
2834 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2835 if (unlikely(err))
2836 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2837 return err;
2838 }
2839
2840 static u32 get_rss_table_size(struct net_device *dev)
2841 {
2842 const struct port_info *pi = netdev_priv(dev);
2843
2844 return pi->rss_size;
2845 }
2846
2847 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
2848 {
2849 const struct port_info *pi = netdev_priv(dev);
2850 unsigned int n = pi->rss_size;
2851
2852 if (hfunc)
2853 *hfunc = ETH_RSS_HASH_TOP;
2854 if (!p)
2855 return 0;
2856 while (n--)
2857 p[n] = pi->rss[n];
2858 return 0;
2859 }
2860
2861 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2862 const u8 hfunc)
2863 {
2864 unsigned int i;
2865 struct port_info *pi = netdev_priv(dev);
2866
2867 /* We require at least one supported parameter to be changed and no
2868 * change in any of the unsupported parameters
2869 */
2870 if (key ||
2871 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2872 return -EOPNOTSUPP;
2873 if (!p)
2874 return 0;
2875
2876 for (i = 0; i < pi->rss_size; i++)
2877 pi->rss[i] = p[i];
2878 if (pi->adapter->flags & FULL_INIT_DONE)
2879 return write_rss(pi, pi->rss);
2880 return 0;
2881 }
2882
2883 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2884 u32 *rules)
2885 {
2886 const struct port_info *pi = netdev_priv(dev);
2887
2888 switch (info->cmd) {
2889 case ETHTOOL_GRXFH: {
2890 unsigned int v = pi->rss_mode;
2891
2892 info->data = 0;
2893 switch (info->flow_type) {
2894 case TCP_V4_FLOW:
2895 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
2896 info->data = RXH_IP_SRC | RXH_IP_DST |
2897 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2898 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2899 info->data = RXH_IP_SRC | RXH_IP_DST;
2900 break;
2901 case UDP_V4_FLOW:
2902 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2903 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2904 info->data = RXH_IP_SRC | RXH_IP_DST |
2905 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2906 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2907 info->data = RXH_IP_SRC | RXH_IP_DST;
2908 break;
2909 case SCTP_V4_FLOW:
2910 case AH_ESP_V4_FLOW:
2911 case IPV4_FLOW:
2912 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2913 info->data = RXH_IP_SRC | RXH_IP_DST;
2914 break;
2915 case TCP_V6_FLOW:
2916 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
2917 info->data = RXH_IP_SRC | RXH_IP_DST |
2918 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2919 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2920 info->data = RXH_IP_SRC | RXH_IP_DST;
2921 break;
2922 case UDP_V6_FLOW:
2923 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2924 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2925 info->data = RXH_IP_SRC | RXH_IP_DST |
2926 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2927 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2928 info->data = RXH_IP_SRC | RXH_IP_DST;
2929 break;
2930 case SCTP_V6_FLOW:
2931 case AH_ESP_V6_FLOW:
2932 case IPV6_FLOW:
2933 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2934 info->data = RXH_IP_SRC | RXH_IP_DST;
2935 break;
2936 }
2937 return 0;
2938 }
2939 case ETHTOOL_GRXRINGS:
2940 info->data = pi->nqsets;
2941 return 0;
2942 }
2943 return -EOPNOTSUPP;
2944 }
2945
2946 static const struct ethtool_ops cxgb_ethtool_ops = {
2947 .get_settings = get_settings,
2948 .set_settings = set_settings,
2949 .get_drvinfo = get_drvinfo,
2950 .get_msglevel = get_msglevel,
2951 .set_msglevel = set_msglevel,
2952 .get_ringparam = get_sge_param,
2953 .set_ringparam = set_sge_param,
2954 .get_coalesce = get_coalesce,
2955 .set_coalesce = set_coalesce,
2956 .get_eeprom_len = get_eeprom_len,
2957 .get_eeprom = get_eeprom,
2958 .set_eeprom = set_eeprom,
2959 .get_pauseparam = get_pauseparam,
2960 .set_pauseparam = set_pauseparam,
2961 .get_link = ethtool_op_get_link,
2962 .get_strings = get_strings,
2963 .set_phys_id = identify_port,
2964 .nway_reset = restart_autoneg,
2965 .get_sset_count = get_sset_count,
2966 .get_ethtool_stats = get_stats,
2967 .get_regs_len = get_regs_len,
2968 .get_regs = get_regs,
2969 .get_wol = get_wol,
2970 .set_wol = set_wol,
2971 .get_rxnfc = get_rxnfc,
2972 .get_rxfh_indir_size = get_rss_table_size,
2973 .get_rxfh = get_rss_table,
2974 .set_rxfh = set_rss_table,
2975 .flash_device = set_flash,
2976 };
2977
2978 static int setup_debugfs(struct adapter *adap)
2979 {
2980 if (IS_ERR_OR_NULL(adap->debugfs_root))
2981 return -1;
2982
2983 #ifdef CONFIG_DEBUG_FS
2984 t4_setup_debugfs(adap);
2985 #endif
2986 return 0;
2987 }
2988
2989 /*
2990 * upper-layer driver support
2991 */
2992
2993 /*
2994 * Allocate an active-open TID and set it to the supplied value.
2995 */
2996 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2997 {
2998 int atid = -1;
2999
3000 spin_lock_bh(&t->atid_lock);
3001 if (t->afree) {
3002 union aopen_entry *p = t->afree;
3003
3004 atid = (p - t->atid_tab) + t->atid_base;
3005 t->afree = p->next;
3006 p->data = data;
3007 t->atids_in_use++;
3008 }
3009 spin_unlock_bh(&t->atid_lock);
3010 return atid;
3011 }
3012 EXPORT_SYMBOL(cxgb4_alloc_atid);
3013
3014 /*
3015 * Release an active-open TID.
3016 */
3017 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3018 {
3019 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
3020
3021 spin_lock_bh(&t->atid_lock);
3022 p->next = t->afree;
3023 t->afree = p;
3024 t->atids_in_use--;
3025 spin_unlock_bh(&t->atid_lock);
3026 }
3027 EXPORT_SYMBOL(cxgb4_free_atid);
3028
3029 /*
3030 * Allocate a server TID and set it to the supplied value.
3031 */
3032 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3033 {
3034 int stid;
3035
3036 spin_lock_bh(&t->stid_lock);
3037 if (family == PF_INET) {
3038 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3039 if (stid < t->nstids)
3040 __set_bit(stid, t->stid_bmap);
3041 else
3042 stid = -1;
3043 } else {
3044 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3045 if (stid < 0)
3046 stid = -1;
3047 }
3048 if (stid >= 0) {
3049 t->stid_tab[stid].data = data;
3050 stid += t->stid_base;
3051 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3052 * This is equivalent to 4 TIDs. With CLIP enabled it
3053 * needs 2 TIDs.
3054 */
3055 if (family == PF_INET)
3056 t->stids_in_use++;
3057 else
3058 t->stids_in_use += 4;
3059 }
3060 spin_unlock_bh(&t->stid_lock);
3061 return stid;
3062 }
3063 EXPORT_SYMBOL(cxgb4_alloc_stid);
3064
3065 /* Allocate a server filter TID and set it to the supplied value.
3066 */
3067 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3068 {
3069 int stid;
3070
3071 spin_lock_bh(&t->stid_lock);
3072 if (family == PF_INET) {
3073 stid = find_next_zero_bit(t->stid_bmap,
3074 t->nstids + t->nsftids, t->nstids);
3075 if (stid < (t->nstids + t->nsftids))
3076 __set_bit(stid, t->stid_bmap);
3077 else
3078 stid = -1;
3079 } else {
3080 stid = -1;
3081 }
3082 if (stid >= 0) {
3083 t->stid_tab[stid].data = data;
3084 stid -= t->nstids;
3085 stid += t->sftid_base;
3086 t->stids_in_use++;
3087 }
3088 spin_unlock_bh(&t->stid_lock);
3089 return stid;
3090 }
3091 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3092
3093 /* Release a server TID.
3094 */
3095 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3096 {
3097 /* Is it a server filter TID? */
3098 if (t->nsftids && (stid >= t->sftid_base)) {
3099 stid -= t->sftid_base;
3100 stid += t->nstids;
3101 } else {
3102 stid -= t->stid_base;
3103 }
3104
3105 spin_lock_bh(&t->stid_lock);
3106 if (family == PF_INET)
3107 __clear_bit(stid, t->stid_bmap);
3108 else
3109 bitmap_release_region(t->stid_bmap, stid, 2);
3110 t->stid_tab[stid].data = NULL;
3111 if (family == PF_INET)
3112 t->stids_in_use--;
3113 else
3114 t->stids_in_use -= 4;
3115 spin_unlock_bh(&t->stid_lock);
3116 }
3117 EXPORT_SYMBOL(cxgb4_free_stid);
3118
3119 /*
3120 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3121 */
3122 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3123 unsigned int tid)
3124 {
3125 struct cpl_tid_release *req;
3126
3127 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3128 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3129 INIT_TP_WR(req, tid);
3130 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3131 }
3132
3133 /*
3134 * Queue a TID release request and if necessary schedule a work queue to
3135 * process it.
3136 */
3137 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3138 unsigned int tid)
3139 {
3140 void **p = &t->tid_tab[tid];
3141 struct adapter *adap = container_of(t, struct adapter, tids);
3142
3143 spin_lock_bh(&adap->tid_release_lock);
3144 *p = adap->tid_release_head;
3145 /* Low 2 bits encode the Tx channel number */
3146 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3147 if (!adap->tid_release_task_busy) {
3148 adap->tid_release_task_busy = true;
3149 queue_work(adap->workq, &adap->tid_release_task);
3150 }
3151 spin_unlock_bh(&adap->tid_release_lock);
3152 }
3153
3154 /*
3155 * Process the list of pending TID release requests.
3156 */
3157 static void process_tid_release_list(struct work_struct *work)
3158 {
3159 struct sk_buff *skb;
3160 struct adapter *adap;
3161
3162 adap = container_of(work, struct adapter, tid_release_task);
3163
3164 spin_lock_bh(&adap->tid_release_lock);
3165 while (adap->tid_release_head) {
3166 void **p = adap->tid_release_head;
3167 unsigned int chan = (uintptr_t)p & 3;
3168 p = (void *)p - chan;
3169
3170 adap->tid_release_head = *p;
3171 *p = NULL;
3172 spin_unlock_bh(&adap->tid_release_lock);
3173
3174 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3175 GFP_KERNEL)))
3176 schedule_timeout_uninterruptible(1);
3177
3178 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3179 t4_ofld_send(adap, skb);
3180 spin_lock_bh(&adap->tid_release_lock);
3181 }
3182 adap->tid_release_task_busy = false;
3183 spin_unlock_bh(&adap->tid_release_lock);
3184 }
3185
3186 /*
3187 * Release a TID and inform HW. If we are unable to allocate the release
3188 * message we defer to a work queue.
3189 */
3190 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3191 {
3192 void *old;
3193 struct sk_buff *skb;
3194 struct adapter *adap = container_of(t, struct adapter, tids);
3195
3196 old = t->tid_tab[tid];
3197 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3198 if (likely(skb)) {
3199 t->tid_tab[tid] = NULL;
3200 mk_tid_release(skb, chan, tid);
3201 t4_ofld_send(adap, skb);
3202 } else
3203 cxgb4_queue_tid_release(t, chan, tid);
3204 if (old)
3205 atomic_dec(&t->tids_in_use);
3206 }
3207 EXPORT_SYMBOL(cxgb4_remove_tid);
3208
3209 /*
3210 * Allocate and initialize the TID tables. Returns 0 on success.
3211 */
3212 static int tid_init(struct tid_info *t)
3213 {
3214 size_t size;
3215 unsigned int stid_bmap_size;
3216 unsigned int natids = t->natids;
3217 struct adapter *adap = container_of(t, struct adapter, tids);
3218
3219 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3220 size = t->ntids * sizeof(*t->tid_tab) +
3221 natids * sizeof(*t->atid_tab) +
3222 t->nstids * sizeof(*t->stid_tab) +
3223 t->nsftids * sizeof(*t->stid_tab) +
3224 stid_bmap_size * sizeof(long) +
3225 t->nftids * sizeof(*t->ftid_tab) +
3226 t->nsftids * sizeof(*t->ftid_tab);
3227
3228 t->tid_tab = t4_alloc_mem(size);
3229 if (!t->tid_tab)
3230 return -ENOMEM;
3231
3232 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3233 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3234 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3235 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3236 spin_lock_init(&t->stid_lock);
3237 spin_lock_init(&t->atid_lock);
3238
3239 t->stids_in_use = 0;
3240 t->afree = NULL;
3241 t->atids_in_use = 0;
3242 atomic_set(&t->tids_in_use, 0);
3243
3244 /* Setup the free list for atid_tab and clear the stid bitmap. */
3245 if (natids) {
3246 while (--natids)
3247 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3248 t->afree = t->atid_tab;
3249 }
3250 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3251 /* Reserve stid 0 for T4/T5 adapters */
3252 if (!t->stid_base &&
3253 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3254 __set_bit(0, t->stid_bmap);
3255
3256 return 0;
3257 }
3258
3259 /**
3260 * cxgb4_create_server - create an IP server
3261 * @dev: the device
3262 * @stid: the server TID
3263 * @sip: local IP address to bind server to
3264 * @sport: the server's TCP port
3265 * @queue: queue to direct messages from this server to
3266 *
3267 * Create an IP server for the given port and address.
3268 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3269 */
3270 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3271 __be32 sip, __be16 sport, __be16 vlan,
3272 unsigned int queue)
3273 {
3274 unsigned int chan;
3275 struct sk_buff *skb;
3276 struct adapter *adap;
3277 struct cpl_pass_open_req *req;
3278 int ret;
3279
3280 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3281 if (!skb)
3282 return -ENOMEM;
3283
3284 adap = netdev2adap(dev);
3285 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3286 INIT_TP_WR(req, 0);
3287 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3288 req->local_port = sport;
3289 req->peer_port = htons(0);
3290 req->local_ip = sip;
3291 req->peer_ip = htonl(0);
3292 chan = rxq_to_chan(&adap->sge, queue);
3293 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3294 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3295 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3296 ret = t4_mgmt_tx(adap, skb);
3297 return net_xmit_eval(ret);
3298 }
3299 EXPORT_SYMBOL(cxgb4_create_server);
3300
3301 /* cxgb4_create_server6 - create an IPv6 server
3302 * @dev: the device
3303 * @stid: the server TID
3304 * @sip: local IPv6 address to bind server to
3305 * @sport: the server's TCP port
3306 * @queue: queue to direct messages from this server to
3307 *
3308 * Create an IPv6 server for the given port and address.
3309 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3310 */
3311 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3312 const struct in6_addr *sip, __be16 sport,
3313 unsigned int queue)
3314 {
3315 unsigned int chan;
3316 struct sk_buff *skb;
3317 struct adapter *adap;
3318 struct cpl_pass_open_req6 *req;
3319 int ret;
3320
3321 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3322 if (!skb)
3323 return -ENOMEM;
3324
3325 adap = netdev2adap(dev);
3326 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3327 INIT_TP_WR(req, 0);
3328 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3329 req->local_port = sport;
3330 req->peer_port = htons(0);
3331 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3332 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3333 req->peer_ip_hi = cpu_to_be64(0);
3334 req->peer_ip_lo = cpu_to_be64(0);
3335 chan = rxq_to_chan(&adap->sge, queue);
3336 req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3337 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3338 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3339 ret = t4_mgmt_tx(adap, skb);
3340 return net_xmit_eval(ret);
3341 }
3342 EXPORT_SYMBOL(cxgb4_create_server6);
3343
3344 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3345 unsigned int queue, bool ipv6)
3346 {
3347 struct sk_buff *skb;
3348 struct adapter *adap;
3349 struct cpl_close_listsvr_req *req;
3350 int ret;
3351
3352 adap = netdev2adap(dev);
3353
3354 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3355 if (!skb)
3356 return -ENOMEM;
3357
3358 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3359 INIT_TP_WR(req, 0);
3360 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3361 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3362 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
3363 ret = t4_mgmt_tx(adap, skb);
3364 return net_xmit_eval(ret);
3365 }
3366 EXPORT_SYMBOL(cxgb4_remove_server);
3367
3368 /**
3369 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3370 * @mtus: the HW MTU table
3371 * @mtu: the target MTU
3372 * @idx: index of selected entry in the MTU table
3373 *
3374 * Returns the index and the value in the HW MTU table that is closest to
3375 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3376 * table, in which case that smallest available value is selected.
3377 */
3378 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3379 unsigned int *idx)
3380 {
3381 unsigned int i = 0;
3382
3383 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3384 ++i;
3385 if (idx)
3386 *idx = i;
3387 return mtus[i];
3388 }
3389 EXPORT_SYMBOL(cxgb4_best_mtu);
3390
3391 /**
3392 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3393 * @mtus: the HW MTU table
3394 * @header_size: Header Size
3395 * @data_size_max: maximum Data Segment Size
3396 * @data_size_align: desired Data Segment Size Alignment (2^N)
3397 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3398 *
3399 * Similar to cxgb4_best_mtu() but instead of searching the Hardware
3400 * MTU Table based solely on a Maximum MTU parameter, we break that
3401 * parameter up into a Header Size and Maximum Data Segment Size, and
3402 * provide a desired Data Segment Size Alignment. If we find an MTU in
3403 * the Hardware MTU Table which will result in a Data Segment Size with
3404 * the requested alignment _and_ that MTU isn't "too far" from the
3405 * closest MTU, then we'll return that rather than the closest MTU.
3406 */
3407 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3408 unsigned short header_size,
3409 unsigned short data_size_max,
3410 unsigned short data_size_align,
3411 unsigned int *mtu_idxp)
3412 {
3413 unsigned short max_mtu = header_size + data_size_max;
3414 unsigned short data_size_align_mask = data_size_align - 1;
3415 int mtu_idx, aligned_mtu_idx;
3416
3417 /* Scan the MTU Table till we find an MTU which is larger than our
3418 * Maximum MTU or we reach the end of the table. Along the way,
3419 * record the last MTU found, if any, which will result in a Data
3420 * Segment Length matching the requested alignment.
3421 */
3422 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3423 unsigned short data_size = mtus[mtu_idx] - header_size;
3424
3425 /* If this MTU minus the Header Size would result in a
3426 * Data Segment Size of the desired alignment, remember it.
3427 */
3428 if ((data_size & data_size_align_mask) == 0)
3429 aligned_mtu_idx = mtu_idx;
3430
3431 /* If we're not at the end of the Hardware MTU Table and the
3432 * next element is larger than our Maximum MTU, drop out of
3433 * the loop.
3434 */
3435 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3436 break;
3437 }
3438
3439 /* If we fell out of the loop because we ran to the end of the table,
3440 * then we just have to use the last [largest] entry.
3441 */
3442 if (mtu_idx == NMTUS)
3443 mtu_idx--;
3444
3445 /* If we found an MTU which resulted in the requested Data Segment
3446 * Length alignment and that's "not far" from the largest MTU which is
3447 * less than or equal to the maximum MTU, then use that.
3448 */
3449 if (aligned_mtu_idx >= 0 &&
3450 mtu_idx - aligned_mtu_idx <= 1)
3451 mtu_idx = aligned_mtu_idx;
3452
3453 /* If the caller has passed in an MTU Index pointer, pass the
3454 * MTU Index back. Return the MTU value.
3455 */
3456 if (mtu_idxp)
3457 *mtu_idxp = mtu_idx;
3458 return mtus[mtu_idx];
3459 }
3460 EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3461
3462 /**
3463 * cxgb4_port_chan - get the HW channel of a port
3464 * @dev: the net device for the port
3465 *
3466 * Return the HW Tx channel of the given port.
3467 */
3468 unsigned int cxgb4_port_chan(const struct net_device *dev)
3469 {
3470 return netdev2pinfo(dev)->tx_chan;
3471 }
3472 EXPORT_SYMBOL(cxgb4_port_chan);
3473
3474 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3475 {
3476 struct adapter *adap = netdev2adap(dev);
3477 u32 v1, v2, lp_count, hp_count;
3478
3479 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3480 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3481 if (is_t4(adap->params.chip)) {
3482 lp_count = LP_COUNT_G(v1);
3483 hp_count = HP_COUNT_G(v1);
3484 } else {
3485 lp_count = LP_COUNT_T5_G(v1);
3486 hp_count = HP_COUNT_T5_G(v2);
3487 }
3488 return lpfifo ? lp_count : hp_count;
3489 }
3490 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3491
3492 /**
3493 * cxgb4_port_viid - get the VI id of a port
3494 * @dev: the net device for the port
3495 *
3496 * Return the VI id of the given port.
3497 */
3498 unsigned int cxgb4_port_viid(const struct net_device *dev)
3499 {
3500 return netdev2pinfo(dev)->viid;
3501 }
3502 EXPORT_SYMBOL(cxgb4_port_viid);
3503
3504 /**
3505 * cxgb4_port_idx - get the index of a port
3506 * @dev: the net device for the port
3507 *
3508 * Return the index of the given port.
3509 */
3510 unsigned int cxgb4_port_idx(const struct net_device *dev)
3511 {
3512 return netdev2pinfo(dev)->port_id;
3513 }
3514 EXPORT_SYMBOL(cxgb4_port_idx);
3515
3516 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3517 struct tp_tcp_stats *v6)
3518 {
3519 struct adapter *adap = pci_get_drvdata(pdev);
3520
3521 spin_lock(&adap->stats_lock);
3522 t4_tp_get_tcp_stats(adap, v4, v6);
3523 spin_unlock(&adap->stats_lock);
3524 }
3525 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3526
3527 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3528 const unsigned int *pgsz_order)
3529 {
3530 struct adapter *adap = netdev2adap(dev);
3531
3532 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3533 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3534 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3535 HPZ3_V(pgsz_order[3]));
3536 }
3537 EXPORT_SYMBOL(cxgb4_iscsi_init);
3538
3539 int cxgb4_flush_eq_cache(struct net_device *dev)
3540 {
3541 struct adapter *adap = netdev2adap(dev);
3542 int ret;
3543
3544 ret = t4_fwaddrspace_write(adap, adap->mbox,
3545 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3546 return ret;
3547 }
3548 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3549
3550 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3551 {
3552 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3553 __be64 indices;
3554 int ret;
3555
3556 spin_lock(&adap->win0_lock);
3557 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3558 sizeof(indices), (__be32 *)&indices,
3559 T4_MEMORY_READ);
3560 spin_unlock(&adap->win0_lock);
3561 if (!ret) {
3562 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3563 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3564 }
3565 return ret;
3566 }
3567
3568 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3569 u16 size)
3570 {
3571 struct adapter *adap = netdev2adap(dev);
3572 u16 hw_pidx, hw_cidx;
3573 int ret;
3574
3575 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3576 if (ret)
3577 goto out;
3578
3579 if (pidx != hw_pidx) {
3580 u16 delta;
3581 u32 val;
3582
3583 if (pidx >= hw_pidx)
3584 delta = pidx - hw_pidx;
3585 else
3586 delta = size - hw_pidx + pidx;
3587
3588 if (is_t4(adap->params.chip))
3589 val = PIDX_V(delta);
3590 else
3591 val = PIDX_T5_V(delta);
3592 wmb();
3593 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3594 QID_V(qid) | val);
3595 }
3596 out:
3597 return ret;
3598 }
3599 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3600
3601 void cxgb4_disable_db_coalescing(struct net_device *dev)
3602 {
3603 struct adapter *adap;
3604
3605 adap = netdev2adap(dev);
3606 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
3607 NOCOALESCE_F);
3608 }
3609 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3610
3611 void cxgb4_enable_db_coalescing(struct net_device *dev)
3612 {
3613 struct adapter *adap;
3614
3615 adap = netdev2adap(dev);
3616 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3617 }
3618 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3619
3620 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3621 {
3622 struct adapter *adap;
3623 u32 offset, memtype, memaddr;
3624 u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
3625 u32 edc0_end, edc1_end, mc0_end, mc1_end;
3626 int ret;
3627
3628 adap = netdev2adap(dev);
3629
3630 offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3631
3632 /* Figure out where the offset lands in the Memory Type/Address scheme.
3633 * This code assumes that the memory is laid out starting at offset 0
3634 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3635 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have
3636 * MC0, and some have both MC0 and MC1.
3637 */
3638 size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3639 edc0_size = EDRAM0_SIZE_G(size) << 20;
3640 size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3641 edc1_size = EDRAM1_SIZE_G(size) << 20;
3642 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3643 mc0_size = EXT_MEM0_SIZE_G(size) << 20;
3644
3645 edc0_end = edc0_size;
3646 edc1_end = edc0_end + edc1_size;
3647 mc0_end = edc1_end + mc0_size;
3648
3649 if (offset < edc0_end) {
3650 memtype = MEM_EDC0;
3651 memaddr = offset;
3652 } else if (offset < edc1_end) {
3653 memtype = MEM_EDC1;
3654 memaddr = offset - edc0_end;
3655 } else {
3656 if (offset < mc0_end) {
3657 memtype = MEM_MC0;
3658 memaddr = offset - edc1_end;
3659 } else if (is_t4(adap->params.chip)) {
3660 /* T4 only has a single memory channel */
3661 goto err;
3662 } else {
3663 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3664 mc1_size = EXT_MEM1_SIZE_G(size) << 20;
3665 mc1_end = mc0_end + mc1_size;
3666 if (offset < mc1_end) {
3667 memtype = MEM_MC1;
3668 memaddr = offset - mc0_end;
3669 } else {
3670 /* offset beyond the end of any memory */
3671 goto err;
3672 }
3673 }
3674 }
3675
3676 spin_lock(&adap->win0_lock);
3677 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3678 spin_unlock(&adap->win0_lock);
3679 return ret;
3680
3681 err:
3682 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3683 stag, offset);
3684 return -EINVAL;
3685 }
3686 EXPORT_SYMBOL(cxgb4_read_tpte);
3687
3688 u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3689 {
3690 u32 hi, lo;
3691 struct adapter *adap;
3692
3693 adap = netdev2adap(dev);
3694 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3695 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
3696
3697 return ((u64)hi << 32) | (u64)lo;
3698 }
3699 EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3700
3701 int cxgb4_bar2_sge_qregs(struct net_device *dev,
3702 unsigned int qid,
3703 enum cxgb4_bar2_qtype qtype,
3704 u64 *pbar2_qoffset,
3705 unsigned int *pbar2_qid)
3706 {
3707 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
3708 qid,
3709 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3710 ? T4_BAR2_QTYPE_EGRESS
3711 : T4_BAR2_QTYPE_INGRESS),
3712 pbar2_qoffset,
3713 pbar2_qid);
3714 }
3715 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3716
3717 static struct pci_driver cxgb4_driver;
3718
3719 static void check_neigh_update(struct neighbour *neigh)
3720 {
3721 const struct device *parent;
3722 const struct net_device *netdev = neigh->dev;
3723
3724 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3725 netdev = vlan_dev_real_dev(netdev);
3726 parent = netdev->dev.parent;
3727 if (parent && parent->driver == &cxgb4_driver.driver)
3728 t4_l2t_update(dev_get_drvdata(parent), neigh);
3729 }
3730
3731 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3732 void *data)
3733 {
3734 switch (event) {
3735 case NETEVENT_NEIGH_UPDATE:
3736 check_neigh_update(data);
3737 break;
3738 case NETEVENT_REDIRECT:
3739 default:
3740 break;
3741 }
3742 return 0;
3743 }
3744
3745 static bool netevent_registered;
3746 static struct notifier_block cxgb4_netevent_nb = {
3747 .notifier_call = netevent_cb
3748 };
3749
3750 static void drain_db_fifo(struct adapter *adap, int usecs)
3751 {
3752 u32 v1, v2, lp_count, hp_count;
3753
3754 do {
3755 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3756 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3757 if (is_t4(adap->params.chip)) {
3758 lp_count = LP_COUNT_G(v1);
3759 hp_count = HP_COUNT_G(v1);
3760 } else {
3761 lp_count = LP_COUNT_T5_G(v1);
3762 hp_count = HP_COUNT_T5_G(v2);
3763 }
3764
3765 if (lp_count == 0 && hp_count == 0)
3766 break;
3767 set_current_state(TASK_UNINTERRUPTIBLE);
3768 schedule_timeout(usecs_to_jiffies(usecs));
3769 } while (1);
3770 }
3771
3772 static void disable_txq_db(struct sge_txq *q)
3773 {
3774 unsigned long flags;
3775
3776 spin_lock_irqsave(&q->db_lock, flags);
3777 q->db_disabled = 1;
3778 spin_unlock_irqrestore(&q->db_lock, flags);
3779 }
3780
3781 static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3782 {
3783 spin_lock_irq(&q->db_lock);
3784 if (q->db_pidx_inc) {
3785 /* Make sure that all writes to the TX descriptors
3786 * are committed before we tell HW about them.
3787 */
3788 wmb();
3789 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3790 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
3791 q->db_pidx_inc = 0;
3792 }
3793 q->db_disabled = 0;
3794 spin_unlock_irq(&q->db_lock);
3795 }
3796
3797 static void disable_dbs(struct adapter *adap)
3798 {
3799 int i;
3800
3801 for_each_ethrxq(&adap->sge, i)
3802 disable_txq_db(&adap->sge.ethtxq[i].q);
3803 for_each_ofldrxq(&adap->sge, i)
3804 disable_txq_db(&adap->sge.ofldtxq[i].q);
3805 for_each_port(adap, i)
3806 disable_txq_db(&adap->sge.ctrlq[i].q);
3807 }
3808
3809 static void enable_dbs(struct adapter *adap)
3810 {
3811 int i;
3812
3813 for_each_ethrxq(&adap->sge, i)
3814 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3815 for_each_ofldrxq(&adap->sge, i)
3816 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3817 for_each_port(adap, i)
3818 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3819 }
3820
3821 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3822 {
3823 if (adap->uld_handle[CXGB4_ULD_RDMA])
3824 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3825 cmd);
3826 }
3827
3828 static void process_db_full(struct work_struct *work)
3829 {
3830 struct adapter *adap;
3831
3832 adap = container_of(work, struct adapter, db_full_task);
3833
3834 drain_db_fifo(adap, dbfifo_drain_delay);
3835 enable_dbs(adap);
3836 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3837 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3838 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3839 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3840 }
3841
3842 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3843 {
3844 u16 hw_pidx, hw_cidx;
3845 int ret;
3846
3847 spin_lock_irq(&q->db_lock);
3848 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3849 if (ret)
3850 goto out;
3851 if (q->db_pidx != hw_pidx) {
3852 u16 delta;
3853 u32 val;
3854
3855 if (q->db_pidx >= hw_pidx)
3856 delta = q->db_pidx - hw_pidx;
3857 else
3858 delta = q->size - hw_pidx + q->db_pidx;
3859
3860 if (is_t4(adap->params.chip))
3861 val = PIDX_V(delta);
3862 else
3863 val = PIDX_T5_V(delta);
3864 wmb();
3865 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3866 QID_V(q->cntxt_id) | val);
3867 }
3868 out:
3869 q->db_disabled = 0;
3870 q->db_pidx_inc = 0;
3871 spin_unlock_irq(&q->db_lock);
3872 if (ret)
3873 CH_WARN(adap, "DB drop recovery failed.\n");
3874 }
3875 static void recover_all_queues(struct adapter *adap)
3876 {
3877 int i;
3878
3879 for_each_ethrxq(&adap->sge, i)
3880 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3881 for_each_ofldrxq(&adap->sge, i)
3882 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3883 for_each_port(adap, i)
3884 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3885 }
3886
3887 static void process_db_drop(struct work_struct *work)
3888 {
3889 struct adapter *adap;
3890
3891 adap = container_of(work, struct adapter, db_drop_task);
3892
3893 if (is_t4(adap->params.chip)) {
3894 drain_db_fifo(adap, dbfifo_drain_delay);
3895 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3896 drain_db_fifo(adap, dbfifo_drain_delay);
3897 recover_all_queues(adap);
3898 drain_db_fifo(adap, dbfifo_drain_delay);
3899 enable_dbs(adap);
3900 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3901 } else {
3902 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3903 u16 qid = (dropped_db >> 15) & 0x1ffff;
3904 u16 pidx_inc = dropped_db & 0x1fff;
3905 u64 bar2_qoffset;
3906 unsigned int bar2_qid;
3907 int ret;
3908
3909 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
3910 &bar2_qoffset, &bar2_qid);
3911 if (ret)
3912 dev_err(adap->pdev_dev, "doorbell drop recovery: "
3913 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3914 else
3915 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
3916 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
3917
3918 /* Re-enable BAR2 WC */
3919 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3920 }
3921
3922 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
3923 }
3924
3925 void t4_db_full(struct adapter *adap)
3926 {
3927 if (is_t4(adap->params.chip)) {
3928 disable_dbs(adap);
3929 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3930 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3931 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
3932 queue_work(adap->workq, &adap->db_full_task);
3933 }
3934 }
3935
3936 void t4_db_dropped(struct adapter *adap)
3937 {
3938 if (is_t4(adap->params.chip)) {
3939 disable_dbs(adap);
3940 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3941 }
3942 queue_work(adap->workq, &adap->db_drop_task);
3943 }
3944
3945 static void uld_attach(struct adapter *adap, unsigned int uld)
3946 {
3947 void *handle;
3948 struct cxgb4_lld_info lli;
3949 unsigned short i;
3950
3951 lli.pdev = adap->pdev;
3952 lli.pf = adap->fn;
3953 lli.l2t = adap->l2t;
3954 lli.tids = &adap->tids;
3955 lli.ports = adap->port;
3956 lli.vr = &adap->vres;
3957 lli.mtus = adap->params.mtus;
3958 if (uld == CXGB4_ULD_RDMA) {
3959 lli.rxq_ids = adap->sge.rdma_rxq;
3960 lli.ciq_ids = adap->sge.rdma_ciq;
3961 lli.nrxq = adap->sge.rdmaqs;
3962 lli.nciq = adap->sge.rdmaciqs;
3963 } else if (uld == CXGB4_ULD_ISCSI) {
3964 lli.rxq_ids = adap->sge.ofld_rxq;
3965 lli.nrxq = adap->sge.ofldqsets;
3966 }
3967 lli.ntxq = adap->sge.ofldqsets;
3968 lli.nchan = adap->params.nports;
3969 lli.nports = adap->params.nports;
3970 lli.wr_cred = adap->params.ofldq_wr_cred;
3971 lli.adapter_type = adap->params.chip;
3972 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
3973 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
3974 lli.udb_density = 1 << adap->params.sge.eq_qpp;
3975 lli.ucq_density = 1 << adap->params.sge.iq_qpp;
3976 lli.filt_mode = adap->params.tp.vlan_pri_map;
3977 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3978 for (i = 0; i < NCHAN; i++)
3979 lli.tx_modq[i] = i;
3980 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
3981 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
3982 lli.fw_vers = adap->params.fw_vers;
3983 lli.dbfifo_int_thresh = dbfifo_int_thresh;
3984 lli.sge_ingpadboundary = adap->sge.fl_align;
3985 lli.sge_egrstatuspagesize = adap->sge.stat_len;
3986 lli.sge_pktshift = adap->sge.pktshift;
3987 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3988 lli.max_ordird_qp = adap->params.max_ordird_qp;
3989 lli.max_ird_adapter = adap->params.max_ird_adapter;
3990 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
3991
3992 handle = ulds[uld].add(&lli);
3993 if (IS_ERR(handle)) {
3994 dev_warn(adap->pdev_dev,
3995 "could not attach to the %s driver, error %ld\n",
3996 uld_str[uld], PTR_ERR(handle));
3997 return;
3998 }
3999
4000 adap->uld_handle[uld] = handle;
4001
4002 if (!netevent_registered) {
4003 register_netevent_notifier(&cxgb4_netevent_nb);
4004 netevent_registered = true;
4005 }
4006
4007 if (adap->flags & FULL_INIT_DONE)
4008 ulds[uld].state_change(handle, CXGB4_STATE_UP);
4009 }
4010
4011 static void attach_ulds(struct adapter *adap)
4012 {
4013 unsigned int i;
4014
4015 spin_lock(&adap_rcu_lock);
4016 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4017 spin_unlock(&adap_rcu_lock);
4018
4019 mutex_lock(&uld_mutex);
4020 list_add_tail(&adap->list_node, &adapter_list);
4021 for (i = 0; i < CXGB4_ULD_MAX; i++)
4022 if (ulds[i].add)
4023 uld_attach(adap, i);
4024 mutex_unlock(&uld_mutex);
4025 }
4026
4027 static void detach_ulds(struct adapter *adap)
4028 {
4029 unsigned int i;
4030
4031 mutex_lock(&uld_mutex);
4032 list_del(&adap->list_node);
4033 for (i = 0; i < CXGB4_ULD_MAX; i++)
4034 if (adap->uld_handle[i]) {
4035 ulds[i].state_change(adap->uld_handle[i],
4036 CXGB4_STATE_DETACH);
4037 adap->uld_handle[i] = NULL;
4038 }
4039 if (netevent_registered && list_empty(&adapter_list)) {
4040 unregister_netevent_notifier(&cxgb4_netevent_nb);
4041 netevent_registered = false;
4042 }
4043 mutex_unlock(&uld_mutex);
4044
4045 spin_lock(&adap_rcu_lock);
4046 list_del_rcu(&adap->rcu_node);
4047 spin_unlock(&adap_rcu_lock);
4048 }
4049
4050 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4051 {
4052 unsigned int i;
4053
4054 mutex_lock(&uld_mutex);
4055 for (i = 0; i < CXGB4_ULD_MAX; i++)
4056 if (adap->uld_handle[i])
4057 ulds[i].state_change(adap->uld_handle[i], new_state);
4058 mutex_unlock(&uld_mutex);
4059 }
4060
4061 /**
4062 * cxgb4_register_uld - register an upper-layer driver
4063 * @type: the ULD type
4064 * @p: the ULD methods
4065 *
4066 * Registers an upper-layer driver with this driver and notifies the ULD
4067 * about any presently available devices that support its type. Returns
4068 * %-EBUSY if a ULD of the same type is already registered.
4069 */
4070 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4071 {
4072 int ret = 0;
4073 struct adapter *adap;
4074
4075 if (type >= CXGB4_ULD_MAX)
4076 return -EINVAL;
4077 mutex_lock(&uld_mutex);
4078 if (ulds[type].add) {
4079 ret = -EBUSY;
4080 goto out;
4081 }
4082 ulds[type] = *p;
4083 list_for_each_entry(adap, &adapter_list, list_node)
4084 uld_attach(adap, type);
4085 out: mutex_unlock(&uld_mutex);
4086 return ret;
4087 }
4088 EXPORT_SYMBOL(cxgb4_register_uld);
4089
4090 /**
4091 * cxgb4_unregister_uld - unregister an upper-layer driver
4092 * @type: the ULD type
4093 *
4094 * Unregisters an existing upper-layer driver.
4095 */
4096 int cxgb4_unregister_uld(enum cxgb4_uld type)
4097 {
4098 struct adapter *adap;
4099
4100 if (type >= CXGB4_ULD_MAX)
4101 return -EINVAL;
4102 mutex_lock(&uld_mutex);
4103 list_for_each_entry(adap, &adapter_list, list_node)
4104 adap->uld_handle[type] = NULL;
4105 ulds[type].add = NULL;
4106 mutex_unlock(&uld_mutex);
4107 return 0;
4108 }
4109 EXPORT_SYMBOL(cxgb4_unregister_uld);
4110
4111 #if IS_ENABLED(CONFIG_IPV6)
4112 static int cxgb4_inet6addr_handler(struct notifier_block *this,
4113 unsigned long event, void *data)
4114 {
4115 struct inet6_ifaddr *ifa = data;
4116 struct net_device *event_dev = ifa->idev->dev;
4117 const struct device *parent = NULL;
4118 #if IS_ENABLED(CONFIG_BONDING)
4119 struct adapter *adap;
4120 #endif
4121 if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4122 event_dev = vlan_dev_real_dev(event_dev);
4123 #if IS_ENABLED(CONFIG_BONDING)
4124 if (event_dev->flags & IFF_MASTER) {
4125 list_for_each_entry(adap, &adapter_list, list_node) {
4126 switch (event) {
4127 case NETDEV_UP:
4128 cxgb4_clip_get(adap->port[0],
4129 (const u32 *)ifa, 1);
4130 break;
4131 case NETDEV_DOWN:
4132 cxgb4_clip_release(adap->port[0],
4133 (const u32 *)ifa, 1);
4134 break;
4135 default:
4136 break;
4137 }
4138 }
4139 return NOTIFY_OK;
4140 }
4141 #endif
4142
4143 if (event_dev)
4144 parent = event_dev->dev.parent;
4145
4146 if (parent && parent->driver == &cxgb4_driver.driver) {
4147 switch (event) {
4148 case NETDEV_UP:
4149 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
4150 break;
4151 case NETDEV_DOWN:
4152 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
4153 break;
4154 default:
4155 break;
4156 }
4157 }
4158 return NOTIFY_OK;
4159 }
4160
4161 static bool inet6addr_registered;
4162 static struct notifier_block cxgb4_inet6addr_notifier = {
4163 .notifier_call = cxgb4_inet6addr_handler
4164 };
4165
4166 static void update_clip(const struct adapter *adap)
4167 {
4168 int i;
4169 struct net_device *dev;
4170 int ret;
4171
4172 rcu_read_lock();
4173
4174 for (i = 0; i < MAX_NPORTS; i++) {
4175 dev = adap->port[i];
4176 ret = 0;
4177
4178 if (dev)
4179 ret = cxgb4_update_root_dev_clip(dev);
4180
4181 if (ret < 0)
4182 break;
4183 }
4184 rcu_read_unlock();
4185 }
4186 #endif /* IS_ENABLED(CONFIG_IPV6) */
4187
4188 /**
4189 * cxgb_up - enable the adapter
4190 * @adap: adapter being enabled
4191 *
4192 * Called when the first port is enabled, this function performs the
4193 * actions necessary to make an adapter operational, such as completing
4194 * the initialization of HW modules, and enabling interrupts.
4195 *
4196 * Must be called with the rtnl lock held.
4197 */
4198 static int cxgb_up(struct adapter *adap)
4199 {
4200 int err;
4201
4202 err = setup_sge_queues(adap);
4203 if (err)
4204 goto out;
4205 err = setup_rss(adap);
4206 if (err)
4207 goto freeq;
4208
4209 if (adap->flags & USING_MSIX) {
4210 name_msix_vecs(adap);
4211 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4212 adap->msix_info[0].desc, adap);
4213 if (err)
4214 goto irq_err;
4215
4216 err = request_msix_queue_irqs(adap);
4217 if (err) {
4218 free_irq(adap->msix_info[0].vec, adap);
4219 goto irq_err;
4220 }
4221 } else {
4222 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4223 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4224 adap->port[0]->name, adap);
4225 if (err)
4226 goto irq_err;
4227 }
4228 enable_rx(adap);
4229 t4_sge_start(adap);
4230 t4_intr_enable(adap);
4231 adap->flags |= FULL_INIT_DONE;
4232 notify_ulds(adap, CXGB4_STATE_UP);
4233 #if IS_ENABLED(CONFIG_IPV6)
4234 update_clip(adap);
4235 #endif
4236 out:
4237 return err;
4238 irq_err:
4239 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4240 freeq:
4241 t4_free_sge_resources(adap);
4242 goto out;
4243 }
4244
4245 static void cxgb_down(struct adapter *adapter)
4246 {
4247 t4_intr_disable(adapter);
4248 cancel_work_sync(&adapter->tid_release_task);
4249 cancel_work_sync(&adapter->db_full_task);
4250 cancel_work_sync(&adapter->db_drop_task);
4251 adapter->tid_release_task_busy = false;
4252 adapter->tid_release_head = NULL;
4253
4254 if (adapter->flags & USING_MSIX) {
4255 free_msix_queue_irqs(adapter);
4256 free_irq(adapter->msix_info[0].vec, adapter);
4257 } else
4258 free_irq(adapter->pdev->irq, adapter);
4259 quiesce_rx(adapter);
4260 t4_sge_stop(adapter);
4261 t4_free_sge_resources(adapter);
4262 adapter->flags &= ~FULL_INIT_DONE;
4263 }
4264
4265 /*
4266 * net_device operations
4267 */
4268 static int cxgb_open(struct net_device *dev)
4269 {
4270 int err;
4271 struct port_info *pi = netdev_priv(dev);
4272 struct adapter *adapter = pi->adapter;
4273
4274 netif_carrier_off(dev);
4275
4276 if (!(adapter->flags & FULL_INIT_DONE)) {
4277 err = cxgb_up(adapter);
4278 if (err < 0)
4279 return err;
4280 }
4281
4282 err = link_start(dev);
4283 if (!err)
4284 netif_tx_start_all_queues(dev);
4285 return err;
4286 }
4287
4288 static int cxgb_close(struct net_device *dev)
4289 {
4290 struct port_info *pi = netdev_priv(dev);
4291 struct adapter *adapter = pi->adapter;
4292
4293 netif_tx_stop_all_queues(dev);
4294 netif_carrier_off(dev);
4295 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4296 }
4297
4298 /* Return an error number if the indicated filter isn't writable ...
4299 */
4300 static int writable_filter(struct filter_entry *f)
4301 {
4302 if (f->locked)
4303 return -EPERM;
4304 if (f->pending)
4305 return -EBUSY;
4306
4307 return 0;
4308 }
4309
4310 /* Delete the filter at the specified index (if valid). The checks for all
4311 * the common problems with doing this like the filter being locked, currently
4312 * pending in another operation, etc.
4313 */
4314 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4315 {
4316 struct filter_entry *f;
4317 int ret;
4318
4319 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4320 return -EINVAL;
4321
4322 f = &adapter->tids.ftid_tab[fidx];
4323 ret = writable_filter(f);
4324 if (ret)
4325 return ret;
4326 if (f->valid)
4327 return del_filter_wr(adapter, fidx);
4328
4329 return 0;
4330 }
4331
4332 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4333 __be32 sip, __be16 sport, __be16 vlan,
4334 unsigned int queue, unsigned char port, unsigned char mask)
4335 {
4336 int ret;
4337 struct filter_entry *f;
4338 struct adapter *adap;
4339 int i;
4340 u8 *val;
4341
4342 adap = netdev2adap(dev);
4343
4344 /* Adjust stid to correct filter index */
4345 stid -= adap->tids.sftid_base;
4346 stid += adap->tids.nftids;
4347
4348 /* Check to make sure the filter requested is writable ...
4349 */
4350 f = &adap->tids.ftid_tab[stid];
4351 ret = writable_filter(f);
4352 if (ret)
4353 return ret;
4354
4355 /* Clear out any old resources being used by the filter before
4356 * we start constructing the new filter.
4357 */
4358 if (f->valid)
4359 clear_filter(adap, f);
4360
4361 /* Clear out filter specifications */
4362 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4363 f->fs.val.lport = cpu_to_be16(sport);
4364 f->fs.mask.lport = ~0;
4365 val = (u8 *)&sip;
4366 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4367 for (i = 0; i < 4; i++) {
4368 f->fs.val.lip[i] = val[i];
4369 f->fs.mask.lip[i] = ~0;
4370 }
4371 if (adap->params.tp.vlan_pri_map & PORT_F) {
4372 f->fs.val.iport = port;
4373 f->fs.mask.iport = mask;
4374 }
4375 }
4376
4377 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
4378 f->fs.val.proto = IPPROTO_TCP;
4379 f->fs.mask.proto = ~0;
4380 }
4381
4382 f->fs.dirsteer = 1;
4383 f->fs.iq = queue;
4384 /* Mark filter as locked */
4385 f->locked = 1;
4386 f->fs.rpttid = 1;
4387
4388 ret = set_filter_wr(adap, stid);
4389 if (ret) {
4390 clear_filter(adap, f);
4391 return ret;
4392 }
4393
4394 return 0;
4395 }
4396 EXPORT_SYMBOL(cxgb4_create_server_filter);
4397
4398 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4399 unsigned int queue, bool ipv6)
4400 {
4401 int ret;
4402 struct filter_entry *f;
4403 struct adapter *adap;
4404
4405 adap = netdev2adap(dev);
4406
4407 /* Adjust stid to correct filter index */
4408 stid -= adap->tids.sftid_base;
4409 stid += adap->tids.nftids;
4410
4411 f = &adap->tids.ftid_tab[stid];
4412 /* Unlock the filter */
4413 f->locked = 0;
4414
4415 ret = delete_filter(adap, stid);
4416 if (ret)
4417 return ret;
4418
4419 return 0;
4420 }
4421 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4422
4423 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4424 struct rtnl_link_stats64 *ns)
4425 {
4426 struct port_stats stats;
4427 struct port_info *p = netdev_priv(dev);
4428 struct adapter *adapter = p->adapter;
4429
4430 /* Block retrieving statistics during EEH error
4431 * recovery. Otherwise, the recovery might fail
4432 * and the PCI device will be removed permanently
4433 */
4434 spin_lock(&adapter->stats_lock);
4435 if (!netif_device_present(dev)) {
4436 spin_unlock(&adapter->stats_lock);
4437 return ns;
4438 }
4439 t4_get_port_stats(adapter, p->tx_chan, &stats);
4440 spin_unlock(&adapter->stats_lock);
4441
4442 ns->tx_bytes = stats.tx_octets;
4443 ns->tx_packets = stats.tx_frames;
4444 ns->rx_bytes = stats.rx_octets;
4445 ns->rx_packets = stats.rx_frames;
4446 ns->multicast = stats.rx_mcast_frames;
4447
4448 /* detailed rx_errors */
4449 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4450 stats.rx_runt;
4451 ns->rx_over_errors = 0;
4452 ns->rx_crc_errors = stats.rx_fcs_err;
4453 ns->rx_frame_errors = stats.rx_symbol_err;
4454 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4455 stats.rx_ovflow2 + stats.rx_ovflow3 +
4456 stats.rx_trunc0 + stats.rx_trunc1 +
4457 stats.rx_trunc2 + stats.rx_trunc3;
4458 ns->rx_missed_errors = 0;
4459
4460 /* detailed tx_errors */
4461 ns->tx_aborted_errors = 0;
4462 ns->tx_carrier_errors = 0;
4463 ns->tx_fifo_errors = 0;
4464 ns->tx_heartbeat_errors = 0;
4465 ns->tx_window_errors = 0;
4466
4467 ns->tx_errors = stats.tx_error_frames;
4468 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4469 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4470 return ns;
4471 }
4472
4473 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4474 {
4475 unsigned int mbox;
4476 int ret = 0, prtad, devad;
4477 struct port_info *pi = netdev_priv(dev);
4478 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4479
4480 switch (cmd) {
4481 case SIOCGMIIPHY:
4482 if (pi->mdio_addr < 0)
4483 return -EOPNOTSUPP;
4484 data->phy_id = pi->mdio_addr;
4485 break;
4486 case SIOCGMIIREG:
4487 case SIOCSMIIREG:
4488 if (mdio_phy_id_is_c45(data->phy_id)) {
4489 prtad = mdio_phy_id_prtad(data->phy_id);
4490 devad = mdio_phy_id_devad(data->phy_id);
4491 } else if (data->phy_id < 32) {
4492 prtad = data->phy_id;
4493 devad = 0;
4494 data->reg_num &= 0x1f;
4495 } else
4496 return -EINVAL;
4497
4498 mbox = pi->adapter->fn;
4499 if (cmd == SIOCGMIIREG)
4500 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4501 data->reg_num, &data->val_out);
4502 else
4503 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4504 data->reg_num, data->val_in);
4505 break;
4506 default:
4507 return -EOPNOTSUPP;
4508 }
4509 return ret;
4510 }
4511
4512 static void cxgb_set_rxmode(struct net_device *dev)
4513 {
4514 /* unfortunately we can't return errors to the stack */
4515 set_rxmode(dev, -1, false);
4516 }
4517
4518 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4519 {
4520 int ret;
4521 struct port_info *pi = netdev_priv(dev);
4522
4523 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4524 return -EINVAL;
4525 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4526 -1, -1, -1, true);
4527 if (!ret)
4528 dev->mtu = new_mtu;
4529 return ret;
4530 }
4531
4532 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4533 {
4534 int ret;
4535 struct sockaddr *addr = p;
4536 struct port_info *pi = netdev_priv(dev);
4537
4538 if (!is_valid_ether_addr(addr->sa_data))
4539 return -EADDRNOTAVAIL;
4540
4541 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4542 pi->xact_addr_filt, addr->sa_data, true, true);
4543 if (ret < 0)
4544 return ret;
4545
4546 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4547 pi->xact_addr_filt = ret;
4548 return 0;
4549 }
4550
4551 #ifdef CONFIG_NET_POLL_CONTROLLER
4552 static void cxgb_netpoll(struct net_device *dev)
4553 {
4554 struct port_info *pi = netdev_priv(dev);
4555 struct adapter *adap = pi->adapter;
4556
4557 if (adap->flags & USING_MSIX) {
4558 int i;
4559 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4560
4561 for (i = pi->nqsets; i; i--, rx++)
4562 t4_sge_intr_msix(0, &rx->rspq);
4563 } else
4564 t4_intr_handler(adap)(0, adap);
4565 }
4566 #endif
4567
4568 static const struct net_device_ops cxgb4_netdev_ops = {
4569 .ndo_open = cxgb_open,
4570 .ndo_stop = cxgb_close,
4571 .ndo_start_xmit = t4_eth_xmit,
4572 .ndo_select_queue = cxgb_select_queue,
4573 .ndo_get_stats64 = cxgb_get_stats,
4574 .ndo_set_rx_mode = cxgb_set_rxmode,
4575 .ndo_set_mac_address = cxgb_set_mac_addr,
4576 .ndo_set_features = cxgb_set_features,
4577 .ndo_validate_addr = eth_validate_addr,
4578 .ndo_do_ioctl = cxgb_ioctl,
4579 .ndo_change_mtu = cxgb_change_mtu,
4580 #ifdef CONFIG_NET_POLL_CONTROLLER
4581 .ndo_poll_controller = cxgb_netpoll,
4582 #endif
4583 #ifdef CONFIG_NET_RX_BUSY_POLL
4584 .ndo_busy_poll = cxgb_busy_poll,
4585 #endif
4586
4587 };
4588
4589 void t4_fatal_err(struct adapter *adap)
4590 {
4591 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
4592 t4_intr_disable(adap);
4593 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4594 }
4595
4596 /* Return the specified PCI-E Configuration Space register from our Physical
4597 * Function. We try first via a Firmware LDST Command since we prefer to let
4598 * the firmware own all of these registers, but if that fails we go for it
4599 * directly ourselves.
4600 */
4601 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4602 {
4603 struct fw_ldst_cmd ldst_cmd;
4604 u32 val;
4605 int ret;
4606
4607 /* Construct and send the Firmware LDST Command to retrieve the
4608 * specified PCI-E Configuration Space register.
4609 */
4610 memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4611 ldst_cmd.op_to_addrspace =
4612 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4613 FW_CMD_REQUEST_F |
4614 FW_CMD_READ_F |
4615 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
4616 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
4617 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
4618 ldst_cmd.u.pcie.ctrl_to_fn =
4619 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
4620 ldst_cmd.u.pcie.r = reg;
4621 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4622 &ldst_cmd);
4623
4624 /* If the LDST Command suucceeded, exctract the returned register
4625 * value. Otherwise read it directly ourself.
4626 */
4627 if (ret == 0)
4628 val = ntohl(ldst_cmd.u.pcie.data[0]);
4629 else
4630 t4_hw_pci_read_cfg4(adap, reg, &val);
4631
4632 return val;
4633 }
4634
4635 static void setup_memwin(struct adapter *adap)
4636 {
4637 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
4638
4639 if (is_t4(adap->params.chip)) {
4640 u32 bar0;
4641
4642 /* Truncation intentional: we only read the bottom 32-bits of
4643 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor
4644 * mechanism to read BAR0 instead of using
4645 * pci_resource_start() because we could be operating from
4646 * within a Virtual Machine which is trapping our accesses to
4647 * our Configuration Space and we need to set up the PCI-E
4648 * Memory Window decoders with the actual addresses which will
4649 * be coming across the PCI-E link.
4650 */
4651 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4652 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4653 adap->t4_bar0 = bar0;
4654
4655 mem_win0_base = bar0 + MEMWIN0_BASE;
4656 mem_win1_base = bar0 + MEMWIN1_BASE;
4657 mem_win2_base = bar0 + MEMWIN2_BASE;
4658 mem_win2_aperture = MEMWIN2_APERTURE;
4659 } else {
4660 /* For T5, only relative offset inside the PCIe BAR is passed */
4661 mem_win0_base = MEMWIN0_BASE;
4662 mem_win1_base = MEMWIN1_BASE;
4663 mem_win2_base = MEMWIN2_BASE_T5;
4664 mem_win2_aperture = MEMWIN2_APERTURE_T5;
4665 }
4666 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4667 mem_win0_base | BIR_V(0) |
4668 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4669 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4670 mem_win1_base | BIR_V(0) |
4671 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4672 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4673 mem_win2_base | BIR_V(0) |
4674 WINDOW_V(ilog2(mem_win2_aperture) - 10));
4675 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
4676 }
4677
4678 static void setup_memwin_rdma(struct adapter *adap)
4679 {
4680 if (adap->vres.ocq.size) {
4681 u32 start;
4682 unsigned int sz_kb;
4683
4684 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4685 start &= PCI_BASE_ADDRESS_MEM_MASK;
4686 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4687 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4688 t4_write_reg(adap,
4689 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4690 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
4691 t4_write_reg(adap,
4692 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
4693 adap->vres.ocq.start);
4694 t4_read_reg(adap,
4695 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
4696 }
4697 }
4698
4699 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4700 {
4701 u32 v;
4702 int ret;
4703
4704 /* get device capabilities */
4705 memset(c, 0, sizeof(*c));
4706 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4707 FW_CMD_REQUEST_F | FW_CMD_READ_F);
4708 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4709 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4710 if (ret < 0)
4711 return ret;
4712
4713 /* select capabilities we'll be using */
4714 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4715 if (!vf_acls)
4716 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4717 else
4718 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4719 } else if (vf_acls) {
4720 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4721 return ret;
4722 }
4723 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4724 FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
4725 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4726 if (ret < 0)
4727 return ret;
4728
4729 ret = t4_config_glbl_rss(adap, adap->fn,
4730 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4731 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4732 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
4733 if (ret < 0)
4734 return ret;
4735
4736 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4737 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4738 if (ret < 0)
4739 return ret;
4740
4741 t4_sge_init(adap);
4742
4743 /* tweak some settings */
4744 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
4745 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
4746 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4747 v = t4_read_reg(adap, TP_PIO_DATA_A);
4748 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
4749
4750 /* first 4 Tx modulation queues point to consecutive Tx channels */
4751 adap->params.tp.tx_modq_map = 0xE4;
4752 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4753 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
4754
4755 /* associate each Tx modulation queue with consecutive Tx channels */
4756 v = 0x84218421;
4757 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4758 &v, 1, TP_TX_SCHED_HDR_A);
4759 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4760 &v, 1, TP_TX_SCHED_FIFO_A);
4761 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4762 &v, 1, TP_TX_SCHED_PCMD_A);
4763
4764 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4765 if (is_offload(adap)) {
4766 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4767 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4768 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4769 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4770 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4771 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4772 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4773 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4774 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4775 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4776 }
4777
4778 /* get basic stuff going */
4779 return t4_early_init(adap, adap->fn);
4780 }
4781
4782 /*
4783 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4784 */
4785 #define MAX_ATIDS 8192U
4786
4787 /*
4788 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4789 *
4790 * If the firmware we're dealing with has Configuration File support, then
4791 * we use that to perform all configuration
4792 */
4793
4794 /*
4795 * Tweak configuration based on module parameters, etc. Most of these have
4796 * defaults assigned to them by Firmware Configuration Files (if we're using
4797 * them) but need to be explicitly set if we're using hard-coded
4798 * initialization. But even in the case of using Firmware Configuration
4799 * Files, we'd like to expose the ability to change these via module
4800 * parameters so these are essentially common tweaks/settings for
4801 * Configuration Files and hard-coded initialization ...
4802 */
4803 static int adap_init0_tweaks(struct adapter *adapter)
4804 {
4805 /*
4806 * Fix up various Host-Dependent Parameters like Page Size, Cache
4807 * Line Size, etc. The firmware default is for a 4KB Page Size and
4808 * 64B Cache Line Size ...
4809 */
4810 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4811
4812 /*
4813 * Process module parameters which affect early initialization.
4814 */
4815 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4816 dev_err(&adapter->pdev->dev,
4817 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4818 rx_dma_offset);
4819 rx_dma_offset = 2;
4820 }
4821 t4_set_reg_field(adapter, SGE_CONTROL_A,
4822 PKTSHIFT_V(PKTSHIFT_M),
4823 PKTSHIFT_V(rx_dma_offset));
4824
4825 /*
4826 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4827 * adds the pseudo header itself.
4828 */
4829 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4830 CSUM_HAS_PSEUDO_HDR_F, 0);
4831
4832 return 0;
4833 }
4834
4835 /*
4836 * Attempt to initialize the adapter via a Firmware Configuration File.
4837 */
4838 static int adap_init0_config(struct adapter *adapter, int reset)
4839 {
4840 struct fw_caps_config_cmd caps_cmd;
4841 const struct firmware *cf;
4842 unsigned long mtype = 0, maddr = 0;
4843 u32 finiver, finicsum, cfcsum;
4844 int ret;
4845 int config_issued = 0;
4846 char *fw_config_file, fw_config_file_path[256];
4847 char *config_name = NULL;
4848
4849 /*
4850 * Reset device if necessary.
4851 */
4852 if (reset) {
4853 ret = t4_fw_reset(adapter, adapter->mbox,
4854 PIORSTMODE_F | PIORST_F);
4855 if (ret < 0)
4856 goto bye;
4857 }
4858
4859 /*
4860 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4861 * then use that. Otherwise, use the configuration file stored
4862 * in the adapter flash ...
4863 */
4864 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
4865 case CHELSIO_T4:
4866 fw_config_file = FW4_CFNAME;
4867 break;
4868 case CHELSIO_T5:
4869 fw_config_file = FW5_CFNAME;
4870 break;
4871 default:
4872 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4873 adapter->pdev->device);
4874 ret = -EINVAL;
4875 goto bye;
4876 }
4877
4878 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4879 if (ret < 0) {
4880 config_name = "On FLASH";
4881 mtype = FW_MEMTYPE_CF_FLASH;
4882 maddr = t4_flash_cfg_addr(adapter);
4883 } else {
4884 u32 params[7], val[7];
4885
4886 sprintf(fw_config_file_path,
4887 "/lib/firmware/%s", fw_config_file);
4888 config_name = fw_config_file_path;
4889
4890 if (cf->size >= FLASH_CFG_MAX_SIZE)
4891 ret = -ENOMEM;
4892 else {
4893 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4894 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
4895 ret = t4_query_params(adapter, adapter->mbox,
4896 adapter->fn, 0, 1, params, val);
4897 if (ret == 0) {
4898 /*
4899 * For t4_memory_rw() below addresses and
4900 * sizes have to be in terms of multiples of 4
4901 * bytes. So, if the Configuration File isn't
4902 * a multiple of 4 bytes in length we'll have
4903 * to write that out separately since we can't
4904 * guarantee that the bytes following the
4905 * residual byte in the buffer returned by
4906 * request_firmware() are zeroed out ...
4907 */
4908 size_t resid = cf->size & 0x3;
4909 size_t size = cf->size & ~0x3;
4910 __be32 *data = (__be32 *)cf->data;
4911
4912 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4913 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
4914
4915 spin_lock(&adapter->win0_lock);
4916 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4917 size, data, T4_MEMORY_WRITE);
4918 if (ret == 0 && resid != 0) {
4919 union {
4920 __be32 word;
4921 char buf[4];
4922 } last;
4923 int i;
4924
4925 last.word = data[size >> 2];
4926 for (i = resid; i < 4; i++)
4927 last.buf[i] = 0;
4928 ret = t4_memory_rw(adapter, 0, mtype,
4929 maddr + size,
4930 4, &last.word,
4931 T4_MEMORY_WRITE);
4932 }
4933 spin_unlock(&adapter->win0_lock);
4934 }
4935 }
4936
4937 release_firmware(cf);
4938 if (ret)
4939 goto bye;
4940 }
4941
4942 /*
4943 * Issue a Capability Configuration command to the firmware to get it
4944 * to parse the Configuration File. We don't use t4_fw_config_file()
4945 * because we want the ability to modify various features after we've
4946 * processed the configuration file ...
4947 */
4948 memset(&caps_cmd, 0, sizeof(caps_cmd));
4949 caps_cmd.op_to_write =
4950 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4951 FW_CMD_REQUEST_F |
4952 FW_CMD_READ_F);
4953 caps_cmd.cfvalid_to_len16 =
4954 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4955 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4956 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
4957 FW_LEN16(caps_cmd));
4958 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4959 &caps_cmd);
4960
4961 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4962 * Configuration File in FLASH), our last gasp effort is to use the
4963 * Firmware Configuration File which is embedded in the firmware. A
4964 * very few early versions of the firmware didn't have one embedded
4965 * but we can ignore those.
4966 */
4967 if (ret == -ENOENT) {
4968 memset(&caps_cmd, 0, sizeof(caps_cmd));
4969 caps_cmd.op_to_write =
4970 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4971 FW_CMD_REQUEST_F |
4972 FW_CMD_READ_F);
4973 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4974 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4975 sizeof(caps_cmd), &caps_cmd);
4976 config_name = "Firmware Default";
4977 }
4978
4979 config_issued = 1;
4980 if (ret < 0)
4981 goto bye;
4982
4983 finiver = ntohl(caps_cmd.finiver);
4984 finicsum = ntohl(caps_cmd.finicsum);
4985 cfcsum = ntohl(caps_cmd.cfcsum);
4986 if (finicsum != cfcsum)
4987 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4988 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4989 finicsum, cfcsum);
4990
4991 /*
4992 * And now tell the firmware to use the configuration we just loaded.
4993 */
4994 caps_cmd.op_to_write =
4995 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4996 FW_CMD_REQUEST_F |
4997 FW_CMD_WRITE_F);
4998 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4999 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5000 NULL);
5001 if (ret < 0)
5002 goto bye;
5003
5004 /*
5005 * Tweak configuration based on system architecture, module
5006 * parameters, etc.
5007 */
5008 ret = adap_init0_tweaks(adapter);
5009 if (ret < 0)
5010 goto bye;
5011
5012 /*
5013 * And finally tell the firmware to initialize itself using the
5014 * parameters from the Configuration File.
5015 */
5016 ret = t4_fw_initialize(adapter, adapter->mbox);
5017 if (ret < 0)
5018 goto bye;
5019
5020 /* Emit Firmware Configuration File information and return
5021 * successfully.
5022 */
5023 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
5024 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5025 config_name, finiver, cfcsum);
5026 return 0;
5027
5028 /*
5029 * Something bad happened. Return the error ... (If the "error"
5030 * is that there's no Configuration File on the adapter we don't
5031 * want to issue a warning since this is fairly common.)
5032 */
5033 bye:
5034 if (config_issued && ret != -ENOENT)
5035 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5036 config_name, -ret);
5037 return ret;
5038 }
5039
5040 static struct fw_info fw_info_array[] = {
5041 {
5042 .chip = CHELSIO_T4,
5043 .fs_name = FW4_CFNAME,
5044 .fw_mod_name = FW4_FNAME,
5045 .fw_hdr = {
5046 .chip = FW_HDR_CHIP_T4,
5047 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5048 .intfver_nic = FW_INTFVER(T4, NIC),
5049 .intfver_vnic = FW_INTFVER(T4, VNIC),
5050 .intfver_ri = FW_INTFVER(T4, RI),
5051 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5052 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5053 },
5054 }, {
5055 .chip = CHELSIO_T5,
5056 .fs_name = FW5_CFNAME,
5057 .fw_mod_name = FW5_FNAME,
5058 .fw_hdr = {
5059 .chip = FW_HDR_CHIP_T5,
5060 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5061 .intfver_nic = FW_INTFVER(T5, NIC),
5062 .intfver_vnic = FW_INTFVER(T5, VNIC),
5063 .intfver_ri = FW_INTFVER(T5, RI),
5064 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5065 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5066 },
5067 }
5068 };
5069
5070 static struct fw_info *find_fw_info(int chip)
5071 {
5072 int i;
5073
5074 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5075 if (fw_info_array[i].chip == chip)
5076 return &fw_info_array[i];
5077 }
5078 return NULL;
5079 }
5080
5081 /*
5082 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5083 */
5084 static int adap_init0(struct adapter *adap)
5085 {
5086 int ret;
5087 u32 v, port_vec;
5088 enum dev_state state;
5089 u32 params[7], val[7];
5090 struct fw_caps_config_cmd caps_cmd;
5091 struct fw_devlog_cmd devlog_cmd;
5092 u32 devlog_meminfo;
5093 int reset = 1;
5094
5095 /* Contact FW, advertising Master capability */
5096 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
5097 if (ret < 0) {
5098 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5099 ret);
5100 return ret;
5101 }
5102 if (ret == adap->mbox)
5103 adap->flags |= MASTER_PF;
5104
5105 /*
5106 * If we're the Master PF Driver and the device is uninitialized,
5107 * then let's consider upgrading the firmware ... (We always want
5108 * to check the firmware version number in order to A. get it for
5109 * later reporting and B. to warn if the currently loaded firmware
5110 * is excessively mismatched relative to the driver.)
5111 */
5112 t4_get_fw_version(adap, &adap->params.fw_vers);
5113 t4_get_tp_version(adap, &adap->params.tp_vers);
5114 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5115 struct fw_info *fw_info;
5116 struct fw_hdr *card_fw;
5117 const struct firmware *fw;
5118 const u8 *fw_data = NULL;
5119 unsigned int fw_size = 0;
5120
5121 /* This is the firmware whose headers the driver was compiled
5122 * against
5123 */
5124 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5125 if (fw_info == NULL) {
5126 dev_err(adap->pdev_dev,
5127 "unable to get firmware info for chip %d.\n",
5128 CHELSIO_CHIP_VERSION(adap->params.chip));
5129 return -EINVAL;
5130 }
5131
5132 /* allocate memory to read the header of the firmware on the
5133 * card
5134 */
5135 card_fw = t4_alloc_mem(sizeof(*card_fw));
5136
5137 /* Get FW from from /lib/firmware/ */
5138 ret = request_firmware(&fw, fw_info->fw_mod_name,
5139 adap->pdev_dev);
5140 if (ret < 0) {
5141 dev_err(adap->pdev_dev,
5142 "unable to load firmware image %s, error %d\n",
5143 fw_info->fw_mod_name, ret);
5144 } else {
5145 fw_data = fw->data;
5146 fw_size = fw->size;
5147 }
5148
5149 /* upgrade FW logic */
5150 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5151 state, &reset);
5152
5153 /* Cleaning up */
5154 release_firmware(fw);
5155 t4_free_mem(card_fw);
5156
5157 if (ret < 0)
5158 goto bye;
5159 }
5160
5161 /*
5162 * Grab VPD parameters. This should be done after we establish a
5163 * connection to the firmware since some of the VPD parameters
5164 * (notably the Core Clock frequency) are retrieved via requests to
5165 * the firmware. On the other hand, we need these fairly early on
5166 * so we do this right after getting ahold of the firmware.
5167 */
5168 ret = get_vpd_params(adap, &adap->params.vpd);
5169 if (ret < 0)
5170 goto bye;
5171
5172 /* Read firmware device log parameters. We really need to find a way
5173 * to get these parameters initialized with some default values (which
5174 * are likely to be correct) for the case where we either don't
5175 * attache to the firmware or it's crashed when we probe the adapter.
5176 * That way we'll still be able to perform early firmware startup
5177 * debugging ... If the request to get the Firmware's Device Log
5178 * parameters fails, we'll live so we don't make that a fatal error.
5179 */
5180 memset(&devlog_cmd, 0, sizeof(devlog_cmd));
5181 devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) |
5182 FW_CMD_REQUEST_F | FW_CMD_READ_F);
5183 devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd));
5184 ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
5185 &devlog_cmd);
5186 if (ret == 0) {
5187 devlog_meminfo =
5188 ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog);
5189 adap->params.devlog.memtype =
5190 FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
5191 adap->params.devlog.start =
5192 FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
5193 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog);
5194 }
5195
5196 /*
5197 * Find out what ports are available to us. Note that we need to do
5198 * this before calling adap_init0_no_config() since it needs nports
5199 * and portvec ...
5200 */
5201 v =
5202 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5203 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
5204 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5205 if (ret < 0)
5206 goto bye;
5207
5208 adap->params.nports = hweight32(port_vec);
5209 adap->params.portvec = port_vec;
5210
5211 /* If the firmware is initialized already, emit a simply note to that
5212 * effect. Otherwise, it's time to try initializing the adapter.
5213 */
5214 if (state == DEV_STATE_INIT) {
5215 dev_info(adap->pdev_dev, "Coming up as %s: "\
5216 "Adapter already initialized\n",
5217 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5218 } else {
5219 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5220 "Initializing adapter\n");
5221
5222 /* Find out whether we're dealing with a version of the
5223 * firmware which has configuration file support.
5224 */
5225 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5226 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5227 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5228 params, val);
5229
5230 /* If the firmware doesn't support Configuration Files,
5231 * return an error.
5232 */
5233 if (ret < 0) {
5234 dev_err(adap->pdev_dev, "firmware doesn't support "
5235 "Firmware Configuration Files\n");
5236 goto bye;
5237 }
5238
5239 /* The firmware provides us with a memory buffer where we can
5240 * load a Configuration File from the host if we want to
5241 * override the Configuration File in flash.
5242 */
5243 ret = adap_init0_config(adap, reset);
5244 if (ret == -ENOENT) {
5245 dev_err(adap->pdev_dev, "no Configuration File "
5246 "present on adapter.\n");
5247 goto bye;
5248 }
5249 if (ret < 0) {
5250 dev_err(adap->pdev_dev, "could not initialize "
5251 "adapter, error %d\n", -ret);
5252 goto bye;
5253 }
5254 }
5255
5256 /* Give the SGE code a chance to pull in anything that it needs ...
5257 * Note that this must be called after we retrieve our VPD parameters
5258 * in order to know how to convert core ticks to seconds, etc.
5259 */
5260 ret = t4_sge_init(adap);
5261 if (ret < 0)
5262 goto bye;
5263
5264 if (is_bypass_device(adap->pdev->device))
5265 adap->params.bypass = 1;
5266
5267 /*
5268 * Grab some of our basic fundamental operating parameters.
5269 */
5270 #define FW_PARAM_DEV(param) \
5271 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5272 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
5273
5274 #define FW_PARAM_PFVF(param) \
5275 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5276 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
5277 FW_PARAMS_PARAM_Y_V(0) | \
5278 FW_PARAMS_PARAM_Z_V(0)
5279
5280 params[0] = FW_PARAM_PFVF(EQ_START);
5281 params[1] = FW_PARAM_PFVF(L2T_START);
5282 params[2] = FW_PARAM_PFVF(L2T_END);
5283 params[3] = FW_PARAM_PFVF(FILTER_START);
5284 params[4] = FW_PARAM_PFVF(FILTER_END);
5285 params[5] = FW_PARAM_PFVF(IQFLINT_START);
5286 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5287 if (ret < 0)
5288 goto bye;
5289 adap->sge.egr_start = val[0];
5290 adap->l2t_start = val[1];
5291 adap->l2t_end = val[2];
5292 adap->tids.ftid_base = val[3];
5293 adap->tids.nftids = val[4] - val[3] + 1;
5294 adap->sge.ingr_start = val[5];
5295
5296 params[0] = FW_PARAM_PFVF(CLIP_START);
5297 params[1] = FW_PARAM_PFVF(CLIP_END);
5298 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5299 if (ret < 0)
5300 goto bye;
5301 adap->clipt_start = val[0];
5302 adap->clipt_end = val[1];
5303
5304 /* query params related to active filter region */
5305 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5306 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5307 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5308 /* If Active filter size is set we enable establishing
5309 * offload connection through firmware work request
5310 */
5311 if ((val[0] != val[1]) && (ret >= 0)) {
5312 adap->flags |= FW_OFLD_CONN;
5313 adap->tids.aftid_base = val[0];
5314 adap->tids.aftid_end = val[1];
5315 }
5316
5317 /* If we're running on newer firmware, let it know that we're
5318 * prepared to deal with encapsulated CPL messages. Older
5319 * firmware won't understand this and we'll just get
5320 * unencapsulated messages ...
5321 */
5322 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5323 val[0] = 1;
5324 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5325
5326 /*
5327 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5328 * capability. Earlier versions of the firmware didn't have the
5329 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5330 * permission to use ULPTX MEMWRITE DSGL.
5331 */
5332 if (is_t4(adap->params.chip)) {
5333 adap->params.ulptx_memwrite_dsgl = false;
5334 } else {
5335 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5336 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5337 1, params, val);
5338 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5339 }
5340
5341 /*
5342 * Get device capabilities so we can determine what resources we need
5343 * to manage.
5344 */
5345 memset(&caps_cmd, 0, sizeof(caps_cmd));
5346 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5347 FW_CMD_REQUEST_F | FW_CMD_READ_F);
5348 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5349 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5350 &caps_cmd);
5351 if (ret < 0)
5352 goto bye;
5353
5354 if (caps_cmd.ofldcaps) {
5355 /* query offload-related parameters */
5356 params[0] = FW_PARAM_DEV(NTID);
5357 params[1] = FW_PARAM_PFVF(SERVER_START);
5358 params[2] = FW_PARAM_PFVF(SERVER_END);
5359 params[3] = FW_PARAM_PFVF(TDDP_START);
5360 params[4] = FW_PARAM_PFVF(TDDP_END);
5361 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5362 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5363 params, val);
5364 if (ret < 0)
5365 goto bye;
5366 adap->tids.ntids = val[0];
5367 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5368 adap->tids.stid_base = val[1];
5369 adap->tids.nstids = val[2] - val[1] + 1;
5370 /*
5371 * Setup server filter region. Divide the availble filter
5372 * region into two parts. Regular filters get 1/3rd and server
5373 * filters get 2/3rd part. This is only enabled if workarond
5374 * path is enabled.
5375 * 1. For regular filters.
5376 * 2. Server filter: This are special filters which are used
5377 * to redirect SYN packets to offload queue.
5378 */
5379 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5380 adap->tids.sftid_base = adap->tids.ftid_base +
5381 DIV_ROUND_UP(adap->tids.nftids, 3);
5382 adap->tids.nsftids = adap->tids.nftids -
5383 DIV_ROUND_UP(adap->tids.nftids, 3);
5384 adap->tids.nftids = adap->tids.sftid_base -
5385 adap->tids.ftid_base;
5386 }
5387 adap->vres.ddp.start = val[3];
5388 adap->vres.ddp.size = val[4] - val[3] + 1;
5389 adap->params.ofldq_wr_cred = val[5];
5390
5391 adap->params.offload = 1;
5392 }
5393 if (caps_cmd.rdmacaps) {
5394 params[0] = FW_PARAM_PFVF(STAG_START);
5395 params[1] = FW_PARAM_PFVF(STAG_END);
5396 params[2] = FW_PARAM_PFVF(RQ_START);
5397 params[3] = FW_PARAM_PFVF(RQ_END);
5398 params[4] = FW_PARAM_PFVF(PBL_START);
5399 params[5] = FW_PARAM_PFVF(PBL_END);
5400 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5401 params, val);
5402 if (ret < 0)
5403 goto bye;
5404 adap->vres.stag.start = val[0];
5405 adap->vres.stag.size = val[1] - val[0] + 1;
5406 adap->vres.rq.start = val[2];
5407 adap->vres.rq.size = val[3] - val[2] + 1;
5408 adap->vres.pbl.start = val[4];
5409 adap->vres.pbl.size = val[5] - val[4] + 1;
5410
5411 params[0] = FW_PARAM_PFVF(SQRQ_START);
5412 params[1] = FW_PARAM_PFVF(SQRQ_END);
5413 params[2] = FW_PARAM_PFVF(CQ_START);
5414 params[3] = FW_PARAM_PFVF(CQ_END);
5415 params[4] = FW_PARAM_PFVF(OCQ_START);
5416 params[5] = FW_PARAM_PFVF(OCQ_END);
5417 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5418 val);
5419 if (ret < 0)
5420 goto bye;
5421 adap->vres.qp.start = val[0];
5422 adap->vres.qp.size = val[1] - val[0] + 1;
5423 adap->vres.cq.start = val[2];
5424 adap->vres.cq.size = val[3] - val[2] + 1;
5425 adap->vres.ocq.start = val[4];
5426 adap->vres.ocq.size = val[5] - val[4] + 1;
5427
5428 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5429 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5430 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5431 val);
5432 if (ret < 0) {
5433 adap->params.max_ordird_qp = 8;
5434 adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5435 ret = 0;
5436 } else {
5437 adap->params.max_ordird_qp = val[0];
5438 adap->params.max_ird_adapter = val[1];
5439 }
5440 dev_info(adap->pdev_dev,
5441 "max_ordird_qp %d max_ird_adapter %d\n",
5442 adap->params.max_ordird_qp,
5443 adap->params.max_ird_adapter);
5444 }
5445 if (caps_cmd.iscsicaps) {
5446 params[0] = FW_PARAM_PFVF(ISCSI_START);
5447 params[1] = FW_PARAM_PFVF(ISCSI_END);
5448 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5449 params, val);
5450 if (ret < 0)
5451 goto bye;
5452 adap->vres.iscsi.start = val[0];
5453 adap->vres.iscsi.size = val[1] - val[0] + 1;
5454 }
5455 #undef FW_PARAM_PFVF
5456 #undef FW_PARAM_DEV
5457
5458 /* The MTU/MSS Table is initialized by now, so load their values. If
5459 * we're initializing the adapter, then we'll make any modifications
5460 * we want to the MTU/MSS Table and also initialize the congestion
5461 * parameters.
5462 */
5463 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5464 if (state != DEV_STATE_INIT) {
5465 int i;
5466
5467 /* The default MTU Table contains values 1492 and 1500.
5468 * However, for TCP, it's better to have two values which are
5469 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5470 * This allows us to have a TCP Data Payload which is a
5471 * multiple of 8 regardless of what combination of TCP Options
5472 * are in use (always a multiple of 4 bytes) which is
5473 * important for performance reasons. For instance, if no
5474 * options are in use, then we have a 20-byte IP header and a
5475 * 20-byte TCP header. In this case, a 1500-byte MSS would
5476 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5477 * which is not a multiple of 8. So using an MSS of 1488 in
5478 * this case results in a TCP Data Payload of 1448 bytes which
5479 * is a multiple of 8. On the other hand, if 12-byte TCP Time
5480 * Stamps have been negotiated, then an MTU of 1500 bytes
5481 * results in a TCP Data Payload of 1448 bytes which, as
5482 * above, is a multiple of 8 bytes ...
5483 */
5484 for (i = 0; i < NMTUS; i++)
5485 if (adap->params.mtus[i] == 1492) {
5486 adap->params.mtus[i] = 1488;
5487 break;
5488 }
5489
5490 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5491 adap->params.b_wnd);
5492 }
5493 t4_init_sge_params(adap);
5494 t4_init_tp_params(adap);
5495 adap->flags |= FW_OK;
5496 return 0;
5497
5498 /*
5499 * Something bad happened. If a command timed out or failed with EIO
5500 * FW does not operate within its spec or something catastrophic
5501 * happened to HW/FW, stop issuing commands.
5502 */
5503 bye:
5504 if (ret != -ETIMEDOUT && ret != -EIO)
5505 t4_fw_bye(adap, adap->mbox);
5506 return ret;
5507 }
5508
5509 /* EEH callbacks */
5510
5511 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5512 pci_channel_state_t state)
5513 {
5514 int i;
5515 struct adapter *adap = pci_get_drvdata(pdev);
5516
5517 if (!adap)
5518 goto out;
5519
5520 rtnl_lock();
5521 adap->flags &= ~FW_OK;
5522 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5523 spin_lock(&adap->stats_lock);
5524 for_each_port(adap, i) {
5525 struct net_device *dev = adap->port[i];
5526
5527 netif_device_detach(dev);
5528 netif_carrier_off(dev);
5529 }
5530 spin_unlock(&adap->stats_lock);
5531 if (adap->flags & FULL_INIT_DONE)
5532 cxgb_down(adap);
5533 rtnl_unlock();
5534 if ((adap->flags & DEV_ENABLED)) {
5535 pci_disable_device(pdev);
5536 adap->flags &= ~DEV_ENABLED;
5537 }
5538 out: return state == pci_channel_io_perm_failure ?
5539 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5540 }
5541
5542 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5543 {
5544 int i, ret;
5545 struct fw_caps_config_cmd c;
5546 struct adapter *adap = pci_get_drvdata(pdev);
5547
5548 if (!adap) {
5549 pci_restore_state(pdev);
5550 pci_save_state(pdev);
5551 return PCI_ERS_RESULT_RECOVERED;
5552 }
5553
5554 if (!(adap->flags & DEV_ENABLED)) {
5555 if (pci_enable_device(pdev)) {
5556 dev_err(&pdev->dev, "Cannot reenable PCI "
5557 "device after reset\n");
5558 return PCI_ERS_RESULT_DISCONNECT;
5559 }
5560 adap->flags |= DEV_ENABLED;
5561 }
5562
5563 pci_set_master(pdev);
5564 pci_restore_state(pdev);
5565 pci_save_state(pdev);
5566 pci_cleanup_aer_uncorrect_error_status(pdev);
5567
5568 if (t4_wait_dev_ready(adap->regs) < 0)
5569 return PCI_ERS_RESULT_DISCONNECT;
5570 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5571 return PCI_ERS_RESULT_DISCONNECT;
5572 adap->flags |= FW_OK;
5573 if (adap_init1(adap, &c))
5574 return PCI_ERS_RESULT_DISCONNECT;
5575
5576 for_each_port(adap, i) {
5577 struct port_info *p = adap2pinfo(adap, i);
5578
5579 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5580 NULL, NULL);
5581 if (ret < 0)
5582 return PCI_ERS_RESULT_DISCONNECT;
5583 p->viid = ret;
5584 p->xact_addr_filt = -1;
5585 }
5586
5587 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5588 adap->params.b_wnd);
5589 setup_memwin(adap);
5590 if (cxgb_up(adap))
5591 return PCI_ERS_RESULT_DISCONNECT;
5592 return PCI_ERS_RESULT_RECOVERED;
5593 }
5594
5595 static void eeh_resume(struct pci_dev *pdev)
5596 {
5597 int i;
5598 struct adapter *adap = pci_get_drvdata(pdev);
5599
5600 if (!adap)
5601 return;
5602
5603 rtnl_lock();
5604 for_each_port(adap, i) {
5605 struct net_device *dev = adap->port[i];
5606
5607 if (netif_running(dev)) {
5608 link_start(dev);
5609 cxgb_set_rxmode(dev);
5610 }
5611 netif_device_attach(dev);
5612 }
5613 rtnl_unlock();
5614 }
5615
5616 static const struct pci_error_handlers cxgb4_eeh = {
5617 .error_detected = eeh_err_detected,
5618 .slot_reset = eeh_slot_reset,
5619 .resume = eeh_resume,
5620 };
5621
5622 static inline bool is_x_10g_port(const struct link_config *lc)
5623 {
5624 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5625 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
5626 }
5627
5628 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5629 unsigned int us, unsigned int cnt,
5630 unsigned int size, unsigned int iqe_size)
5631 {
5632 q->adap = adap;
5633 set_rspq_intr_params(q, us, cnt);
5634 q->iqe_len = iqe_size;
5635 q->size = size;
5636 }
5637
5638 /*
5639 * Perform default configuration of DMA queues depending on the number and type
5640 * of ports we found and the number of available CPUs. Most settings can be
5641 * modified by the admin prior to actual use.
5642 */
5643 static void cfg_queues(struct adapter *adap)
5644 {
5645 struct sge *s = &adap->sge;
5646 int i, n10g = 0, qidx = 0;
5647 #ifndef CONFIG_CHELSIO_T4_DCB
5648 int q10g = 0;
5649 #endif
5650 int ciq_size;
5651
5652 for_each_port(adap, i)
5653 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
5654 #ifdef CONFIG_CHELSIO_T4_DCB
5655 /* For Data Center Bridging support we need to be able to support up
5656 * to 8 Traffic Priorities; each of which will be assigned to its
5657 * own TX Queue in order to prevent Head-Of-Line Blocking.
5658 */
5659 if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5660 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5661 MAX_ETH_QSETS, adap->params.nports * 8);
5662 BUG_ON(1);
5663 }
5664
5665 for_each_port(adap, i) {
5666 struct port_info *pi = adap2pinfo(adap, i);
5667
5668 pi->first_qset = qidx;
5669 pi->nqsets = 8;
5670 qidx += pi->nqsets;
5671 }
5672 #else /* !CONFIG_CHELSIO_T4_DCB */
5673 /*
5674 * We default to 1 queue per non-10G port and up to # of cores queues
5675 * per 10G port.
5676 */
5677 if (n10g)
5678 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5679 if (q10g > netif_get_num_default_rss_queues())
5680 q10g = netif_get_num_default_rss_queues();
5681
5682 for_each_port(adap, i) {
5683 struct port_info *pi = adap2pinfo(adap, i);
5684
5685 pi->first_qset = qidx;
5686 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
5687 qidx += pi->nqsets;
5688 }
5689 #endif /* !CONFIG_CHELSIO_T4_DCB */
5690
5691 s->ethqsets = qidx;
5692 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5693
5694 if (is_offload(adap)) {
5695 /*
5696 * For offload we use 1 queue/channel if all ports are up to 1G,
5697 * otherwise we divide all available queues amongst the channels
5698 * capped by the number of available cores.
5699 */
5700 if (n10g) {
5701 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5702 num_online_cpus());
5703 s->ofldqsets = roundup(i, adap->params.nports);
5704 } else
5705 s->ofldqsets = adap->params.nports;
5706 /* For RDMA one Rx queue per channel suffices */
5707 s->rdmaqs = adap->params.nports;
5708 s->rdmaciqs = adap->params.nports;
5709 }
5710
5711 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5712 struct sge_eth_rxq *r = &s->ethrxq[i];
5713
5714 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
5715 r->fl.size = 72;
5716 }
5717
5718 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5719 s->ethtxq[i].q.size = 1024;
5720
5721 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5722 s->ctrlq[i].q.size = 512;
5723
5724 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5725 s->ofldtxq[i].q.size = 1024;
5726
5727 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5728 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5729
5730 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
5731 r->rspq.uld = CXGB4_ULD_ISCSI;
5732 r->fl.size = 72;
5733 }
5734
5735 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5736 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5737
5738 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
5739 r->rspq.uld = CXGB4_ULD_RDMA;
5740 r->fl.size = 72;
5741 }
5742
5743 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5744 if (ciq_size > SGE_MAX_IQ_SIZE) {
5745 CH_WARN(adap, "CIQ size too small for available IQs\n");
5746 ciq_size = SGE_MAX_IQ_SIZE;
5747 }
5748
5749 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5750 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5751
5752 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
5753 r->rspq.uld = CXGB4_ULD_RDMA;
5754 }
5755
5756 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5757 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
5758 }
5759
5760 /*
5761 * Reduce the number of Ethernet queues across all ports to at most n.
5762 * n provides at least one queue per port.
5763 */
5764 static void reduce_ethqs(struct adapter *adap, int n)
5765 {
5766 int i;
5767 struct port_info *pi;
5768
5769 while (n < adap->sge.ethqsets)
5770 for_each_port(adap, i) {
5771 pi = adap2pinfo(adap, i);
5772 if (pi->nqsets > 1) {
5773 pi->nqsets--;
5774 adap->sge.ethqsets--;
5775 if (adap->sge.ethqsets <= n)
5776 break;
5777 }
5778 }
5779
5780 n = 0;
5781 for_each_port(adap, i) {
5782 pi = adap2pinfo(adap, i);
5783 pi->first_qset = n;
5784 n += pi->nqsets;
5785 }
5786 }
5787
5788 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5789 #define EXTRA_VECS 2
5790
5791 static int enable_msix(struct adapter *adap)
5792 {
5793 int ofld_need = 0;
5794 int i, want, need;
5795 struct sge *s = &adap->sge;
5796 unsigned int nchan = adap->params.nports;
5797 struct msix_entry entries[MAX_INGQ + 1];
5798
5799 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5800 entries[i].entry = i;
5801
5802 want = s->max_ethqsets + EXTRA_VECS;
5803 if (is_offload(adap)) {
5804 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
5805 /* need nchan for each possible ULD */
5806 ofld_need = 3 * nchan;
5807 }
5808 #ifdef CONFIG_CHELSIO_T4_DCB
5809 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5810 * each port.
5811 */
5812 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5813 #else
5814 need = adap->params.nports + EXTRA_VECS + ofld_need;
5815 #endif
5816 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5817 if (want < 0)
5818 return want;
5819
5820 /*
5821 * Distribute available vectors to the various queue groups.
5822 * Every group gets its minimum requirement and NIC gets top
5823 * priority for leftovers.
5824 */
5825 i = want - EXTRA_VECS - ofld_need;
5826 if (i < s->max_ethqsets) {
5827 s->max_ethqsets = i;
5828 if (i < s->ethqsets)
5829 reduce_ethqs(adap, i);
5830 }
5831 if (is_offload(adap)) {
5832 i = want - EXTRA_VECS - s->max_ethqsets;
5833 i -= ofld_need - nchan;
5834 s->ofldqsets = (i / nchan) * nchan; /* round down */
5835 }
5836 for (i = 0; i < want; ++i)
5837 adap->msix_info[i].vec = entries[i].vector;
5838
5839 return 0;
5840 }
5841
5842 #undef EXTRA_VECS
5843
5844 static int init_rss(struct adapter *adap)
5845 {
5846 unsigned int i, j;
5847
5848 for_each_port(adap, i) {
5849 struct port_info *pi = adap2pinfo(adap, i);
5850
5851 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5852 if (!pi->rss)
5853 return -ENOMEM;
5854 for (j = 0; j < pi->rss_size; j++)
5855 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5856 }
5857 return 0;
5858 }
5859
5860 static void print_port_info(const struct net_device *dev)
5861 {
5862 char buf[80];
5863 char *bufp = buf;
5864 const char *spd = "";
5865 const struct port_info *pi = netdev_priv(dev);
5866 const struct adapter *adap = pi->adapter;
5867
5868 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5869 spd = " 2.5 GT/s";
5870 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5871 spd = " 5 GT/s";
5872 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5873 spd = " 8 GT/s";
5874
5875 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5876 bufp += sprintf(bufp, "100/");
5877 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5878 bufp += sprintf(bufp, "1000/");
5879 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5880 bufp += sprintf(bufp, "10G/");
5881 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5882 bufp += sprintf(bufp, "40G/");
5883 if (bufp != buf)
5884 --bufp;
5885 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5886
5887 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5888 adap->params.vpd.id,
5889 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
5890 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5891 (adap->flags & USING_MSIX) ? " MSI-X" :
5892 (adap->flags & USING_MSI) ? " MSI" : "");
5893 netdev_info(dev, "S/N: %s, P/N: %s\n",
5894 adap->params.vpd.sn, adap->params.vpd.pn);
5895 }
5896
5897 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5898 {
5899 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5900 }
5901
5902 /*
5903 * Free the following resources:
5904 * - memory used for tables
5905 * - MSI/MSI-X
5906 * - net devices
5907 * - resources FW is holding for us
5908 */
5909 static void free_some_resources(struct adapter *adapter)
5910 {
5911 unsigned int i;
5912
5913 t4_free_mem(adapter->l2t);
5914 t4_free_mem(adapter->tids.tid_tab);
5915 disable_msi(adapter);
5916
5917 for_each_port(adapter, i)
5918 if (adapter->port[i]) {
5919 kfree(adap2pinfo(adapter, i)->rss);
5920 free_netdev(adapter->port[i]);
5921 }
5922 if (adapter->flags & FW_OK)
5923 t4_fw_bye(adapter, adapter->fn);
5924 }
5925
5926 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5927 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5928 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5929 #define SEGMENT_SIZE 128
5930
5931 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5932 {
5933 int func, i, err, s_qpp, qpp, num_seg;
5934 struct port_info *pi;
5935 bool highdma = false;
5936 struct adapter *adapter = NULL;
5937 void __iomem *regs;
5938
5939 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5940
5941 err = pci_request_regions(pdev, KBUILD_MODNAME);
5942 if (err) {
5943 /* Just info, some other driver may have claimed the device. */
5944 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5945 return err;
5946 }
5947
5948 err = pci_enable_device(pdev);
5949 if (err) {
5950 dev_err(&pdev->dev, "cannot enable PCI device\n");
5951 goto out_release_regions;
5952 }
5953
5954 regs = pci_ioremap_bar(pdev, 0);
5955 if (!regs) {
5956 dev_err(&pdev->dev, "cannot map device registers\n");
5957 err = -ENOMEM;
5958 goto out_disable_device;
5959 }
5960
5961 err = t4_wait_dev_ready(regs);
5962 if (err < 0)
5963 goto out_unmap_bar0;
5964
5965 /* We control everything through one PF */
5966 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
5967 if (func != ent->driver_data) {
5968 iounmap(regs);
5969 pci_disable_device(pdev);
5970 pci_save_state(pdev); /* to restore SR-IOV later */
5971 goto sriov;
5972 }
5973
5974 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
5975 highdma = true;
5976 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5977 if (err) {
5978 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5979 "coherent allocations\n");
5980 goto out_unmap_bar0;
5981 }
5982 } else {
5983 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5984 if (err) {
5985 dev_err(&pdev->dev, "no usable DMA configuration\n");
5986 goto out_unmap_bar0;
5987 }
5988 }
5989
5990 pci_enable_pcie_error_reporting(pdev);
5991 enable_pcie_relaxed_ordering(pdev);
5992 pci_set_master(pdev);
5993 pci_save_state(pdev);
5994
5995 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5996 if (!adapter) {
5997 err = -ENOMEM;
5998 goto out_unmap_bar0;
5999 }
6000
6001 adapter->workq = create_singlethread_workqueue("cxgb4");
6002 if (!adapter->workq) {
6003 err = -ENOMEM;
6004 goto out_free_adapter;
6005 }
6006
6007 /* PCI device has been enabled */
6008 adapter->flags |= DEV_ENABLED;
6009
6010 adapter->regs = regs;
6011 adapter->pdev = pdev;
6012 adapter->pdev_dev = &pdev->dev;
6013 adapter->mbox = func;
6014 adapter->fn = func;
6015 adapter->msg_enable = dflt_msg_enable;
6016 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6017
6018 spin_lock_init(&adapter->stats_lock);
6019 spin_lock_init(&adapter->tid_release_lock);
6020 spin_lock_init(&adapter->win0_lock);
6021
6022 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6023 INIT_WORK(&adapter->db_full_task, process_db_full);
6024 INIT_WORK(&adapter->db_drop_task, process_db_drop);
6025
6026 err = t4_prep_adapter(adapter);
6027 if (err)
6028 goto out_free_adapter;
6029
6030
6031 if (!is_t4(adapter->params.chip)) {
6032 s_qpp = (QUEUESPERPAGEPF0_S +
6033 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6034 adapter->fn);
6035 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6036 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
6037 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6038
6039 /* Each segment size is 128B. Write coalescing is enabled only
6040 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6041 * queue is less no of segments that can be accommodated in
6042 * a page size.
6043 */
6044 if (qpp > num_seg) {
6045 dev_err(&pdev->dev,
6046 "Incorrect number of egress queues per page\n");
6047 err = -EINVAL;
6048 goto out_free_adapter;
6049 }
6050 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6051 pci_resource_len(pdev, 2));
6052 if (!adapter->bar2) {
6053 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6054 err = -ENOMEM;
6055 goto out_free_adapter;
6056 }
6057 }
6058
6059 setup_memwin(adapter);
6060 err = adap_init0(adapter);
6061 setup_memwin_rdma(adapter);
6062 if (err)
6063 goto out_unmap_bar;
6064
6065 for_each_port(adapter, i) {
6066 struct net_device *netdev;
6067
6068 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6069 MAX_ETH_QSETS);
6070 if (!netdev) {
6071 err = -ENOMEM;
6072 goto out_free_dev;
6073 }
6074
6075 SET_NETDEV_DEV(netdev, &pdev->dev);
6076
6077 adapter->port[i] = netdev;
6078 pi = netdev_priv(netdev);
6079 pi->adapter = adapter;
6080 pi->xact_addr_filt = -1;
6081 pi->port_id = i;
6082 netdev->irq = pdev->irq;
6083
6084 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6085 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6086 NETIF_F_RXCSUM | NETIF_F_RXHASH |
6087 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6088 if (highdma)
6089 netdev->hw_features |= NETIF_F_HIGHDMA;
6090 netdev->features |= netdev->hw_features;
6091 netdev->vlan_features = netdev->features & VLAN_FEAT;
6092
6093 netdev->priv_flags |= IFF_UNICAST_FLT;
6094
6095 netdev->netdev_ops = &cxgb4_netdev_ops;
6096 #ifdef CONFIG_CHELSIO_T4_DCB
6097 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6098 cxgb4_dcb_state_init(netdev);
6099 #endif
6100 netdev->ethtool_ops = &cxgb_ethtool_ops;
6101 }
6102
6103 pci_set_drvdata(pdev, adapter);
6104
6105 if (adapter->flags & FW_OK) {
6106 err = t4_port_init(adapter, func, func, 0);
6107 if (err)
6108 goto out_free_dev;
6109 }
6110
6111 /*
6112 * Configure queues and allocate tables now, they can be needed as
6113 * soon as the first register_netdev completes.
6114 */
6115 cfg_queues(adapter);
6116
6117 adapter->l2t = t4_init_l2t();
6118 if (!adapter->l2t) {
6119 /* We tolerate a lack of L2T, giving up some functionality */
6120 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6121 adapter->params.offload = 0;
6122 }
6123
6124 #if IS_ENABLED(CONFIG_IPV6)
6125 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6126 adapter->clipt_end);
6127 if (!adapter->clipt) {
6128 /* We tolerate a lack of clip_table, giving up
6129 * some functionality
6130 */
6131 dev_warn(&pdev->dev,
6132 "could not allocate Clip table, continuing\n");
6133 adapter->params.offload = 0;
6134 }
6135 #endif
6136 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6137 dev_warn(&pdev->dev, "could not allocate TID table, "
6138 "continuing\n");
6139 adapter->params.offload = 0;
6140 }
6141
6142 /* See what interrupts we'll be using */
6143 if (msi > 1 && enable_msix(adapter) == 0)
6144 adapter->flags |= USING_MSIX;
6145 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6146 adapter->flags |= USING_MSI;
6147
6148 err = init_rss(adapter);
6149 if (err)
6150 goto out_free_dev;
6151
6152 /*
6153 * The card is now ready to go. If any errors occur during device
6154 * registration we do not fail the whole card but rather proceed only
6155 * with the ports we manage to register successfully. However we must
6156 * register at least one net device.
6157 */
6158 for_each_port(adapter, i) {
6159 pi = adap2pinfo(adapter, i);
6160 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6161 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6162
6163 err = register_netdev(adapter->port[i]);
6164 if (err)
6165 break;
6166 adapter->chan_map[pi->tx_chan] = i;
6167 print_port_info(adapter->port[i]);
6168 }
6169 if (i == 0) {
6170 dev_err(&pdev->dev, "could not register any net devices\n");
6171 goto out_free_dev;
6172 }
6173 if (err) {
6174 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6175 err = 0;
6176 }
6177
6178 if (cxgb4_debugfs_root) {
6179 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6180 cxgb4_debugfs_root);
6181 setup_debugfs(adapter);
6182 }
6183
6184 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6185 pdev->needs_freset = 1;
6186
6187 if (is_offload(adapter))
6188 attach_ulds(adapter);
6189
6190 sriov:
6191 #ifdef CONFIG_PCI_IOV
6192 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6193 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6194 dev_info(&pdev->dev,
6195 "instantiated %u virtual functions\n",
6196 num_vf[func]);
6197 #endif
6198 return 0;
6199
6200 out_free_dev:
6201 free_some_resources(adapter);
6202 out_unmap_bar:
6203 if (!is_t4(adapter->params.chip))
6204 iounmap(adapter->bar2);
6205 out_free_adapter:
6206 if (adapter->workq)
6207 destroy_workqueue(adapter->workq);
6208
6209 kfree(adapter);
6210 out_unmap_bar0:
6211 iounmap(regs);
6212 out_disable_device:
6213 pci_disable_pcie_error_reporting(pdev);
6214 pci_disable_device(pdev);
6215 out_release_regions:
6216 pci_release_regions(pdev);
6217 return err;
6218 }
6219
6220 static void remove_one(struct pci_dev *pdev)
6221 {
6222 struct adapter *adapter = pci_get_drvdata(pdev);
6223
6224 #ifdef CONFIG_PCI_IOV
6225 pci_disable_sriov(pdev);
6226
6227 #endif
6228
6229 if (adapter) {
6230 int i;
6231
6232 /* Tear down per-adapter Work Queue first since it can contain
6233 * references to our adapter data structure.
6234 */
6235 destroy_workqueue(adapter->workq);
6236
6237 if (is_offload(adapter))
6238 detach_ulds(adapter);
6239
6240 for_each_port(adapter, i)
6241 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6242 unregister_netdev(adapter->port[i]);
6243
6244 debugfs_remove_recursive(adapter->debugfs_root);
6245
6246 /* If we allocated filters, free up state associated with any
6247 * valid filters ...
6248 */
6249 if (adapter->tids.ftid_tab) {
6250 struct filter_entry *f = &adapter->tids.ftid_tab[0];
6251 for (i = 0; i < (adapter->tids.nftids +
6252 adapter->tids.nsftids); i++, f++)
6253 if (f->valid)
6254 clear_filter(adapter, f);
6255 }
6256
6257 if (adapter->flags & FULL_INIT_DONE)
6258 cxgb_down(adapter);
6259
6260 free_some_resources(adapter);
6261 #if IS_ENABLED(CONFIG_IPV6)
6262 t4_cleanup_clip_tbl(adapter);
6263 #endif
6264 iounmap(adapter->regs);
6265 if (!is_t4(adapter->params.chip))
6266 iounmap(adapter->bar2);
6267 pci_disable_pcie_error_reporting(pdev);
6268 if ((adapter->flags & DEV_ENABLED)) {
6269 pci_disable_device(pdev);
6270 adapter->flags &= ~DEV_ENABLED;
6271 }
6272 pci_release_regions(pdev);
6273 synchronize_rcu();
6274 kfree(adapter);
6275 } else
6276 pci_release_regions(pdev);
6277 }
6278
6279 static struct pci_driver cxgb4_driver = {
6280 .name = KBUILD_MODNAME,
6281 .id_table = cxgb4_pci_tbl,
6282 .probe = init_one,
6283 .remove = remove_one,
6284 .shutdown = remove_one,
6285 .err_handler = &cxgb4_eeh,
6286 };
6287
6288 static int __init cxgb4_init_module(void)
6289 {
6290 int ret;
6291
6292 /* Debugfs support is optional, just warn if this fails */
6293 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6294 if (!cxgb4_debugfs_root)
6295 pr_warn("could not create debugfs entry, continuing\n");
6296
6297 ret = pci_register_driver(&cxgb4_driver);
6298 if (ret < 0)
6299 debugfs_remove(cxgb4_debugfs_root);
6300
6301 #if IS_ENABLED(CONFIG_IPV6)
6302 if (!inet6addr_registered) {
6303 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6304 inet6addr_registered = true;
6305 }
6306 #endif
6307
6308 return ret;
6309 }
6310
6311 static void __exit cxgb4_cleanup_module(void)
6312 {
6313 #if IS_ENABLED(CONFIG_IPV6)
6314 if (inet6addr_registered) {
6315 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6316 inet6addr_registered = false;
6317 }
6318 #endif
6319 pci_unregister_driver(&cxgb4_driver);
6320 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
6321 }
6322
6323 module_init(cxgb4_init_module);
6324 module_exit(cxgb4_cleanup_module);
Linux kernel - Deliverables
This page took 0.161155 seconds and 6 git commands to generate.