projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tools: bpf_dbg: various misc code cleanups
[deliverable/linux.git] / drivers / net / ethernet / chelsio / cxgb4 / cxgb4_main.c
CommitLineData
b8ff05a9
DM		 1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
01789349 44#include <linux/if.h>
b8ff05a9
DM		 45#include <linux/if_vlan.h>
46#include <linux/init.h>
47#include <linux/log2.h>
48#include <linux/mdio.h>
49#include <linux/module.h>
50#include <linux/moduleparam.h>
51#include <linux/mutex.h>
52#include <linux/netdevice.h>
53#include <linux/pci.h>
54#include <linux/aer.h>
55#include <linux/rtnetlink.h>
56#include <linux/sched.h>
57#include <linux/seq_file.h>
58#include <linux/sockios.h>
59#include <linux/vmalloc.h>
60#include <linux/workqueue.h>
61#include <net/neighbour.h>
62#include <net/netevent.h>
01bcca68 63#include <net/addrconf.h>
b8ff05a9
DM		 64#include <asm/uaccess.h>
65
66#include "cxgb4.h"
67#include "t4_regs.h"
68#include "t4_msg.h"
69#include "t4fw_api.h"
70#include "l2t.h"
71
01bcca68
VP		 72#include <../drivers/net/bonding/bonding.h>
73
74#ifdef DRV_VERSION
75#undef DRV_VERSION
76#endif
3a7f8554
SR		 77#define DRV_VERSION "2.0.0-ko"
78#define DRV_DESC "Chelsio T4/T5 Network Driver"
b8ff05a9
DM		 79
80/*
81 * Max interrupt hold-off timer value in us. Queues fall back to this value
82 * under extreme memory pressure so it's largish to give the system time to
83 * recover.
84 */
85#define MAX_SGE_TIMERVAL 200U
86
7ee9ff94 87enum {
13ee15d3
VP		 88 /*
89 * Physical Function provisioning constants.
90 */
91 PFRES_NVI = 4, /* # of Virtual Interfaces */
92 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */
93 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr
94 */
95 PFRES_NEQ = 256, /* # of egress queues */
96 PFRES_NIQ = 0, /* # of ingress queues */
97 PFRES_TC = 0, /* PCI-E traffic class */
98 PFRES_NEXACTF = 128, /* # of exact MPS filters */
99
100 PFRES_R_CAPS = FW_CMD_CAP_PF,
101 PFRES_WX_CAPS = FW_CMD_CAP_PF,
102
103#ifdef CONFIG_PCI_IOV
104 /*
105 * Virtual Function provisioning constants. We need two extra Ingress
106 * Queues with Interrupt capability to serve as the VF's Firmware
107 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
108 * neither will have Free Lists associated with them). For each
109 * Ethernet/Control Egress Queue and for each Free List, we need an
110 * Egress Context.
111 */
7ee9ff94
CL		 112 VFRES_NPORTS = 1, /* # of "ports" per VF */
113 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
114
115 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
116 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
117 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
7ee9ff94 118 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
13ee15d3 119 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
7ee9ff94
CL		 120 VFRES_TC = 0, /* PCI-E traffic class */
121 VFRES_NEXACTF = 16, /* # of exact MPS filters */
122
123 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
124 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
13ee15d3 125#endif
7ee9ff94
CL		 126};
127
128/*
129 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
130 * static and likely not to be useful in the long run. We really need to
131 * implement some form of persistent configuration which the firmware
132 * controls.
133 */
134static unsigned int pfvfres_pmask(struct adapter *adapter,
135 unsigned int pf, unsigned int vf)
136{
137 unsigned int portn, portvec;
138
139 /*
140 * Give PF's access to all of the ports.
141 */
142 if (vf == 0)
143 return FW_PFVF_CMD_PMASK_MASK;
144
145 /*
146 * For VFs, we'll assign them access to the ports based purely on the
147 * PF. We assign active ports in order, wrapping around if there are
148 * fewer active ports than PFs: e.g. active port[pf % nports].
149 * Unfortunately the adapter's port_info structs haven't been
150 * initialized yet so we have to compute this.
151 */
152 if (adapter->params.nports == 0)
153 return 0;
154
155 portn = pf % adapter->params.nports;
156 portvec = adapter->params.portvec;
157 for (;;) {
158 /*
159 * Isolate the lowest set bit in the port vector. If we're at
160 * the port number that we want, return that as the pmask.
161 * otherwise mask that bit out of the port vector and
162 * decrement our port number ...
163 */
164 unsigned int pmask = portvec ^ (portvec & (portvec-1));
165 if (portn == 0)
166 return pmask;
167 portn--;
168 portvec &= ~pmask;
169 }
170 /*NOTREACHED*/
171}
7ee9ff94 172
b8ff05a9
DM		 173enum {
174 MAX_TXQ_ENTRIES = 16384,
175 MAX_CTRL_TXQ_ENTRIES = 1024,
176 MAX_RSPQ_ENTRIES = 16384,
177 MAX_RX_BUFFERS = 16384,
178 MIN_TXQ_ENTRIES = 32,
179 MIN_CTRL_TXQ_ENTRIES = 32,
180 MIN_RSPQ_ENTRIES = 128,
181 MIN_FL_ENTRIES = 16
182};
183
f2b7e78d
VP		 184/* Host shadow copy of ingress filter entry. This is in host native format
185 * and doesn't match the ordering or bit order, etc. of the hardware of the
186 * firmware command. The use of bit-field structure elements is purely to
187 * remind ourselves of the field size limitations and save memory in the case
188 * where the filter table is large.
189 */
190struct filter_entry {
191 /* Administrative fields for filter.
192 */
193 u32 valid:1; /* filter allocated and valid */
194 u32 locked:1; /* filter is administratively locked */
195
196 u32 pending:1; /* filter action is pending firmware reply */
197 u32 smtidx:8; /* Source MAC Table index for smac */
198 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */
199
200 /* The filter itself. Most of this is a straight copy of information
201 * provided by the extended ioctl(). Some fields are translated to
202 * internal forms -- for instance the Ingress Queue ID passed in from
203 * the ioctl() is translated into the Absolute Ingress Queue ID.
204 */
205 struct ch_filter_specification fs;
206};
207
b8ff05a9
DM		 208#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
209 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
210 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
211
060e0c75 212#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
b8ff05a9
DM		 213
214static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
060e0c75 215 CH_DEVICE(0xa000, 0), /* PE10K */
ccea790e
DM		 216 CH_DEVICE(0x4001, -1),
217 CH_DEVICE(0x4002, -1),
218 CH_DEVICE(0x4003, -1),
219 CH_DEVICE(0x4004, -1),
220 CH_DEVICE(0x4005, -1),
221 CH_DEVICE(0x4006, -1),
222 CH_DEVICE(0x4007, -1),
223 CH_DEVICE(0x4008, -1),
224 CH_DEVICE(0x4009, -1),
225 CH_DEVICE(0x400a, -1),
226 CH_DEVICE(0x4401, 4),
227 CH_DEVICE(0x4402, 4),
228 CH_DEVICE(0x4403, 4),
229 CH_DEVICE(0x4404, 4),
230 CH_DEVICE(0x4405, 4),
231 CH_DEVICE(0x4406, 4),
232 CH_DEVICE(0x4407, 4),
233 CH_DEVICE(0x4408, 4),
234 CH_DEVICE(0x4409, 4),
235 CH_DEVICE(0x440a, 4),
f637d577
VP		 236 CH_DEVICE(0x440d, 4),
237 CH_DEVICE(0x440e, 4),
9ef603a0
VP		 238 CH_DEVICE(0x5001, 4),
239 CH_DEVICE(0x5002, 4),
240 CH_DEVICE(0x5003, 4),
241 CH_DEVICE(0x5004, 4),
242 CH_DEVICE(0x5005, 4),
243 CH_DEVICE(0x5006, 4),
244 CH_DEVICE(0x5007, 4),
245 CH_DEVICE(0x5008, 4),
246 CH_DEVICE(0x5009, 4),
247 CH_DEVICE(0x500A, 4),
248 CH_DEVICE(0x500B, 4),
249 CH_DEVICE(0x500C, 4),
250 CH_DEVICE(0x500D, 4),
251 CH_DEVICE(0x500E, 4),
252 CH_DEVICE(0x500F, 4),
253 CH_DEVICE(0x5010, 4),
254 CH_DEVICE(0x5011, 4),
255 CH_DEVICE(0x5012, 4),
256 CH_DEVICE(0x5013, 4),
f0a8e6de
HS		 257 CH_DEVICE(0x5014, 4),
258 CH_DEVICE(0x5015, 4),
9ef603a0
VP		 259 CH_DEVICE(0x5401, 4),
260 CH_DEVICE(0x5402, 4),
261 CH_DEVICE(0x5403, 4),
262 CH_DEVICE(0x5404, 4),
263 CH_DEVICE(0x5405, 4),
264 CH_DEVICE(0x5406, 4),
265 CH_DEVICE(0x5407, 4),
266 CH_DEVICE(0x5408, 4),
267 CH_DEVICE(0x5409, 4),
268 CH_DEVICE(0x540A, 4),
269 CH_DEVICE(0x540B, 4),
270 CH_DEVICE(0x540C, 4),
271 CH_DEVICE(0x540D, 4),
272 CH_DEVICE(0x540E, 4),
273 CH_DEVICE(0x540F, 4),
274 CH_DEVICE(0x5410, 4),
275 CH_DEVICE(0x5411, 4),
276 CH_DEVICE(0x5412, 4),
277 CH_DEVICE(0x5413, 4),
f0a8e6de
HS		 278 CH_DEVICE(0x5414, 4),
279 CH_DEVICE(0x5415, 4),
b8ff05a9
DM		 280 { 0, }
281};
282
16e47624 283#define FW4_FNAME "cxgb4/t4fw.bin"
0a57a536 284#define FW5_FNAME "cxgb4/t5fw.bin"
16e47624 285#define FW4_CFNAME "cxgb4/t4-config.txt"
0a57a536 286#define FW5_CFNAME "cxgb4/t5-config.txt"
b8ff05a9
DM		 287
288MODULE_DESCRIPTION(DRV_DESC);
289MODULE_AUTHOR("Chelsio Communications");
290MODULE_LICENSE("Dual BSD/GPL");
291MODULE_VERSION(DRV_VERSION);
292MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
16e47624 293MODULE_FIRMWARE(FW4_FNAME);
0a57a536 294MODULE_FIRMWARE(FW5_FNAME);
b8ff05a9 295
636f9d37
VP		 296/*
297 * Normally we're willing to become the firmware's Master PF but will be happy
298 * if another PF has already become the Master and initialized the adapter.
299 * Setting "force_init" will cause this driver to forcibly establish itself as
300 * the Master PF and initialize the adapter.
301 */
302static uint force_init;
303
304module_param(force_init, uint, 0644);
305MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
306
13ee15d3
VP		 307/*
308 * Normally if the firmware we connect to has Configuration File support, we
309 * use that and only fall back to the old Driver-based initialization if the
310 * Configuration File fails for some reason. If force_old_init is set, then
311 * we'll always use the old Driver-based initialization sequence.
312 */
313static uint force_old_init;
314
315module_param(force_old_init, uint, 0644);
316MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
317
b8ff05a9
DM		 318static int dflt_msg_enable = DFLT_MSG_ENABLE;
319
320module_param(dflt_msg_enable, int, 0644);
321MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
322
323/*
324 * The driver uses the best interrupt scheme available on a platform in the
325 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
326 * of these schemes the driver may consider as follows:
327 *
328 * msi = 2: choose from among all three options
329 * msi = 1: only consider MSI and INTx interrupts
330 * msi = 0: force INTx interrupts
331 */
332static int msi = 2;
333
334module_param(msi, int, 0644);
335MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
336
337/*
338 * Queue interrupt hold-off timer values. Queues default to the first of these
339 * upon creation.
340 */
341static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
342
343module_param_array(intr_holdoff, uint, NULL, 0644);
344MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
345 "0..4 in microseconds");
346
347static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
348
349module_param_array(intr_cnt, uint, NULL, 0644);
350MODULE_PARM_DESC(intr_cnt,
351 "thresholds 1..3 for queue interrupt packet counters");
352
636f9d37
VP		 353/*
354 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
355 * offset by 2 bytes in order to have the IP headers line up on 4-byte
356 * boundaries. This is a requirement for many architectures which will throw
357 * a machine check fault if an attempt is made to access one of the 4-byte IP
358 * header fields on a non-4-byte boundary. And it's a major performance issue
359 * even on some architectures which allow it like some implementations of the
360 * x86 ISA. However, some architectures don't mind this and for some very
361 * edge-case performance sensitive applications (like forwarding large volumes
362 * of small packets), setting this DMA offset to 0 will decrease the number of
363 * PCI-E Bus transfers enough to measurably affect performance.
364 */
365static int rx_dma_offset = 2;
366
eb939922 367static bool vf_acls;
b8ff05a9
DM		 368
369#ifdef CONFIG_PCI_IOV
370module_param(vf_acls, bool, 0644);
371MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
372
7d6727cf
SR		 373/* Configure the number of PCI-E Virtual Function which are to be instantiated
374 * on SR-IOV Capable Physical Functions.
0a57a536 375 */
7d6727cf 376static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
b8ff05a9
DM		 377
378module_param_array(num_vf, uint, NULL, 0644);
7d6727cf 379MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
b8ff05a9
DM		 380#endif
381
13ee15d3
VP		 382/*
383 * The filter TCAM has a fixed portion and a variable portion. The fixed
384 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
385 * ports. The variable portion is 36 bits which can include things like Exact
386 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
387 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
388 * far exceed the 36-bit budget for this "compressed" header portion of the
389 * filter. Thus, we have a scarce resource which must be carefully managed.
390 *
391 * By default we set this up to mostly match the set of filter matching
392 * capabilities of T3 but with accommodations for some of T4's more
393 * interesting features:
394 *
395 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
396 * [Inner] VLAN (17), Port (3), FCoE (1) }
397 */
398enum {
399 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
400 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
401 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
402};
403
404static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
405
f2b7e78d
VP		 406module_param(tp_vlan_pri_map, uint, 0644);
407MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
408
b8ff05a9
DM		 409static struct dentry *cxgb4_debugfs_root;
410
411static LIST_HEAD(adapter_list);
412static DEFINE_MUTEX(uld_mutex);
01bcca68
VP		 413/* Adapter list to be accessed from atomic context */
414static LIST_HEAD(adap_rcu_list);
415static DEFINE_SPINLOCK(adap_rcu_lock);
b8ff05a9
DM		 416static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
417static const char *uld_str[] = { "RDMA", "iSCSI" };
418
419static void link_report(struct net_device *dev)
420{
421 if (!netif_carrier_ok(dev))
422 netdev_info(dev, "link down\n");
423 else {
424 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
425
426 const char *s = "10Mbps";
427 const struct port_info *p = netdev_priv(dev);
428
429 switch (p->link_cfg.speed) {
430 case SPEED_10000:
431 s = "10Gbps";
432 break;
433 case SPEED_1000:
434 s = "1000Mbps";
435 break;
436 case SPEED_100:
437 s = "100Mbps";
438 break;
72aca4bf
KS		 439 case 40000: /* Need a SPEED_40000 in ethtool.h */
440 s = "40Gbps";
441 break;
b8ff05a9
DM		 442 }
443
444 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
445 fc[p->link_cfg.fc]);
446 }
447}
448
449void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
450{
451 struct net_device *dev = adapter->port[port_id];
452
453 /* Skip changes from disabled ports. */
454 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
455 if (link_stat)
456 netif_carrier_on(dev);
457 else
458 netif_carrier_off(dev);
459
460 link_report(dev);
461 }
462}
463
464void t4_os_portmod_changed(const struct adapter *adap, int port_id)
465{
466 static const char *mod_str[] = {
a0881cab 467 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
b8ff05a9
DM		 468 };
469
470 const struct net_device *dev = adap->port[port_id];
471 const struct port_info *pi = netdev_priv(dev);
472
473 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
474 netdev_info(dev, "port module unplugged\n");
a0881cab 475 else if (pi->mod_type < ARRAY_SIZE(mod_str))
b8ff05a9
DM		 476 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
477}
478
479/*
480 * Configure the exact and hash address filters to handle a port's multicast
481 * and secondary unicast MAC addresses.
482 */
483static int set_addr_filters(const struct net_device *dev, bool sleep)
484{
485 u64 mhash = 0;
486 u64 uhash = 0;
487 bool free = true;
488 u16 filt_idx[7];
489 const u8 *addr[7];
490 int ret, naddr = 0;
b8ff05a9
DM		 491 const struct netdev_hw_addr *ha;
492 int uc_cnt = netdev_uc_count(dev);
4a35ecf8 493 int mc_cnt = netdev_mc_count(dev);
b8ff05a9 494 const struct port_info *pi = netdev_priv(dev);
060e0c75 495 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 496
497 /* first do the secondary unicast addresses */
498 netdev_for_each_uc_addr(ha, dev) {
499 addr[naddr++] = ha->addr;
500 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 501 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 502 naddr, addr, filt_idx, &uhash, sleep);
503 if (ret < 0)
504 return ret;
505
506 free = false;
507 naddr = 0;
508 }
509 }
510
511 /* next set up the multicast addresses */
4a35ecf8
DM		 512 netdev_for_each_mc_addr(ha, dev) {
513 addr[naddr++] = ha->addr;
514 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
060e0c75 515 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
b8ff05a9
DM		 516 naddr, addr, filt_idx, &mhash, sleep);
517 if (ret < 0)
518 return ret;
519
520 free = false;
521 naddr = 0;
522 }
523 }
524
060e0c75 525 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
b8ff05a9
DM		 526 uhash | mhash, sleep);
527}
528
3069ee9b
VP		 529int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
530module_param(dbfifo_int_thresh, int, 0644);
531MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
532
404d9e3f
VP		 533/*
534 * usecs to sleep while draining the dbfifo
535 */
536static int dbfifo_drain_delay = 1000;
3069ee9b
VP		 537module_param(dbfifo_drain_delay, int, 0644);
538MODULE_PARM_DESC(dbfifo_drain_delay,
539 "usecs to sleep while draining the dbfifo");
540
b8ff05a9
DM		 541/*
542 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
543 * If @mtu is -1 it is left unchanged.
544 */
545static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
546{
547 int ret;
548 struct port_info *pi = netdev_priv(dev);
549
550 ret = set_addr_filters(dev, sleep_ok);
551 if (ret == 0)
060e0c75 552 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
b8ff05a9 553 (dev->flags & IFF_PROMISC) ? 1 : 0,
f8f5aafa 554 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
b8ff05a9
DM		 555 sleep_ok);
556 return ret;
557}
558
3069ee9b
VP		 559static struct workqueue_struct *workq;
560
b8ff05a9
DM		 561/**
562 * link_start - enable a port
563 * @dev: the port to enable
564 *
565 * Performs the MAC and PHY actions needed to enable a port.
566 */
567static int link_start(struct net_device *dev)
568{
569 int ret;
570 struct port_info *pi = netdev_priv(dev);
060e0c75 571 unsigned int mb = pi->adapter->fn;
b8ff05a9
DM		 572
573 /*
574 * We do not set address filters and promiscuity here, the stack does
575 * that step explicitly.
576 */
060e0c75 577 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
f646968f 578 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
b8ff05a9 579 if (ret == 0) {
060e0c75 580 ret = t4_change_mac(pi->adapter, mb, pi->viid,
b8ff05a9 581 pi->xact_addr_filt, dev->dev_addr, true,
b6bd29e7 582 true);
b8ff05a9
DM		 583 if (ret >= 0) {
584 pi->xact_addr_filt = ret;
585 ret = 0;
586 }
587 }
588 if (ret == 0)
060e0c75
DM		 589 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
590 &pi->link_cfg);
b8ff05a9 591 if (ret == 0)
060e0c75 592 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
b8ff05a9
DM		 593 return ret;
594}
595
f2b7e78d
VP		 596/* Clear a filter and release any of its resources that we own. This also
597 * clears the filter's "pending" status.
598 */
599static void clear_filter(struct adapter *adap, struct filter_entry *f)
600{
601 /* If the new or old filter have loopback rewriteing rules then we'll
602 * need to free any existing Layer Two Table (L2T) entries of the old
603 * filter rule. The firmware will handle freeing up any Source MAC
604 * Table (SMT) entries used for rewriting Source MAC Addresses in
605 * loopback rules.
606 */
607 if (f->l2t)
608 cxgb4_l2t_release(f->l2t);
609
610 /* The zeroing of the filter rule below clears the filter valid,
611 * pending, locked flags, l2t pointer, etc. so it's all we need for
612 * this operation.
613 */
614 memset(f, 0, sizeof(*f));
615}
616
617/* Handle a filter write/deletion reply.
618 */
619static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
620{
621 unsigned int idx = GET_TID(rpl);
622 unsigned int nidx = idx - adap->tids.ftid_base;
623 unsigned int ret;
624 struct filter_entry *f;
625
626 if (idx >= adap->tids.ftid_base && nidx <
627 (adap->tids.nftids + adap->tids.nsftids)) {
628 idx = nidx;
629 ret = GET_TCB_COOKIE(rpl->cookie);
630 f = &adap->tids.ftid_tab[idx];
631
632 if (ret == FW_FILTER_WR_FLT_DELETED) {
633 /* Clear the filter when we get confirmation from the
634 * hardware that the filter has been deleted.
635 */
636 clear_filter(adap, f);
637 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
638 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
639 idx);
640 clear_filter(adap, f);
641 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
642 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
643 f->pending = 0; /* asynchronous setup completed */
644 f->valid = 1;
645 } else {
646 /* Something went wrong. Issue a warning about the
647 * problem and clear everything out.
648 */
649 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
650 idx, ret);
651 clear_filter(adap, f);
652 }
653 }
654}
655
656/* Response queue handler for the FW event queue.
b8ff05a9
DM		 657 */
658static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
659 const struct pkt_gl *gl)
660{
661 u8 opcode = ((const struct rss_header *)rsp)->opcode;
662
663 rsp++; /* skip RSS header */
b407a4a9
VP		 664
665 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
666 */
667 if (unlikely(opcode == CPL_FW4_MSG &&
668 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
669 rsp++;
670 opcode = ((const struct rss_header *)rsp)->opcode;
671 rsp++;
672 if (opcode != CPL_SGE_EGR_UPDATE) {
673 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
674 , opcode);
675 goto out;
676 }
677 }
678
b8ff05a9
DM		 679 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
680 const struct cpl_sge_egr_update *p = (void *)rsp;
681 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
e46dab4d 682 struct sge_txq *txq;
b8ff05a9 683
e46dab4d 684 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
b8ff05a9 685 txq->restarts++;
e46dab4d 686 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
b8ff05a9
DM		 687 struct sge_eth_txq *eq;
688
689 eq = container_of(txq, struct sge_eth_txq, q);
690 netif_tx_wake_queue(eq->txq);
691 } else {
692 struct sge_ofld_txq *oq;
693
694 oq = container_of(txq, struct sge_ofld_txq, q);
695 tasklet_schedule(&oq->qresume_tsk);
696 }
697 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
698 const struct cpl_fw6_msg *p = (void *)rsp;
699
700 if (p->type == 0)
701 t4_handle_fw_rpl(q->adap, p->data);
702 } else if (opcode == CPL_L2T_WRITE_RPL) {
703 const struct cpl_l2t_write_rpl *p = (void *)rsp;
704
705 do_l2t_write_rpl(q->adap, p);
f2b7e78d
VP		 706 } else if (opcode == CPL_SET_TCB_RPL) {
707 const struct cpl_set_tcb_rpl *p = (void *)rsp;
708
709 filter_rpl(q->adap, p);
b8ff05a9
DM		 710 } else
711 dev_err(q->adap->pdev_dev,
712 "unexpected CPL %#x on FW event queue\n", opcode);
b407a4a9 713out:
b8ff05a9
DM		 714 return 0;
715}
716
717/**
718 * uldrx_handler - response queue handler for ULD queues
719 * @q: the response queue that received the packet
720 * @rsp: the response queue descriptor holding the offload message
721 * @gl: the gather list of packet fragments
722 *
723 * Deliver an ingress offload packet to a ULD. All processing is done by
724 * the ULD, we just maintain statistics.
725 */
726static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
727 const struct pkt_gl *gl)
728{
729 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
730
b407a4a9
VP		 731 /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
732 */
733 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
734 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
735 rsp += 2;
736
b8ff05a9
DM		 737 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
738 rxq->stats.nomem++;
739 return -1;
740 }
741 if (gl == NULL)
742 rxq->stats.imm++;
743 else if (gl == CXGB4_MSG_AN)
744 rxq->stats.an++;
745 else
746 rxq->stats.pkts++;
747 return 0;
748}
749
750static void disable_msi(struct adapter *adapter)
751{
752 if (adapter->flags & USING_MSIX) {
753 pci_disable_msix(adapter->pdev);
754 adapter->flags &= ~USING_MSIX;
755 } else if (adapter->flags & USING_MSI) {
756 pci_disable_msi(adapter->pdev);
757 adapter->flags &= ~USING_MSI;
758 }
759}
760
761/*
762 * Interrupt handler for non-data events used with MSI-X.
763 */
764static irqreturn_t t4_nondata_intr(int irq, void *cookie)
765{
766 struct adapter *adap = cookie;
767
768 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
769 if (v & PFSW) {
770 adap->swintr = 1;
771 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
772 }
773 t4_slow_intr_handler(adap);
774 return IRQ_HANDLED;
775}
776
777/*
778 * Name the MSI-X interrupts.
779 */
780static void name_msix_vecs(struct adapter *adap)
781{
ba27816c 782 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
b8ff05a9
DM		 783
784 /* non-data interrupts */
b1a3c2b6 785 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
b8ff05a9
DM		 786
787 /* FW events */
b1a3c2b6
DM		 788 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
789 adap->port[0]->name);
b8ff05a9
DM		 790
791 /* Ethernet queues */
792 for_each_port(adap, j) {
793 struct net_device *d = adap->port[j];
794 const struct port_info *pi = netdev_priv(d);
795
ba27816c 796 for (i = 0; i < pi->nqsets; i++, msi_idx++)
b8ff05a9
DM		 797 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
798 d->name, i);
b8ff05a9
DM		 799 }
800
801 /* offload queues */
ba27816c
DM		 802 for_each_ofldrxq(&adap->sge, i)
803 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
b1a3c2b6 804 adap->port[0]->name, i);
ba27816c
DM		 805
806 for_each_rdmarxq(&adap->sge, i)
807 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
b1a3c2b6 808 adap->port[0]->name, i);
b8ff05a9
DM		 809}
810
811static int request_msix_queue_irqs(struct adapter *adap)
812{
813 struct sge *s = &adap->sge;
404d9e3f 814 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
b8ff05a9
DM		 815
816 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
817 adap->msix_info[1].desc, &s->fw_evtq);
818 if (err)
819 return err;
820
821 for_each_ethrxq(s, ethqidx) {
404d9e3f
VP		 822 err = request_irq(adap->msix_info[msi_index].vec,
823 t4_sge_intr_msix, 0,
824 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 825 &s->ethrxq[ethqidx].rspq);
826 if (err)
827 goto unwind;
404d9e3f 828 msi_index++;
b8ff05a9
DM		 829 }
830 for_each_ofldrxq(s, ofldqidx) {
404d9e3f
VP		 831 err = request_irq(adap->msix_info[msi_index].vec,
832 t4_sge_intr_msix, 0,
833 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 834 &s->ofldrxq[ofldqidx].rspq);
835 if (err)
836 goto unwind;
404d9e3f 837 msi_index++;
b8ff05a9
DM		 838 }
839 for_each_rdmarxq(s, rdmaqidx) {
404d9e3f
VP		 840 err = request_irq(adap->msix_info[msi_index].vec,
841 t4_sge_intr_msix, 0,
842 adap->msix_info[msi_index].desc,
b8ff05a9
DM		 843 &s->rdmarxq[rdmaqidx].rspq);
844 if (err)
845 goto unwind;
404d9e3f 846 msi_index++;
b8ff05a9
DM		 847 }
848 return 0;
849
850unwind:
851 while (--rdmaqidx >= 0)
404d9e3f 852 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 853 &s->rdmarxq[rdmaqidx].rspq);
854 while (--ofldqidx >= 0)
404d9e3f 855 free_irq(adap->msix_info[--msi_index].vec,
b8ff05a9
DM		 856 &s->ofldrxq[ofldqidx].rspq);
857 while (--ethqidx >= 0)
404d9e3f
VP		 858 free_irq(adap->msix_info[--msi_index].vec,
859 &s->ethrxq[ethqidx].rspq);
b8ff05a9
DM		 860 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
861 return err;
862}
863
864static void free_msix_queue_irqs(struct adapter *adap)
865{
404d9e3f 866 int i, msi_index = 2;
b8ff05a9
DM		 867 struct sge *s = &adap->sge;
868
869 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
870 for_each_ethrxq(s, i)
404d9e3f 871 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
b8ff05a9 872 for_each_ofldrxq(s, i)
404d9e3f 873 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
b8ff05a9 874 for_each_rdmarxq(s, i)
404d9e3f 875 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
b8ff05a9
DM		 876}
877
671b0060
DM		 878/**
879 * write_rss - write the RSS table for a given port
880 * @pi: the port
881 * @queues: array of queue indices for RSS
882 *
883 * Sets up the portion of the HW RSS table for the port's VI to distribute
884 * packets to the Rx queues in @queues.
885 */
886static int write_rss(const struct port_info *pi, const u16 *queues)
887{
888 u16 *rss;
889 int i, err;
890 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
891
892 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
893 if (!rss)
894 return -ENOMEM;
895
896 /* map the queue indices to queue ids */
897 for (i = 0; i < pi->rss_size; i++, queues++)
898 rss[i] = q[*queues].rspq.abs_id;
899
060e0c75
DM		 900 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
901 pi->rss_size, rss, pi->rss_size);
671b0060
DM		 902 kfree(rss);
903 return err;
904}
905
b8ff05a9
DM		 906/**
907 * setup_rss - configure RSS
908 * @adap: the adapter
909 *
671b0060 910 * Sets up RSS for each port.
b8ff05a9
DM		 911 */
912static int setup_rss(struct adapter *adap)
913{
671b0060 914 int i, err;
b8ff05a9
DM		 915
916 for_each_port(adap, i) {
917 const struct port_info *pi = adap2pinfo(adap, i);
b8ff05a9 918
671b0060 919 err = write_rss(pi, pi->rss);
b8ff05a9
DM		 920 if (err)
921 return err;
922 }
923 return 0;
924}
925
e46dab4d
DM		 926/*
927 * Return the channel of the ingress queue with the given qid.
928 */
929static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
930{
931 qid -= p->ingr_start;
932 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
933}
934
b8ff05a9
DM		 935/*
936 * Wait until all NAPI handlers are descheduled.
937 */
938static void quiesce_rx(struct adapter *adap)
939{
940 int i;
941
942 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
943 struct sge_rspq *q = adap->sge.ingr_map[i];
944
945 if (q && q->handler)
946 napi_disable(&q->napi);
947 }
948}
949
950/*
951 * Enable NAPI scheduling and interrupt generation for all Rx queues.
952 */
953static void enable_rx(struct adapter *adap)
954{
955 int i;
956
957 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
958 struct sge_rspq *q = adap->sge.ingr_map[i];
959
960 if (!q)
961 continue;
962 if (q->handler)
963 napi_enable(&q->napi);
964 /* 0-increment GTS to start the timer and enable interrupts */
965 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
966 SEINTARM(q->intr_params) |
967 INGRESSQID(q->cntxt_id));
968 }
969}
970
971/**
972 * setup_sge_queues - configure SGE Tx/Rx/response queues
973 * @adap: the adapter
974 *
975 * Determines how many sets of SGE queues to use and initializes them.
976 * We support multiple queue sets per port if we have MSI-X, otherwise
977 * just one queue set per port.
978 */
979static int setup_sge_queues(struct adapter *adap)
980{
981 int err, msi_idx, i, j;
982 struct sge *s = &adap->sge;
983
984 bitmap_zero(s->starving_fl, MAX_EGRQ);
985 bitmap_zero(s->txq_maperr, MAX_EGRQ);
986
987 if (adap->flags & USING_MSIX)
988 msi_idx = 1; /* vector 0 is for non-queue interrupts */
989 else {
990 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
991 NULL, NULL);
992 if (err)
993 return err;
994 msi_idx = -((int)s->intrq.abs_id + 1);
995 }
996
997 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
998 msi_idx, NULL, fwevtq_handler);
999 if (err) {
1000freeout: t4_free_sge_resources(adap);
1001 return err;
1002 }
1003
1004 for_each_port(adap, i) {
1005 struct net_device *dev = adap->port[i];
1006 struct port_info *pi = netdev_priv(dev);
1007 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1008 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1009
1010 for (j = 0; j < pi->nqsets; j++, q++) {
1011 if (msi_idx > 0)
1012 msi_idx++;
1013 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1014 msi_idx, &q->fl,
1015 t4_ethrx_handler);
1016 if (err)
1017 goto freeout;
1018 q->rspq.idx = j;
1019 memset(&q->stats, 0, sizeof(q->stats));
1020 }
1021 for (j = 0; j < pi->nqsets; j++, t++) {
1022 err = t4_sge_alloc_eth_txq(adap, t, dev,
1023 netdev_get_tx_queue(dev, j),
1024 s->fw_evtq.cntxt_id);
1025 if (err)
1026 goto freeout;
1027 }
1028 }
1029
1030 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1031 for_each_ofldrxq(s, i) {
1032 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1033 struct net_device *dev = adap->port[i / j];
1034
1035 if (msi_idx > 0)
1036 msi_idx++;
1037 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1038 &q->fl, uldrx_handler);
1039 if (err)
1040 goto freeout;
1041 memset(&q->stats, 0, sizeof(q->stats));
1042 s->ofld_rxq[i] = q->rspq.abs_id;
1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1044 s->fw_evtq.cntxt_id);
1045 if (err)
1046 goto freeout;
1047 }
1048
1049 for_each_rdmarxq(s, i) {
1050 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1051
1052 if (msi_idx > 0)
1053 msi_idx++;
1054 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1055 msi_idx, &q->fl, uldrx_handler);
1056 if (err)
1057 goto freeout;
1058 memset(&q->stats, 0, sizeof(q->stats));
1059 s->rdma_rxq[i] = q->rspq.abs_id;
1060 }
1061
1062 for_each_port(adap, i) {
1063 /*
1064 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1065 * have RDMA queues, and that's the right value.
1066 */
1067 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1068 s->fw_evtq.cntxt_id,
1069 s->rdmarxq[i].rspq.cntxt_id);
1070 if (err)
1071 goto freeout;
1072 }
1073
1074 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1075 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1076 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1077 return 0;
1078}
1079
b8ff05a9
DM		 1080/*
1081 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1082 * The allocated memory is cleared.
1083 */
1084void *t4_alloc_mem(size_t size)
1085{
8be04b93 1086 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
b8ff05a9
DM		 1087
1088 if (!p)
89bf67f1 1089 p = vzalloc(size);
b8ff05a9
DM		 1090 return p;
1091}
1092
1093/*
1094 * Free memory allocated through alloc_mem().
1095 */
31b9c19b 1096static void t4_free_mem(void *addr)
b8ff05a9
DM		 1097{
1098 if (is_vmalloc_addr(addr))
1099 vfree(addr);
1100 else
1101 kfree(addr);
1102}
1103
f2b7e78d
VP		 1104/* Send a Work Request to write the filter at a specified index. We construct
1105 * a Firmware Filter Work Request to have the work done and put the indicated
1106 * filter into "pending" mode which will prevent any further actions against
1107 * it till we get a reply from the firmware on the completion status of the
1108 * request.
1109 */
1110static int set_filter_wr(struct adapter *adapter, int fidx)
1111{
1112 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1113 struct sk_buff *skb;
1114 struct fw_filter_wr *fwr;
1115 unsigned int ftid;
1116
1117 /* If the new filter requires loopback Destination MAC and/or VLAN
1118 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1119 * the filter.
1120 */
1121 if (f->fs.newdmac || f->fs.newvlan) {
1122 /* allocate L2T entry for new filter */
1123 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1124 if (f->l2t == NULL)
1125 return -EAGAIN;
1126 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1127 f->fs.eport, f->fs.dmac)) {
1128 cxgb4_l2t_release(f->l2t);
1129 f->l2t = NULL;
1130 return -ENOMEM;
1131 }
1132 }
1133
1134 ftid = adapter->tids.ftid_base + fidx;
1135
1136 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1137 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1138 memset(fwr, 0, sizeof(*fwr));
1139
1140 /* It would be nice to put most of the following in t4_hw.c but most
1141 * of the work is translating the cxgbtool ch_filter_specification
1142 * into the Work Request and the definition of that structure is
1143 * currently in cxgbtool.h which isn't appropriate to pull into the
1144 * common code. We may eventually try to come up with a more neutral
1145 * filter specification structure but for now it's easiest to simply
1146 * put this fairly direct code in line ...
1147 */
1148 fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1149 fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1150 fwr->tid_to_iq =
1151 htonl(V_FW_FILTER_WR_TID(ftid) |
1152 V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1153 V_FW_FILTER_WR_NOREPLY(0) |
1154 V_FW_FILTER_WR_IQ(f->fs.iq));
1155 fwr->del_filter_to_l2tix =
1156 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1157 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1158 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1159 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1160 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1161 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1162 V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1163 V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1164 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1165 f->fs.newvlan == VLAN_REWRITE) |
1166 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1167 f->fs.newvlan == VLAN_REWRITE) |
1168 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1169 V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1170 V_FW_FILTER_WR_PRIO(f->fs.prio) |
1171 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1172 fwr->ethtype = htons(f->fs.val.ethtype);
1173 fwr->ethtypem = htons(f->fs.mask.ethtype);
1174 fwr->frag_to_ovlan_vldm =
1175 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1176 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1177 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1178 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1179 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1180 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1181 fwr->smac_sel = 0;
1182 fwr->rx_chan_rx_rpl_iq =
1183 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1184 V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1185 fwr->maci_to_matchtypem =
1186 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1187 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1188 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1189 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1190 V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1191 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1192 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1193 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1194 fwr->ptcl = f->fs.val.proto;
1195 fwr->ptclm = f->fs.mask.proto;
1196 fwr->ttyp = f->fs.val.tos;
1197 fwr->ttypm = f->fs.mask.tos;
1198 fwr->ivlan = htons(f->fs.val.ivlan);
1199 fwr->ivlanm = htons(f->fs.mask.ivlan);
1200 fwr->ovlan = htons(f->fs.val.ovlan);
1201 fwr->ovlanm = htons(f->fs.mask.ovlan);
1202 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1203 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1204 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1205 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1206 fwr->lp = htons(f->fs.val.lport);
1207 fwr->lpm = htons(f->fs.mask.lport);
1208 fwr->fp = htons(f->fs.val.fport);
1209 fwr->fpm = htons(f->fs.mask.fport);
1210 if (f->fs.newsmac)
1211 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1212
1213 /* Mark the filter as "pending" and ship off the Filter Work Request.
1214 * When we get the Work Request Reply we'll clear the pending status.
1215 */
1216 f->pending = 1;
1217 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1218 t4_ofld_send(adapter, skb);
1219 return 0;
1220}
1221
1222/* Delete the filter at a specified index.
1223 */
1224static int del_filter_wr(struct adapter *adapter, int fidx)
1225{
1226 struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1227 struct sk_buff *skb;
1228 struct fw_filter_wr *fwr;
1229 unsigned int len, ftid;
1230
1231 len = sizeof(*fwr);
1232 ftid = adapter->tids.ftid_base + fidx;
1233
1234 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1235 fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1236 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1237
1238 /* Mark the filter as "pending" and ship off the Filter Work Request.
1239 * When we get the Work Request Reply we'll clear the pending status.
1240 */
1241 f->pending = 1;
1242 t4_mgmt_tx(adapter, skb);
1243 return 0;
1244}
1245
b8ff05a9
DM		 1246static inline int is_offload(const struct adapter *adap)
1247{
1248 return adap->params.offload;
1249}
1250
1251/*
1252 * Implementation of ethtool operations.
1253 */
1254
1255static u32 get_msglevel(struct net_device *dev)
1256{
1257 return netdev2adap(dev)->msg_enable;
1258}
1259
1260static void set_msglevel(struct net_device *dev, u32 val)
1261{
1262 netdev2adap(dev)->msg_enable = val;
1263}
1264
1265static char stats_strings[][ETH_GSTRING_LEN] = {
1266 "TxOctetsOK ",
1267 "TxFramesOK ",
1268 "TxBroadcastFrames ",
1269 "TxMulticastFrames ",
1270 "TxUnicastFrames ",
1271 "TxErrorFrames ",
1272
1273 "TxFrames64 ",
1274 "TxFrames65To127 ",
1275 "TxFrames128To255 ",
1276 "TxFrames256To511 ",
1277 "TxFrames512To1023 ",
1278 "TxFrames1024To1518 ",
1279 "TxFrames1519ToMax ",
1280
1281 "TxFramesDropped ",
1282 "TxPauseFrames ",
1283 "TxPPP0Frames ",
1284 "TxPPP1Frames ",
1285 "TxPPP2Frames ",
1286 "TxPPP3Frames ",
1287 "TxPPP4Frames ",
1288 "TxPPP5Frames ",
1289 "TxPPP6Frames ",
1290 "TxPPP7Frames ",
1291
1292 "RxOctetsOK ",
1293 "RxFramesOK ",
1294 "RxBroadcastFrames ",
1295 "RxMulticastFrames ",
1296 "RxUnicastFrames ",
1297
1298 "RxFramesTooLong ",
1299 "RxJabberErrors ",
1300 "RxFCSErrors ",
1301 "RxLengthErrors ",
1302 "RxSymbolErrors ",
1303 "RxRuntFrames ",
1304
1305 "RxFrames64 ",
1306 "RxFrames65To127 ",
1307 "RxFrames128To255 ",
1308 "RxFrames256To511 ",
1309 "RxFrames512To1023 ",
1310 "RxFrames1024To1518 ",
1311 "RxFrames1519ToMax ",
1312
1313 "RxPauseFrames ",
1314 "RxPPP0Frames ",
1315 "RxPPP1Frames ",
1316 "RxPPP2Frames ",
1317 "RxPPP3Frames ",
1318 "RxPPP4Frames ",
1319 "RxPPP5Frames ",
1320 "RxPPP6Frames ",
1321 "RxPPP7Frames ",
1322
1323 "RxBG0FramesDropped ",
1324 "RxBG1FramesDropped ",
1325 "RxBG2FramesDropped ",
1326 "RxBG3FramesDropped ",
1327 "RxBG0FramesTrunc ",
1328 "RxBG1FramesTrunc ",
1329 "RxBG2FramesTrunc ",
1330 "RxBG3FramesTrunc ",
1331
1332 "TSO ",
1333 "TxCsumOffload ",
1334 "RxCsumGood ",
1335 "VLANextractions ",
1336 "VLANinsertions ",
4a6346d4
DM		 1337 "GROpackets ",
1338 "GROmerged ",
22adfe0a
SR		 1339 "WriteCoalSuccess ",
1340 "WriteCoalFail ",
b8ff05a9
DM		 1341};
1342
1343static int get_sset_count(struct net_device *dev, int sset)
1344{
1345 switch (sset) {
1346 case ETH_SS_STATS:
1347 return ARRAY_SIZE(stats_strings);
1348 default:
1349 return -EOPNOTSUPP;
1350 }
1351}
1352
1353#define T4_REGMAP_SIZE (160 * 1024)
251f9e88 1354#define T5_REGMAP_SIZE (332 * 1024)
b8ff05a9
DM		 1355
1356static int get_regs_len(struct net_device *dev)
1357{
251f9e88 1358 struct adapter *adap = netdev2adap(dev);
d14807dd 1359 if (is_t4(adap->params.chip))
251f9e88
SR		 1360 return T4_REGMAP_SIZE;
1361 else
1362 return T5_REGMAP_SIZE;
b8ff05a9
DM		 1363}
1364
1365static int get_eeprom_len(struct net_device *dev)
1366{
1367 return EEPROMSIZE;
1368}
1369
1370static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1371{
1372 struct adapter *adapter = netdev2adap(dev);
1373
23020ab3
RJ		 1374 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1375 strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1376 strlcpy(info->bus_info, pci_name(adapter->pdev),
1377 sizeof(info->bus_info));
b8ff05a9 1378
84b40501 1379 if (adapter->params.fw_vers)
b8ff05a9
DM		 1380 snprintf(info->fw_version, sizeof(info->fw_version),
1381 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1382 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1383 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1384 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1385 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1386 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1387 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1388 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1389 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1390}
1391
1392static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1393{
1394 if (stringset == ETH_SS_STATS)
1395 memcpy(data, stats_strings, sizeof(stats_strings));
1396}
1397
1398/*
1399 * port stats maintained per queue of the port. They should be in the same
1400 * order as in stats_strings above.
1401 */
1402struct queue_port_stats {
1403 u64 tso;
1404 u64 tx_csum;
1405 u64 rx_csum;
1406 u64 vlan_ex;
1407 u64 vlan_ins;
4a6346d4
DM		 1408 u64 gro_pkts;
1409 u64 gro_merged;
b8ff05a9
DM		 1410};
1411
1412static void collect_sge_port_stats(const struct adapter *adap,
1413 const struct port_info *p, struct queue_port_stats *s)
1414{
1415 int i;
1416 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1417 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1418
1419 memset(s, 0, sizeof(*s));
1420 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1421 s->tso += tx->tso;
1422 s->tx_csum += tx->tx_cso;
1423 s->rx_csum += rx->stats.rx_cso;
1424 s->vlan_ex += rx->stats.vlan_ex;
1425 s->vlan_ins += tx->vlan_ins;
4a6346d4
DM		 1426 s->gro_pkts += rx->stats.lro_pkts;
1427 s->gro_merged += rx->stats.lro_merged;
b8ff05a9
DM		 1428 }
1429}
1430
1431static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1432 u64 *data)
1433{
1434 struct port_info *pi = netdev_priv(dev);
1435 struct adapter *adapter = pi->adapter;
22adfe0a 1436 u32 val1, val2;
b8ff05a9
DM		 1437
1438 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1439
1440 data += sizeof(struct port_stats) / sizeof(u64);
1441 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
22adfe0a 1442 data += sizeof(struct queue_port_stats) / sizeof(u64);
d14807dd 1443 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 1444 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1445 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1446 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1447 *data = val1 - val2;
1448 data++;
1449 *data = val2;
1450 data++;
1451 } else {
1452 memset(data, 0, 2 * sizeof(u64));
1453 *data += 2;
1454 }
b8ff05a9
DM		 1455}
1456
1457/*
1458 * Return a version number to identify the type of adapter. The scheme is:
1459 * - bits 0..9: chip version
1460 * - bits 10..15: chip revision
835bb606 1461 * - bits 16..23: register dump version
b8ff05a9
DM		 1462 */
1463static inline unsigned int mk_adap_vers(const struct adapter *ap)
1464{
d14807dd
HS		 1465 return CHELSIO_CHIP_VERSION(ap->params.chip) |
1466 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
b8ff05a9
DM		 1467}
1468
1469static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1470 unsigned int end)
1471{
1472 u32 *p = buf + start;
1473
1474 for ( ; start <= end; start += sizeof(u32))
1475 *p++ = t4_read_reg(ap, start);
1476}
1477
1478static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1479 void *buf)
1480{
251f9e88 1481 static const unsigned int t4_reg_ranges[] = {
b8ff05a9
DM		 1482 0x1008, 0x1108,
1483 0x1180, 0x11b4,
1484 0x11fc, 0x123c,
1485 0x1300, 0x173c,
1486 0x1800, 0x18fc,
1487 0x3000, 0x30d8,
1488 0x30e0, 0x5924,
1489 0x5960, 0x59d4,
1490 0x5a00, 0x5af8,
1491 0x6000, 0x6098,
1492 0x6100, 0x6150,
1493 0x6200, 0x6208,
1494 0x6240, 0x6248,
1495 0x6280, 0x6338,
1496 0x6370, 0x638c,
1497 0x6400, 0x643c,
1498 0x6500, 0x6524,
1499 0x6a00, 0x6a38,
1500 0x6a60, 0x6a78,
1501 0x6b00, 0x6b84,
1502 0x6bf0, 0x6c84,
1503 0x6cf0, 0x6d84,
1504 0x6df0, 0x6e84,
1505 0x6ef0, 0x6f84,
1506 0x6ff0, 0x7084,
1507 0x70f0, 0x7184,
1508 0x71f0, 0x7284,
1509 0x72f0, 0x7384,
1510 0x73f0, 0x7450,
1511 0x7500, 0x7530,
1512 0x7600, 0x761c,
1513 0x7680, 0x76cc,
1514 0x7700, 0x7798,
1515 0x77c0, 0x77fc,
1516 0x7900, 0x79fc,
1517 0x7b00, 0x7c38,
1518 0x7d00, 0x7efc,
1519 0x8dc0, 0x8e1c,
1520 0x8e30, 0x8e78,
1521 0x8ea0, 0x8f6c,
1522 0x8fc0, 0x9074,
1523 0x90fc, 0x90fc,
1524 0x9400, 0x9458,
1525 0x9600, 0x96bc,
1526 0x9800, 0x9808,
1527 0x9820, 0x983c,
1528 0x9850, 0x9864,
1529 0x9c00, 0x9c6c,
1530 0x9c80, 0x9cec,
1531 0x9d00, 0x9d6c,
1532 0x9d80, 0x9dec,
1533 0x9e00, 0x9e6c,
1534 0x9e80, 0x9eec,
1535 0x9f00, 0x9f6c,
1536 0x9f80, 0x9fec,
1537 0xd004, 0xd03c,
1538 0xdfc0, 0xdfe0,
1539 0xe000, 0xea7c,
1540 0xf000, 0x11190,
835bb606
DM		 1541 0x19040, 0x1906c,
1542 0x19078, 0x19080,
1543 0x1908c, 0x19124,
b8ff05a9
DM		 1544 0x19150, 0x191b0,
1545 0x191d0, 0x191e8,
1546 0x19238, 0x1924c,
1547 0x193f8, 0x19474,
1548 0x19490, 0x194f8,
1549 0x19800, 0x19f30,
1550 0x1a000, 0x1a06c,
1551 0x1a0b0, 0x1a120,
1552 0x1a128, 0x1a138,
1553 0x1a190, 0x1a1c4,
1554 0x1a1fc, 0x1a1fc,
1555 0x1e040, 0x1e04c,
835bb606 1556 0x1e284, 0x1e28c,
b8ff05a9
DM		 1557 0x1e2c0, 0x1e2c0,
1558 0x1e2e0, 0x1e2e0,
1559 0x1e300, 0x1e384,
1560 0x1e3c0, 0x1e3c8,
1561 0x1e440, 0x1e44c,
835bb606 1562 0x1e684, 0x1e68c,
b8ff05a9
DM		 1563 0x1e6c0, 0x1e6c0,
1564 0x1e6e0, 0x1e6e0,
1565 0x1e700, 0x1e784,
1566 0x1e7c0, 0x1e7c8,
1567 0x1e840, 0x1e84c,
835bb606 1568 0x1ea84, 0x1ea8c,
b8ff05a9
DM		 1569 0x1eac0, 0x1eac0,
1570 0x1eae0, 0x1eae0,
1571 0x1eb00, 0x1eb84,
1572 0x1ebc0, 0x1ebc8,
1573 0x1ec40, 0x1ec4c,
835bb606 1574 0x1ee84, 0x1ee8c,
b8ff05a9
DM		 1575 0x1eec0, 0x1eec0,
1576 0x1eee0, 0x1eee0,
1577 0x1ef00, 0x1ef84,
1578 0x1efc0, 0x1efc8,
1579 0x1f040, 0x1f04c,
835bb606 1580 0x1f284, 0x1f28c,
b8ff05a9
DM		 1581 0x1f2c0, 0x1f2c0,
1582 0x1f2e0, 0x1f2e0,
1583 0x1f300, 0x1f384,
1584 0x1f3c0, 0x1f3c8,
1585 0x1f440, 0x1f44c,
835bb606 1586 0x1f684, 0x1f68c,
b8ff05a9
DM		 1587 0x1f6c0, 0x1f6c0,
1588 0x1f6e0, 0x1f6e0,
1589 0x1f700, 0x1f784,
1590 0x1f7c0, 0x1f7c8,
1591 0x1f840, 0x1f84c,
835bb606 1592 0x1fa84, 0x1fa8c,
b8ff05a9
DM		 1593 0x1fac0, 0x1fac0,
1594 0x1fae0, 0x1fae0,
1595 0x1fb00, 0x1fb84,
1596 0x1fbc0, 0x1fbc8,
1597 0x1fc40, 0x1fc4c,
835bb606 1598 0x1fe84, 0x1fe8c,
b8ff05a9
DM		 1599 0x1fec0, 0x1fec0,
1600 0x1fee0, 0x1fee0,
1601 0x1ff00, 0x1ff84,
1602 0x1ffc0, 0x1ffc8,
1603 0x20000, 0x2002c,
1604 0x20100, 0x2013c,
1605 0x20190, 0x201c8,
1606 0x20200, 0x20318,
1607 0x20400, 0x20528,
1608 0x20540, 0x20614,
1609 0x21000, 0x21040,
1610 0x2104c, 0x21060,
1611 0x210c0, 0x210ec,
1612 0x21200, 0x21268,
1613 0x21270, 0x21284,
1614 0x212fc, 0x21388,
1615 0x21400, 0x21404,
1616 0x21500, 0x21518,
1617 0x2152c, 0x2153c,
1618 0x21550, 0x21554,
1619 0x21600, 0x21600,
1620 0x21608, 0x21628,
1621 0x21630, 0x2163c,
1622 0x21700, 0x2171c,
1623 0x21780, 0x2178c,
1624 0x21800, 0x21c38,
1625 0x21c80, 0x21d7c,
1626 0x21e00, 0x21e04,
1627 0x22000, 0x2202c,
1628 0x22100, 0x2213c,
1629 0x22190, 0x221c8,
1630 0x22200, 0x22318,
1631 0x22400, 0x22528,
1632 0x22540, 0x22614,
1633 0x23000, 0x23040,
1634 0x2304c, 0x23060,
1635 0x230c0, 0x230ec,
1636 0x23200, 0x23268,
1637 0x23270, 0x23284,
1638 0x232fc, 0x23388,
1639 0x23400, 0x23404,
1640 0x23500, 0x23518,
1641 0x2352c, 0x2353c,
1642 0x23550, 0x23554,
1643 0x23600, 0x23600,
1644 0x23608, 0x23628,
1645 0x23630, 0x2363c,
1646 0x23700, 0x2371c,
1647 0x23780, 0x2378c,
1648 0x23800, 0x23c38,
1649 0x23c80, 0x23d7c,
1650 0x23e00, 0x23e04,
1651 0x24000, 0x2402c,
1652 0x24100, 0x2413c,
1653 0x24190, 0x241c8,
1654 0x24200, 0x24318,
1655 0x24400, 0x24528,
1656 0x24540, 0x24614,
1657 0x25000, 0x25040,
1658 0x2504c, 0x25060,
1659 0x250c0, 0x250ec,
1660 0x25200, 0x25268,
1661 0x25270, 0x25284,
1662 0x252fc, 0x25388,
1663 0x25400, 0x25404,
1664 0x25500, 0x25518,
1665 0x2552c, 0x2553c,
1666 0x25550, 0x25554,
1667 0x25600, 0x25600,
1668 0x25608, 0x25628,
1669 0x25630, 0x2563c,
1670 0x25700, 0x2571c,
1671 0x25780, 0x2578c,
1672 0x25800, 0x25c38,
1673 0x25c80, 0x25d7c,
1674 0x25e00, 0x25e04,
1675 0x26000, 0x2602c,
1676 0x26100, 0x2613c,
1677 0x26190, 0x261c8,
1678 0x26200, 0x26318,
1679 0x26400, 0x26528,
1680 0x26540, 0x26614,
1681 0x27000, 0x27040,
1682 0x2704c, 0x27060,
1683 0x270c0, 0x270ec,
1684 0x27200, 0x27268,
1685 0x27270, 0x27284,
1686 0x272fc, 0x27388,
1687 0x27400, 0x27404,
1688 0x27500, 0x27518,
1689 0x2752c, 0x2753c,
1690 0x27550, 0x27554,
1691 0x27600, 0x27600,
1692 0x27608, 0x27628,
1693 0x27630, 0x2763c,
1694 0x27700, 0x2771c,
1695 0x27780, 0x2778c,
1696 0x27800, 0x27c38,
1697 0x27c80, 0x27d7c,
1698 0x27e00, 0x27e04
1699 };
1700
251f9e88
SR		 1701 static const unsigned int t5_reg_ranges[] = {
1702 0x1008, 0x1148,
1703 0x1180, 0x11b4,
1704 0x11fc, 0x123c,
1705 0x1280, 0x173c,
1706 0x1800, 0x18fc,
1707 0x3000, 0x3028,
1708 0x3060, 0x30d8,
1709 0x30e0, 0x30fc,
1710 0x3140, 0x357c,
1711 0x35a8, 0x35cc,
1712 0x35ec, 0x35ec,
1713 0x3600, 0x5624,
1714 0x56cc, 0x575c,
1715 0x580c, 0x5814,
1716 0x5890, 0x58bc,
1717 0x5940, 0x59dc,
1718 0x59fc, 0x5a18,
1719 0x5a60, 0x5a9c,
1720 0x5b9c, 0x5bfc,
1721 0x6000, 0x6040,
1722 0x6058, 0x614c,
1723 0x7700, 0x7798,
1724 0x77c0, 0x78fc,
1725 0x7b00, 0x7c54,
1726 0x7d00, 0x7efc,
1727 0x8dc0, 0x8de0,
1728 0x8df8, 0x8e84,
1729 0x8ea0, 0x8f84,
1730 0x8fc0, 0x90f8,
1731 0x9400, 0x9470,
1732 0x9600, 0x96f4,
1733 0x9800, 0x9808,
1734 0x9820, 0x983c,
1735 0x9850, 0x9864,
1736 0x9c00, 0x9c6c,
1737 0x9c80, 0x9cec,
1738 0x9d00, 0x9d6c,
1739 0x9d80, 0x9dec,
1740 0x9e00, 0x9e6c,
1741 0x9e80, 0x9eec,
1742 0x9f00, 0x9f6c,
1743 0x9f80, 0xa020,
1744 0xd004, 0xd03c,
1745 0xdfc0, 0xdfe0,
1746 0xe000, 0x11088,
1747 0x1109c, 0x1117c,
1748 0x11190, 0x11204,
1749 0x19040, 0x1906c,
1750 0x19078, 0x19080,
1751 0x1908c, 0x19124,
1752 0x19150, 0x191b0,
1753 0x191d0, 0x191e8,
1754 0x19238, 0x19290,
1755 0x193f8, 0x19474,
1756 0x19490, 0x194cc,
1757 0x194f0, 0x194f8,
1758 0x19c00, 0x19c60,
1759 0x19c94, 0x19e10,
1760 0x19e50, 0x19f34,
1761 0x19f40, 0x19f50,
1762 0x19f90, 0x19fe4,
1763 0x1a000, 0x1a06c,
1764 0x1a0b0, 0x1a120,
1765 0x1a128, 0x1a138,
1766 0x1a190, 0x1a1c4,
1767 0x1a1fc, 0x1a1fc,
1768 0x1e008, 0x1e00c,
1769 0x1e040, 0x1e04c,
1770 0x1e284, 0x1e290,
1771 0x1e2c0, 0x1e2c0,
1772 0x1e2e0, 0x1e2e0,
1773 0x1e300, 0x1e384,
1774 0x1e3c0, 0x1e3c8,
1775 0x1e408, 0x1e40c,
1776 0x1e440, 0x1e44c,
1777 0x1e684, 0x1e690,
1778 0x1e6c0, 0x1e6c0,
1779 0x1e6e0, 0x1e6e0,
1780 0x1e700, 0x1e784,
1781 0x1e7c0, 0x1e7c8,
1782 0x1e808, 0x1e80c,
1783 0x1e840, 0x1e84c,
1784 0x1ea84, 0x1ea90,
1785 0x1eac0, 0x1eac0,
1786 0x1eae0, 0x1eae0,
1787 0x1eb00, 0x1eb84,
1788 0x1ebc0, 0x1ebc8,
1789 0x1ec08, 0x1ec0c,
1790 0x1ec40, 0x1ec4c,
1791 0x1ee84, 0x1ee90,
1792 0x1eec0, 0x1eec0,
1793 0x1eee0, 0x1eee0,
1794 0x1ef00, 0x1ef84,
1795 0x1efc0, 0x1efc8,
1796 0x1f008, 0x1f00c,
1797 0x1f040, 0x1f04c,
1798 0x1f284, 0x1f290,
1799 0x1f2c0, 0x1f2c0,
1800 0x1f2e0, 0x1f2e0,
1801 0x1f300, 0x1f384,
1802 0x1f3c0, 0x1f3c8,
1803 0x1f408, 0x1f40c,
1804 0x1f440, 0x1f44c,
1805 0x1f684, 0x1f690,
1806 0x1f6c0, 0x1f6c0,
1807 0x1f6e0, 0x1f6e0,
1808 0x1f700, 0x1f784,
1809 0x1f7c0, 0x1f7c8,
1810 0x1f808, 0x1f80c,
1811 0x1f840, 0x1f84c,
1812 0x1fa84, 0x1fa90,
1813 0x1fac0, 0x1fac0,
1814 0x1fae0, 0x1fae0,
1815 0x1fb00, 0x1fb84,
1816 0x1fbc0, 0x1fbc8,
1817 0x1fc08, 0x1fc0c,
1818 0x1fc40, 0x1fc4c,
1819 0x1fe84, 0x1fe90,
1820 0x1fec0, 0x1fec0,
1821 0x1fee0, 0x1fee0,
1822 0x1ff00, 0x1ff84,
1823 0x1ffc0, 0x1ffc8,
1824 0x30000, 0x30030,
1825 0x30100, 0x30144,
1826 0x30190, 0x301d0,
1827 0x30200, 0x30318,
1828 0x30400, 0x3052c,
1829 0x30540, 0x3061c,
1830 0x30800, 0x30834,
1831 0x308c0, 0x30908,
1832 0x30910, 0x309ac,
1833 0x30a00, 0x30a04,
1834 0x30a0c, 0x30a2c,
1835 0x30a44, 0x30a50,
1836 0x30a74, 0x30c24,
1837 0x30d08, 0x30d14,
1838 0x30d1c, 0x30d20,
1839 0x30d3c, 0x30d50,
1840 0x31200, 0x3120c,
1841 0x31220, 0x31220,
1842 0x31240, 0x31240,
1843 0x31600, 0x31600,
1844 0x31608, 0x3160c,
1845 0x31a00, 0x31a1c,
1846 0x31e04, 0x31e20,
1847 0x31e38, 0x31e3c,
1848 0x31e80, 0x31e80,
1849 0x31e88, 0x31ea8,
1850 0x31eb0, 0x31eb4,
1851 0x31ec8, 0x31ed4,
1852 0x31fb8, 0x32004,
1853 0x32208, 0x3223c,
1854 0x32600, 0x32630,
1855 0x32a00, 0x32abc,
1856 0x32b00, 0x32b70,
1857 0x33000, 0x33048,
1858 0x33060, 0x3309c,
1859 0x330f0, 0x33148,
1860 0x33160, 0x3319c,
1861 0x331f0, 0x332e4,
1862 0x332f8, 0x333e4,
1863 0x333f8, 0x33448,
1864 0x33460, 0x3349c,
1865 0x334f0, 0x33548,
1866 0x33560, 0x3359c,
1867 0x335f0, 0x336e4,
1868 0x336f8, 0x337e4,
1869 0x337f8, 0x337fc,
1870 0x33814, 0x33814,
1871 0x3382c, 0x3382c,
1872 0x33880, 0x3388c,
1873 0x338e8, 0x338ec,
1874 0x33900, 0x33948,
1875 0x33960, 0x3399c,
1876 0x339f0, 0x33ae4,
1877 0x33af8, 0x33b10,
1878 0x33b28, 0x33b28,
1879 0x33b3c, 0x33b50,
1880 0x33bf0, 0x33c10,
1881 0x33c28, 0x33c28,
1882 0x33c3c, 0x33c50,
1883 0x33cf0, 0x33cfc,
1884 0x34000, 0x34030,
1885 0x34100, 0x34144,
1886 0x34190, 0x341d0,
1887 0x34200, 0x34318,
1888 0x34400, 0x3452c,
1889 0x34540, 0x3461c,
1890 0x34800, 0x34834,
1891 0x348c0, 0x34908,
1892 0x34910, 0x349ac,
1893 0x34a00, 0x34a04,
1894 0x34a0c, 0x34a2c,
1895 0x34a44, 0x34a50,
1896 0x34a74, 0x34c24,
1897 0x34d08, 0x34d14,
1898 0x34d1c, 0x34d20,
1899 0x34d3c, 0x34d50,
1900 0x35200, 0x3520c,
1901 0x35220, 0x35220,
1902 0x35240, 0x35240,
1903 0x35600, 0x35600,
1904 0x35608, 0x3560c,
1905 0x35a00, 0x35a1c,
1906 0x35e04, 0x35e20,
1907 0x35e38, 0x35e3c,
1908 0x35e80, 0x35e80,
1909 0x35e88, 0x35ea8,
1910 0x35eb0, 0x35eb4,
1911 0x35ec8, 0x35ed4,
1912 0x35fb8, 0x36004,
1913 0x36208, 0x3623c,
1914 0x36600, 0x36630,
1915 0x36a00, 0x36abc,
1916 0x36b00, 0x36b70,
1917 0x37000, 0x37048,
1918 0x37060, 0x3709c,
1919 0x370f0, 0x37148,
1920 0x37160, 0x3719c,
1921 0x371f0, 0x372e4,
1922 0x372f8, 0x373e4,
1923 0x373f8, 0x37448,
1924 0x37460, 0x3749c,
1925 0x374f0, 0x37548,
1926 0x37560, 0x3759c,
1927 0x375f0, 0x376e4,
1928 0x376f8, 0x377e4,
1929 0x377f8, 0x377fc,
1930 0x37814, 0x37814,
1931 0x3782c, 0x3782c,
1932 0x37880, 0x3788c,
1933 0x378e8, 0x378ec,
1934 0x37900, 0x37948,
1935 0x37960, 0x3799c,
1936 0x379f0, 0x37ae4,
1937 0x37af8, 0x37b10,
1938 0x37b28, 0x37b28,
1939 0x37b3c, 0x37b50,
1940 0x37bf0, 0x37c10,
1941 0x37c28, 0x37c28,
1942 0x37c3c, 0x37c50,
1943 0x37cf0, 0x37cfc,
1944 0x38000, 0x38030,
1945 0x38100, 0x38144,
1946 0x38190, 0x381d0,
1947 0x38200, 0x38318,
1948 0x38400, 0x3852c,
1949 0x38540, 0x3861c,
1950 0x38800, 0x38834,
1951 0x388c0, 0x38908,
1952 0x38910, 0x389ac,
1953 0x38a00, 0x38a04,
1954 0x38a0c, 0x38a2c,
1955 0x38a44, 0x38a50,
1956 0x38a74, 0x38c24,
1957 0x38d08, 0x38d14,
1958 0x38d1c, 0x38d20,
1959 0x38d3c, 0x38d50,
1960 0x39200, 0x3920c,
1961 0x39220, 0x39220,
1962 0x39240, 0x39240,
1963 0x39600, 0x39600,
1964 0x39608, 0x3960c,
1965 0x39a00, 0x39a1c,
1966 0x39e04, 0x39e20,
1967 0x39e38, 0x39e3c,
1968 0x39e80, 0x39e80,
1969 0x39e88, 0x39ea8,
1970 0x39eb0, 0x39eb4,
1971 0x39ec8, 0x39ed4,
1972 0x39fb8, 0x3a004,
1973 0x3a208, 0x3a23c,
1974 0x3a600, 0x3a630,
1975 0x3aa00, 0x3aabc,
1976 0x3ab00, 0x3ab70,
1977 0x3b000, 0x3b048,
1978 0x3b060, 0x3b09c,
1979 0x3b0f0, 0x3b148,
1980 0x3b160, 0x3b19c,
1981 0x3b1f0, 0x3b2e4,
1982 0x3b2f8, 0x3b3e4,
1983 0x3b3f8, 0x3b448,
1984 0x3b460, 0x3b49c,
1985 0x3b4f0, 0x3b548,
1986 0x3b560, 0x3b59c,
1987 0x3b5f0, 0x3b6e4,
1988 0x3b6f8, 0x3b7e4,
1989 0x3b7f8, 0x3b7fc,
1990 0x3b814, 0x3b814,
1991 0x3b82c, 0x3b82c,
1992 0x3b880, 0x3b88c,
1993 0x3b8e8, 0x3b8ec,
1994 0x3b900, 0x3b948,
1995 0x3b960, 0x3b99c,
1996 0x3b9f0, 0x3bae4,
1997 0x3baf8, 0x3bb10,
1998 0x3bb28, 0x3bb28,
1999 0x3bb3c, 0x3bb50,
2000 0x3bbf0, 0x3bc10,
2001 0x3bc28, 0x3bc28,
2002 0x3bc3c, 0x3bc50,
2003 0x3bcf0, 0x3bcfc,
2004 0x3c000, 0x3c030,
2005 0x3c100, 0x3c144,
2006 0x3c190, 0x3c1d0,
2007 0x3c200, 0x3c318,
2008 0x3c400, 0x3c52c,
2009 0x3c540, 0x3c61c,
2010 0x3c800, 0x3c834,
2011 0x3c8c0, 0x3c908,
2012 0x3c910, 0x3c9ac,
2013 0x3ca00, 0x3ca04,
2014 0x3ca0c, 0x3ca2c,
2015 0x3ca44, 0x3ca50,
2016 0x3ca74, 0x3cc24,
2017 0x3cd08, 0x3cd14,
2018 0x3cd1c, 0x3cd20,
2019 0x3cd3c, 0x3cd50,
2020 0x3d200, 0x3d20c,
2021 0x3d220, 0x3d220,
2022 0x3d240, 0x3d240,
2023 0x3d600, 0x3d600,
2024 0x3d608, 0x3d60c,
2025 0x3da00, 0x3da1c,
2026 0x3de04, 0x3de20,
2027 0x3de38, 0x3de3c,
2028 0x3de80, 0x3de80,
2029 0x3de88, 0x3dea8,
2030 0x3deb0, 0x3deb4,
2031 0x3dec8, 0x3ded4,
2032 0x3dfb8, 0x3e004,
2033 0x3e208, 0x3e23c,
2034 0x3e600, 0x3e630,
2035 0x3ea00, 0x3eabc,
2036 0x3eb00, 0x3eb70,
2037 0x3f000, 0x3f048,
2038 0x3f060, 0x3f09c,
2039 0x3f0f0, 0x3f148,
2040 0x3f160, 0x3f19c,
2041 0x3f1f0, 0x3f2e4,
2042 0x3f2f8, 0x3f3e4,
2043 0x3f3f8, 0x3f448,
2044 0x3f460, 0x3f49c,
2045 0x3f4f0, 0x3f548,
2046 0x3f560, 0x3f59c,
2047 0x3f5f0, 0x3f6e4,
2048 0x3f6f8, 0x3f7e4,
2049 0x3f7f8, 0x3f7fc,
2050 0x3f814, 0x3f814,
2051 0x3f82c, 0x3f82c,
2052 0x3f880, 0x3f88c,
2053 0x3f8e8, 0x3f8ec,
2054 0x3f900, 0x3f948,
2055 0x3f960, 0x3f99c,
2056 0x3f9f0, 0x3fae4,
2057 0x3faf8, 0x3fb10,
2058 0x3fb28, 0x3fb28,
2059 0x3fb3c, 0x3fb50,
2060 0x3fbf0, 0x3fc10,
2061 0x3fc28, 0x3fc28,
2062 0x3fc3c, 0x3fc50,
2063 0x3fcf0, 0x3fcfc,
2064 0x40000, 0x4000c,
2065 0x40040, 0x40068,
2066 0x40080, 0x40144,
2067 0x40180, 0x4018c,
2068 0x40200, 0x40298,
2069 0x402ac, 0x4033c,
2070 0x403f8, 0x403fc,
c1f49e3e 2071 0x41304, 0x413c4,
251f9e88
SR		 2072 0x41400, 0x4141c,
2073 0x41480, 0x414d0,
2074 0x44000, 0x44078,
2075 0x440c0, 0x44278,
2076 0x442c0, 0x44478,
2077 0x444c0, 0x44678,
2078 0x446c0, 0x44878,
2079 0x448c0, 0x449fc,
2080 0x45000, 0x45068,
2081 0x45080, 0x45084,
2082 0x450a0, 0x450b0,
2083 0x45200, 0x45268,
2084 0x45280, 0x45284,
2085 0x452a0, 0x452b0,
2086 0x460c0, 0x460e4,
2087 0x47000, 0x4708c,
2088 0x47200, 0x47250,
2089 0x47400, 0x47420,
2090 0x47600, 0x47618,
2091 0x47800, 0x47814,
2092 0x48000, 0x4800c,
2093 0x48040, 0x48068,
2094 0x48080, 0x48144,
2095 0x48180, 0x4818c,
2096 0x48200, 0x48298,
2097 0x482ac, 0x4833c,
2098 0x483f8, 0x483fc,
c1f49e3e 2099 0x49304, 0x493c4,
251f9e88
SR		 2100 0x49400, 0x4941c,
2101 0x49480, 0x494d0,
2102 0x4c000, 0x4c078,
2103 0x4c0c0, 0x4c278,
2104 0x4c2c0, 0x4c478,
2105 0x4c4c0, 0x4c678,
2106 0x4c6c0, 0x4c878,
2107 0x4c8c0, 0x4c9fc,
2108 0x4d000, 0x4d068,
2109 0x4d080, 0x4d084,
2110 0x4d0a0, 0x4d0b0,
2111 0x4d200, 0x4d268,
2112 0x4d280, 0x4d284,
2113 0x4d2a0, 0x4d2b0,
2114 0x4e0c0, 0x4e0e4,
2115 0x4f000, 0x4f08c,
2116 0x4f200, 0x4f250,
2117 0x4f400, 0x4f420,
2118 0x4f600, 0x4f618,
2119 0x4f800, 0x4f814,
2120 0x50000, 0x500cc,
2121 0x50400, 0x50400,
2122 0x50800, 0x508cc,
2123 0x50c00, 0x50c00,
2124 0x51000, 0x5101c,
2125 0x51300, 0x51308,
2126 };
2127
b8ff05a9
DM		 2128 int i;
2129 struct adapter *ap = netdev2adap(dev);
251f9e88
SR		 2130 static const unsigned int *reg_ranges;
2131 int arr_size = 0, buf_size = 0;
2132
d14807dd 2133 if (is_t4(ap->params.chip)) {
251f9e88
SR		 2134 reg_ranges = &t4_reg_ranges[0];
2135 arr_size = ARRAY_SIZE(t4_reg_ranges);
2136 buf_size = T4_REGMAP_SIZE;
2137 } else {
2138 reg_ranges = &t5_reg_ranges[0];
2139 arr_size = ARRAY_SIZE(t5_reg_ranges);
2140 buf_size = T5_REGMAP_SIZE;
2141 }
b8ff05a9
DM		 2142
2143 regs->version = mk_adap_vers(ap);
2144
251f9e88
SR		 2145 memset(buf, 0, buf_size);
2146 for (i = 0; i < arr_size; i += 2)
b8ff05a9
DM		 2147 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2148}
2149
2150static int restart_autoneg(struct net_device *dev)
2151{
2152 struct port_info *p = netdev_priv(dev);
2153
2154 if (!netif_running(dev))
2155 return -EAGAIN;
2156 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2157 return -EINVAL;
060e0c75 2158 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
b8ff05a9
DM		 2159 return 0;
2160}
2161
c5e06360
DM		 2162static int identify_port(struct net_device *dev,
2163 enum ethtool_phys_id_state state)
b8ff05a9 2164{
c5e06360 2165 unsigned int val;
060e0c75
DM		 2166 struct adapter *adap = netdev2adap(dev);
2167
c5e06360
DM		 2168 if (state == ETHTOOL_ID_ACTIVE)
2169 val = 0xffff;
2170 else if (state == ETHTOOL_ID_INACTIVE)
2171 val = 0;
2172 else
2173 return -EINVAL;
b8ff05a9 2174
c5e06360 2175 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
b8ff05a9
DM		 2176}
2177
2178static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2179{
2180 unsigned int v = 0;
2181
a0881cab
DM		 2182 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2183 type == FW_PORT_TYPE_BT_XAUI) {
b8ff05a9
DM		 2184 v |= SUPPORTED_TP;
2185 if (caps & FW_PORT_CAP_SPEED_100M)
2186 v |= SUPPORTED_100baseT_Full;
2187 if (caps & FW_PORT_CAP_SPEED_1G)
2188 v |= SUPPORTED_1000baseT_Full;
2189 if (caps & FW_PORT_CAP_SPEED_10G)
2190 v |= SUPPORTED_10000baseT_Full;
2191 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2192 v |= SUPPORTED_Backplane;
2193 if (caps & FW_PORT_CAP_SPEED_1G)
2194 v |= SUPPORTED_1000baseKX_Full;
2195 if (caps & FW_PORT_CAP_SPEED_10G)
2196 v |= SUPPORTED_10000baseKX4_Full;
2197 } else if (type == FW_PORT_TYPE_KR)
2198 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
a0881cab 2199 else if (type == FW_PORT_TYPE_BP_AP)
7d5e77aa
DM		 2200 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2201 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2202 else if (type == FW_PORT_TYPE_BP4_AP)
2203 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2204 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2205 SUPPORTED_10000baseKX4_Full;
a0881cab
DM		 2206 else if (type == FW_PORT_TYPE_FIBER_XFI ||
2207 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
b8ff05a9 2208 v |= SUPPORTED_FIBRE;
72aca4bf
KS		 2209 else if (type == FW_PORT_TYPE_BP40_BA)
2210 v |= SUPPORTED_40000baseSR4_Full;
b8ff05a9
DM		 2211
2212 if (caps & FW_PORT_CAP_ANEG)
2213 v |= SUPPORTED_Autoneg;
2214 return v;
2215}
2216
2217static unsigned int to_fw_linkcaps(unsigned int caps)
2218{
2219 unsigned int v = 0;
2220
2221 if (caps & ADVERTISED_100baseT_Full)
2222 v |= FW_PORT_CAP_SPEED_100M;
2223 if (caps & ADVERTISED_1000baseT_Full)
2224 v |= FW_PORT_CAP_SPEED_1G;
2225 if (caps & ADVERTISED_10000baseT_Full)
2226 v |= FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2227 if (caps & ADVERTISED_40000baseSR4_Full)
2228 v |= FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2229 return v;
2230}
2231
2232static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2233{
2234 const struct port_info *p = netdev_priv(dev);
2235
2236 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
a0881cab 2237 p->port_type == FW_PORT_TYPE_BT_XFI ||
b8ff05a9
DM		 2238 p->port_type == FW_PORT_TYPE_BT_XAUI)
2239 cmd->port = PORT_TP;
a0881cab
DM		 2240 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2241 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
b8ff05a9 2242 cmd->port = PORT_FIBRE;
a0881cab
DM		 2243 else if (p->port_type == FW_PORT_TYPE_SFP) {
2244 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2245 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2246 cmd->port = PORT_DA;
2247 else
2248 cmd->port = PORT_FIBRE;
2249 } else
b8ff05a9
DM		 2250 cmd->port = PORT_OTHER;
2251
2252 if (p->mdio_addr >= 0) {
2253 cmd->phy_address = p->mdio_addr;
2254 cmd->transceiver = XCVR_EXTERNAL;
2255 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2256 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2257 } else {
2258 cmd->phy_address = 0; /* not really, but no better option */
2259 cmd->transceiver = XCVR_INTERNAL;
2260 cmd->mdio_support = 0;
2261 }
2262
2263 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2264 cmd->advertising = from_fw_linkcaps(p->port_type,
2265 p->link_cfg.advertising);
70739497
DD		 2266 ethtool_cmd_speed_set(cmd,
2267 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
b8ff05a9
DM		 2268 cmd->duplex = DUPLEX_FULL;
2269 cmd->autoneg = p->link_cfg.autoneg;
2270 cmd->maxtxpkt = 0;
2271 cmd->maxrxpkt = 0;
2272 return 0;
2273}
2274
2275static unsigned int speed_to_caps(int speed)
2276{
2277 if (speed == SPEED_100)
2278 return FW_PORT_CAP_SPEED_100M;
2279 if (speed == SPEED_1000)
2280 return FW_PORT_CAP_SPEED_1G;
2281 if (speed == SPEED_10000)
2282 return FW_PORT_CAP_SPEED_10G;
72aca4bf
KS		 2283 if (speed == 40000) /* Need SPEED_40000 in ethtool.h */
2284 return FW_PORT_CAP_SPEED_40G;
b8ff05a9
DM		 2285 return 0;
2286}
2287
2288static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2289{
2290 unsigned int cap;
2291 struct port_info *p = netdev_priv(dev);
2292 struct link_config *lc = &p->link_cfg;
25db0338 2293 u32 speed = ethtool_cmd_speed(cmd);
b8ff05a9
DM		 2294
2295 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
2296 return -EINVAL;
2297
2298 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2299 /*
2300 * PHY offers a single speed. See if that's what's
2301 * being requested.
2302 */
2303 if (cmd->autoneg == AUTONEG_DISABLE &&
25db0338
DD		 2304 (lc->supported & speed_to_caps(speed)))
2305 return 0;
b8ff05a9
DM		 2306 return -EINVAL;
2307 }
2308
2309 if (cmd->autoneg == AUTONEG_DISABLE) {
25db0338 2310 cap = speed_to_caps(speed);
b8ff05a9 2311
72aca4bf
KS		 2312 if (!(lc->supported & cap) ||
2313 (speed == SPEED_1000) ||
2314 (speed == SPEED_10000) ||
2315 (speed == 40000))
b8ff05a9
DM		 2316 return -EINVAL;
2317 lc->requested_speed = cap;
2318 lc->advertising = 0;
2319 } else {
2320 cap = to_fw_linkcaps(cmd->advertising);
2321 if (!(lc->supported & cap))
2322 return -EINVAL;
2323 lc->requested_speed = 0;
2324 lc->advertising = cap | FW_PORT_CAP_ANEG;
2325 }
2326 lc->autoneg = cmd->autoneg;
2327
2328 if (netif_running(dev))
060e0c75
DM		 2329 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2330 lc);
b8ff05a9
DM		 2331 return 0;
2332}
2333
2334static void get_pauseparam(struct net_device *dev,
2335 struct ethtool_pauseparam *epause)
2336{
2337 struct port_info *p = netdev_priv(dev);
2338
2339 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2340 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2341 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2342}
2343
2344static int set_pauseparam(struct net_device *dev,
2345 struct ethtool_pauseparam *epause)
2346{
2347 struct port_info *p = netdev_priv(dev);
2348 struct link_config *lc = &p->link_cfg;
2349
2350 if (epause->autoneg == AUTONEG_DISABLE)
2351 lc->requested_fc = 0;
2352 else if (lc->supported & FW_PORT_CAP_ANEG)
2353 lc->requested_fc = PAUSE_AUTONEG;
2354 else
2355 return -EINVAL;
2356
2357 if (epause->rx_pause)
2358 lc->requested_fc |= PAUSE_RX;
2359 if (epause->tx_pause)
2360 lc->requested_fc |= PAUSE_TX;
2361 if (netif_running(dev))
060e0c75
DM		 2362 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2363 lc);
b8ff05a9
DM		 2364 return 0;
2365}
2366
b8ff05a9
DM		 2367static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2368{
2369 const struct port_info *pi = netdev_priv(dev);
2370 const struct sge *s = &pi->adapter->sge;
2371
2372 e->rx_max_pending = MAX_RX_BUFFERS;
2373 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2374 e->rx_jumbo_max_pending = 0;
2375 e->tx_max_pending = MAX_TXQ_ENTRIES;
2376
2377 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2378 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2379 e->rx_jumbo_pending = 0;
2380 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2381}
2382
2383static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2384{
2385 int i;
2386 const struct port_info *pi = netdev_priv(dev);
2387 struct adapter *adapter = pi->adapter;
2388 struct sge *s = &adapter->sge;
2389
2390 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2391 e->tx_pending > MAX_TXQ_ENTRIES ||
2392 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2393 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2394 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2395 return -EINVAL;
2396
2397 if (adapter->flags & FULL_INIT_DONE)
2398 return -EBUSY;
2399
2400 for (i = 0; i < pi->nqsets; ++i) {
2401 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2402 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2403 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2404 }
2405 return 0;
2406}
2407
2408static int closest_timer(const struct sge *s, int time)
2409{
2410 int i, delta, match = 0, min_delta = INT_MAX;
2411
2412 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2413 delta = time - s->timer_val[i];
2414 if (delta < 0)
2415 delta = -delta;
2416 if (delta < min_delta) {
2417 min_delta = delta;
2418 match = i;
2419 }
2420 }
2421 return match;
2422}
2423
2424static int closest_thres(const struct sge *s, int thres)
2425{
2426 int i, delta, match = 0, min_delta = INT_MAX;
2427
2428 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2429 delta = thres - s->counter_val[i];
2430 if (delta < 0)
2431 delta = -delta;
2432 if (delta < min_delta) {
2433 min_delta = delta;
2434 match = i;
2435 }
2436 }
2437 return match;
2438}
2439
2440/*
2441 * Return a queue's interrupt hold-off time in us. 0 means no timer.
2442 */
2443static unsigned int qtimer_val(const struct adapter *adap,
2444 const struct sge_rspq *q)
2445{
2446 unsigned int idx = q->intr_params >> 1;
2447
2448 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2449}
2450
2451/**
2452 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
2453 * @adap: the adapter
2454 * @q: the Rx queue
2455 * @us: the hold-off time in us, or 0 to disable timer
2456 * @cnt: the hold-off packet count, or 0 to disable counter
2457 *
2458 * Sets an Rx queue's interrupt hold-off time and packet count. At least
2459 * one of the two needs to be enabled for the queue to generate interrupts.
2460 */
2461static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2462 unsigned int us, unsigned int cnt)
2463{
2464 if ((us | cnt) == 0)
2465 cnt = 1;
2466
2467 if (cnt) {
2468 int err;
2469 u32 v, new_idx;
2470
2471 new_idx = closest_thres(&adap->sge, cnt);
2472 if (q->desc && q->pktcnt_idx != new_idx) {
2473 /* the queue has already been created, update it */
2474 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2475 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2476 FW_PARAMS_PARAM_YZ(q->cntxt_id);
060e0c75
DM		 2477 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2478 &new_idx);
b8ff05a9
DM		 2479 if (err)
2480 return err;
2481 }
2482 q->pktcnt_idx = new_idx;
2483 }
2484
2485 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2486 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2487 return 0;
2488}
2489
2490static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2491{
2492 const struct port_info *pi = netdev_priv(dev);
2493 struct adapter *adap = pi->adapter;
d4fc9dc2
TLSC		 2494 struct sge_rspq *q;
2495 int i;
2496 int r = 0;
2497
2498 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2499 q = &adap->sge.ethrxq[i].rspq;
2500 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2501 c->rx_max_coalesced_frames);
2502 if (r) {
2503 dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2504 break;
2505 }
2506 }
2507 return r;
b8ff05a9
DM		 2508}
2509
2510static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2511{
2512 const struct port_info *pi = netdev_priv(dev);
2513 const struct adapter *adap = pi->adapter;
2514 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2515
2516 c->rx_coalesce_usecs = qtimer_val(adap, rq);
2517 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2518 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2519 return 0;
2520}
2521
1478b3ee
DM		 2522/**
2523 * eeprom_ptov - translate a physical EEPROM address to virtual
2524 * @phys_addr: the physical EEPROM address
2525 * @fn: the PCI function number
2526 * @sz: size of function-specific area
2527 *
2528 * Translate a physical EEPROM address to virtual. The first 1K is
2529 * accessed through virtual addresses starting at 31K, the rest is
2530 * accessed through virtual addresses starting at 0.
2531 *
2532 * The mapping is as follows:
2533 * [0..1K) -> [31K..32K)
2534 * [1K..1K+A) -> [31K-A..31K)
2535 * [1K+A..ES) -> [0..ES-A-1K)
2536 *
2537 * where A = @fn * @sz, and ES = EEPROM size.
b8ff05a9 2538 */
1478b3ee 2539static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
b8ff05a9 2540{
1478b3ee 2541 fn *= sz;
b8ff05a9
DM		 2542 if (phys_addr < 1024)
2543 return phys_addr + (31 << 10);
1478b3ee
DM		 2544 if (phys_addr < 1024 + fn)
2545 return 31744 - fn + phys_addr - 1024;
b8ff05a9 2546 if (phys_addr < EEPROMSIZE)
1478b3ee 2547 return phys_addr - 1024 - fn;
b8ff05a9
DM		 2548 return -EINVAL;
2549}
2550
2551/*
2552 * The next two routines implement eeprom read/write from physical addresses.
b8ff05a9
DM		 2553 */
2554static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2555{
1478b3ee 2556 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2557
2558 if (vaddr >= 0)
2559 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2560 return vaddr < 0 ? vaddr : 0;
2561}
2562
2563static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2564{
1478b3ee 2565 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
b8ff05a9
DM		 2566
2567 if (vaddr >= 0)
2568 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2569 return vaddr < 0 ? vaddr : 0;
2570}
2571
2572#define EEPROM_MAGIC 0x38E2F10C
2573
2574static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2575 u8 *data)
2576{
2577 int i, err = 0;
2578 struct adapter *adapter = netdev2adap(dev);
2579
2580 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2581 if (!buf)
2582 return -ENOMEM;
2583
2584 e->magic = EEPROM_MAGIC;
2585 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2586 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2587
2588 if (!err)
2589 memcpy(data, buf + e->offset, e->len);
2590 kfree(buf);
2591 return err;
2592}
2593
2594static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2595 u8 *data)
2596{
2597 u8 *buf;
2598 int err = 0;
2599 u32 aligned_offset, aligned_len, *p;
2600 struct adapter *adapter = netdev2adap(dev);
2601
2602 if (eeprom->magic != EEPROM_MAGIC)
2603 return -EINVAL;
2604
2605 aligned_offset = eeprom->offset & ~3;
2606 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2607
1478b3ee
DM		 2608 if (adapter->fn > 0) {
2609 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2610
2611 if (aligned_offset < start ||
2612 aligned_offset + aligned_len > start + EEPROMPFSIZE)
2613 return -EPERM;
2614 }
2615
b8ff05a9
DM		 2616 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2617 /*
2618 * RMW possibly needed for first or last words.
2619 */
2620 buf = kmalloc(aligned_len, GFP_KERNEL);
2621 if (!buf)
2622 return -ENOMEM;
2623 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2624 if (!err && aligned_len > 4)
2625 err = eeprom_rd_phys(adapter,
2626 aligned_offset + aligned_len - 4,
2627 (u32 *)&buf[aligned_len - 4]);
2628 if (err)
2629 goto out;
2630 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2631 } else
2632 buf = data;
2633
2634 err = t4_seeprom_wp(adapter, false);
2635 if (err)
2636 goto out;
2637
2638 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2639 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2640 aligned_offset += 4;
2641 }
2642
2643 if (!err)
2644 err = t4_seeprom_wp(adapter, true);
2645out:
2646 if (buf != data)
2647 kfree(buf);
2648 return err;
2649}
2650
2651static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2652{
2653 int ret;
2654 const struct firmware *fw;
2655 struct adapter *adap = netdev2adap(netdev);
2656
2657 ef->data[sizeof(ef->data) - 1] = '\0';
2658 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2659 if (ret < 0)
2660 return ret;
2661
2662 ret = t4_load_fw(adap, fw->data, fw->size);
2663 release_firmware(fw);
2664 if (!ret)
2665 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2666 return ret;
2667}
2668
2669#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2670#define BCAST_CRC 0xa0ccc1a6
2671
2672static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2673{
2674 wol->supported = WAKE_BCAST | WAKE_MAGIC;
2675 wol->wolopts = netdev2adap(dev)->wol;
2676 memset(&wol->sopass, 0, sizeof(wol->sopass));
2677}
2678
2679static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2680{
2681 int err = 0;
2682 struct port_info *pi = netdev_priv(dev);
2683
2684 if (wol->wolopts & ~WOL_SUPPORTED)
2685 return -EINVAL;
2686 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2687 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2688 if (wol->wolopts & WAKE_BCAST) {
2689 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2690 ~0ULL, 0, false);
2691 if (!err)
2692 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2693 ~6ULL, ~0ULL, BCAST_CRC, true);
2694 } else
2695 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2696 return err;
2697}
2698
c8f44aff 2699static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
87b6cf51 2700{
2ed28baa 2701 const struct port_info *pi = netdev_priv(dev);
c8f44aff 2702 netdev_features_t changed = dev->features ^ features;
19ecae2c 2703 int err;
19ecae2c 2704
f646968f 2705 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2ed28baa 2706 return 0;
19ecae2c 2707
2ed28baa
MM		 2708 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2709 -1, -1, -1,
f646968f 2710 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2ed28baa 2711 if (unlikely(err))
f646968f 2712 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
19ecae2c 2713 return err;
87b6cf51
DM		 2714}
2715
7850f63f 2716static u32 get_rss_table_size(struct net_device *dev)
671b0060
DM		 2717{
2718 const struct port_info *pi = netdev_priv(dev);
671b0060 2719
7850f63f
BH		 2720 return pi->rss_size;
2721}
2722
2723static int get_rss_table(struct net_device *dev, u32 *p)
2724{
2725 const struct port_info *pi = netdev_priv(dev);
2726 unsigned int n = pi->rss_size;
2727
671b0060 2728 while (n--)
7850f63f 2729 p[n] = pi->rss[n];
671b0060
DM		 2730 return 0;
2731}
2732
7850f63f 2733static int set_rss_table(struct net_device *dev, const u32 *p)
671b0060
DM		 2734{
2735 unsigned int i;
2736 struct port_info *pi = netdev_priv(dev);
2737
7850f63f
BH		 2738 for (i = 0; i < pi->rss_size; i++)
2739 pi->rss[i] = p[i];
671b0060
DM		 2740 if (pi->adapter->flags & FULL_INIT_DONE)
2741 return write_rss(pi, pi->rss);
2742 return 0;
2743}
2744
2745static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
815c7db5 2746 u32 *rules)
671b0060 2747{
f796564a
DM		 2748 const struct port_info *pi = netdev_priv(dev);
2749
671b0060 2750 switch (info->cmd) {
f796564a
DM		 2751 case ETHTOOL_GRXFH: {
2752 unsigned int v = pi->rss_mode;
2753
2754 info->data = 0;
2755 switch (info->flow_type) {
2756 case TCP_V4_FLOW:
2757 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2758 info->data = RXH_IP_SRC | RXH_IP_DST |
2759 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2760 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2761 info->data = RXH_IP_SRC | RXH_IP_DST;
2762 break;
2763 case UDP_V4_FLOW:
2764 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2765 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2766 info->data = RXH_IP_SRC | RXH_IP_DST |
2767 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2768 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2769 info->data = RXH_IP_SRC | RXH_IP_DST;
2770 break;
2771 case SCTP_V4_FLOW:
2772 case AH_ESP_V4_FLOW:
2773 case IPV4_FLOW:
2774 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2775 info->data = RXH_IP_SRC | RXH_IP_DST;
2776 break;
2777 case TCP_V6_FLOW:
2778 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2779 info->data = RXH_IP_SRC | RXH_IP_DST |
2780 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2781 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2782 info->data = RXH_IP_SRC | RXH_IP_DST;
2783 break;
2784 case UDP_V6_FLOW:
2785 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2786 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2787 info->data = RXH_IP_SRC | RXH_IP_DST |
2788 RXH_L4_B_0_1 | RXH_L4_B_2_3;
2789 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2790 info->data = RXH_IP_SRC | RXH_IP_DST;
2791 break;
2792 case SCTP_V6_FLOW:
2793 case AH_ESP_V6_FLOW:
2794 case IPV6_FLOW:
2795 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2796 info->data = RXH_IP_SRC | RXH_IP_DST;
2797 break;
2798 }
2799 return 0;
2800 }
671b0060 2801 case ETHTOOL_GRXRINGS:
f796564a 2802 info->data = pi->nqsets;
671b0060
DM		 2803 return 0;
2804 }
2805 return -EOPNOTSUPP;
2806}
2807
9b07be4b 2808static const struct ethtool_ops cxgb_ethtool_ops = {
b8ff05a9
DM		 2809 .get_settings = get_settings,
2810 .set_settings = set_settings,
2811 .get_drvinfo = get_drvinfo,
2812 .get_msglevel = get_msglevel,
2813 .set_msglevel = set_msglevel,
2814 .get_ringparam = get_sge_param,
2815 .set_ringparam = set_sge_param,
2816 .get_coalesce = get_coalesce,
2817 .set_coalesce = set_coalesce,
2818 .get_eeprom_len = get_eeprom_len,
2819 .get_eeprom = get_eeprom,
2820 .set_eeprom = set_eeprom,
2821 .get_pauseparam = get_pauseparam,
2822 .set_pauseparam = set_pauseparam,
b8ff05a9
DM		 2823 .get_link = ethtool_op_get_link,
2824 .get_strings = get_strings,
c5e06360 2825 .set_phys_id = identify_port,
b8ff05a9
DM		 2826 .nway_reset = restart_autoneg,
2827 .get_sset_count = get_sset_count,
2828 .get_ethtool_stats = get_stats,
2829 .get_regs_len = get_regs_len,
2830 .get_regs = get_regs,
2831 .get_wol = get_wol,
2832 .set_wol = set_wol,
671b0060 2833 .get_rxnfc = get_rxnfc,
7850f63f 2834 .get_rxfh_indir_size = get_rss_table_size,
671b0060
DM		 2835 .get_rxfh_indir = get_rss_table,
2836 .set_rxfh_indir = set_rss_table,
b8ff05a9
DM		 2837 .flash_device = set_flash,
2838};
2839
2840/*
2841 * debugfs support
2842 */
b8ff05a9
DM		 2843static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2844 loff_t *ppos)
2845{
2846 loff_t pos = *ppos;
496ad9aa 2847 loff_t avail = file_inode(file)->i_size;
b8ff05a9
DM		 2848 unsigned int mem = (uintptr_t)file->private_data & 3;
2849 struct adapter *adap = file->private_data - mem;
2850
2851 if (pos < 0)
2852 return -EINVAL;
2853 if (pos >= avail)
2854 return 0;
2855 if (count > avail - pos)
2856 count = avail - pos;
2857
2858 while (count) {
2859 size_t len;
2860 int ret, ofst;
2861 __be32 data[16];
2862
19dd37ba
SR		 2863 if ((mem == MEM_MC) || (mem == MEM_MC1))
2864 ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
b8ff05a9
DM		 2865 else
2866 ret = t4_edc_read(adap, mem, pos, data, NULL);
2867 if (ret)
2868 return ret;
2869
2870 ofst = pos % sizeof(data);
2871 len = min(count, sizeof(data) - ofst);
2872 if (copy_to_user(buf, (u8 *)data + ofst, len))
2873 return -EFAULT;
2874
2875 buf += len;
2876 pos += len;
2877 count -= len;
2878 }
2879 count = pos - *ppos;
2880 *ppos = pos;
2881 return count;
2882}
2883
2884static const struct file_operations mem_debugfs_fops = {
2885 .owner = THIS_MODULE,
234e3405 2886 .open = simple_open,
b8ff05a9 2887 .read = mem_read,
6038f373 2888 .llseek = default_llseek,
b8ff05a9
DM		 2889};
2890
91744948 2891static void add_debugfs_mem(struct adapter *adap, const char *name,
1dd06ae8 2892 unsigned int idx, unsigned int size_mb)
b8ff05a9
DM		 2893{
2894 struct dentry *de;
2895
2896 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2897 (void *)adap + idx, &mem_debugfs_fops);
2898 if (de && de->d_inode)
2899 de->d_inode->i_size = size_mb << 20;
2900}
2901
91744948 2902static int setup_debugfs(struct adapter *adap)
b8ff05a9
DM		 2903{
2904 int i;
19dd37ba 2905 u32 size;
b8ff05a9
DM		 2906
2907 if (IS_ERR_OR_NULL(adap->debugfs_root))
2908 return -1;
2909
2910 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
19dd37ba
SR		 2911 if (i & EDRAM0_ENABLE) {
2912 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2913 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2914 }
2915 if (i & EDRAM1_ENABLE) {
2916 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2917 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2918 }
d14807dd 2919 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 2920 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2921 if (i & EXT_MEM_ENABLE)
2922 add_debugfs_mem(adap, "mc", MEM_MC,
2923 EXT_MEM_SIZE_GET(size));
2924 } else {
2925 if (i & EXT_MEM_ENABLE) {
2926 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2927 add_debugfs_mem(adap, "mc0", MEM_MC0,
2928 EXT_MEM_SIZE_GET(size));
2929 }
2930 if (i & EXT_MEM1_ENABLE) {
2931 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2932 add_debugfs_mem(adap, "mc1", MEM_MC1,
2933 EXT_MEM_SIZE_GET(size));
2934 }
2935 }
b8ff05a9
DM		 2936 if (adap->l2t)
2937 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2938 &t4_l2t_fops);
2939 return 0;
2940}
2941
2942/*
2943 * upper-layer driver support
2944 */
2945
2946/*
2947 * Allocate an active-open TID and set it to the supplied value.
2948 */
2949int cxgb4_alloc_atid(struct tid_info *t, void *data)
2950{
2951 int atid = -1;
2952
2953 spin_lock_bh(&t->atid_lock);
2954 if (t->afree) {
2955 union aopen_entry *p = t->afree;
2956
f2b7e78d 2957 atid = (p - t->atid_tab) + t->atid_base;
b8ff05a9
DM		 2958 t->afree = p->next;
2959 p->data = data;
2960 t->atids_in_use++;
2961 }
2962 spin_unlock_bh(&t->atid_lock);
2963 return atid;
2964}
2965EXPORT_SYMBOL(cxgb4_alloc_atid);
2966
2967/*
2968 * Release an active-open TID.
2969 */
2970void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2971{
f2b7e78d 2972 union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
b8ff05a9
DM		 2973
2974 spin_lock_bh(&t->atid_lock);
2975 p->next = t->afree;
2976 t->afree = p;
2977 t->atids_in_use--;
2978 spin_unlock_bh(&t->atid_lock);
2979}
2980EXPORT_SYMBOL(cxgb4_free_atid);
2981
2982/*
2983 * Allocate a server TID and set it to the supplied value.
2984 */
2985int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2986{
2987 int stid;
2988
2989 spin_lock_bh(&t->stid_lock);
2990 if (family == PF_INET) {
2991 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2992 if (stid < t->nstids)
2993 __set_bit(stid, t->stid_bmap);
2994 else
2995 stid = -1;
2996 } else {
2997 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2998 if (stid < 0)
2999 stid = -1;
3000 }
3001 if (stid >= 0) {
3002 t->stid_tab[stid].data = data;
3003 stid += t->stid_base;
15f63b74
KS		 3004 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3005 * This is equivalent to 4 TIDs. With CLIP enabled it
3006 * needs 2 TIDs.
3007 */
3008 if (family == PF_INET)
3009 t->stids_in_use++;
3010 else
3011 t->stids_in_use += 4;
b8ff05a9
DM		 3012 }
3013 spin_unlock_bh(&t->stid_lock);
3014 return stid;
3015}
3016EXPORT_SYMBOL(cxgb4_alloc_stid);
3017
dca4faeb
VP		 3018/* Allocate a server filter TID and set it to the supplied value.
3019 */
3020int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3021{
3022 int stid;
3023
3024 spin_lock_bh(&t->stid_lock);
3025 if (family == PF_INET) {
3026 stid = find_next_zero_bit(t->stid_bmap,
3027 t->nstids + t->nsftids, t->nstids);
3028 if (stid < (t->nstids + t->nsftids))
3029 __set_bit(stid, t->stid_bmap);
3030 else
3031 stid = -1;
3032 } else {
3033 stid = -1;
3034 }
3035 if (stid >= 0) {
3036 t->stid_tab[stid].data = data;
470c60c4
KS		 3037 stid -= t->nstids;
3038 stid += t->sftid_base;
dca4faeb
VP		 3039 t->stids_in_use++;
3040 }
3041 spin_unlock_bh(&t->stid_lock);
3042 return stid;
3043}
3044EXPORT_SYMBOL(cxgb4_alloc_sftid);
3045
3046/* Release a server TID.
b8ff05a9
DM		 3047 */
3048void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3049{
470c60c4
KS		 3050 /* Is it a server filter TID? */
3051 if (t->nsftids && (stid >= t->sftid_base)) {
3052 stid -= t->sftid_base;
3053 stid += t->nstids;
3054 } else {
3055 stid -= t->stid_base;
3056 }
3057
b8ff05a9
DM		 3058 spin_lock_bh(&t->stid_lock);
3059 if (family == PF_INET)
3060 __clear_bit(stid, t->stid_bmap);
3061 else
3062 bitmap_release_region(t->stid_bmap, stid, 2);
3063 t->stid_tab[stid].data = NULL;
15f63b74
KS		 3064 if (family == PF_INET)
3065 t->stids_in_use--;
3066 else
3067 t->stids_in_use -= 4;
b8ff05a9
DM		 3068 spin_unlock_bh(&t->stid_lock);
3069}
3070EXPORT_SYMBOL(cxgb4_free_stid);
3071
3072/*
3073 * Populate a TID_RELEASE WR. Caller must properly size the skb.
3074 */
3075static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3076 unsigned int tid)
3077{
3078 struct cpl_tid_release *req;
3079
3080 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3081 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3082 INIT_TP_WR(req, tid);
3083 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3084}
3085
3086/*
3087 * Queue a TID release request and if necessary schedule a work queue to
3088 * process it.
3089 */
31b9c19b 3090static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3091 unsigned int tid)
b8ff05a9
DM		 3092{
3093 void **p = &t->tid_tab[tid];
3094 struct adapter *adap = container_of(t, struct adapter, tids);
3095
3096 spin_lock_bh(&adap->tid_release_lock);
3097 *p = adap->tid_release_head;
3098 /* Low 2 bits encode the Tx channel number */
3099 adap->tid_release_head = (void **)((uintptr_t)p | chan);
3100 if (!adap->tid_release_task_busy) {
3101 adap->tid_release_task_busy = true;
3069ee9b 3102 queue_work(workq, &adap->tid_release_task);
b8ff05a9
DM		 3103 }
3104 spin_unlock_bh(&adap->tid_release_lock);
3105}
b8ff05a9
DM		 3106
3107/*
3108 * Process the list of pending TID release requests.
3109 */
3110static void process_tid_release_list(struct work_struct *work)
3111{
3112 struct sk_buff *skb;
3113 struct adapter *adap;
3114
3115 adap = container_of(work, struct adapter, tid_release_task);
3116
3117 spin_lock_bh(&adap->tid_release_lock);
3118 while (adap->tid_release_head) {
3119 void **p = adap->tid_release_head;
3120 unsigned int chan = (uintptr_t)p & 3;
3121 p = (void *)p - chan;
3122
3123 adap->tid_release_head = *p;
3124 *p = NULL;
3125 spin_unlock_bh(&adap->tid_release_lock);
3126
3127 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3128 GFP_KERNEL)))
3129 schedule_timeout_uninterruptible(1);
3130
3131 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3132 t4_ofld_send(adap, skb);
3133 spin_lock_bh(&adap->tid_release_lock);
3134 }
3135 adap->tid_release_task_busy = false;
3136 spin_unlock_bh(&adap->tid_release_lock);
3137}
3138
3139/*
3140 * Release a TID and inform HW. If we are unable to allocate the release
3141 * message we defer to a work queue.
3142 */
3143void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3144{
3145 void *old;
3146 struct sk_buff *skb;
3147 struct adapter *adap = container_of(t, struct adapter, tids);
3148
3149 old = t->tid_tab[tid];
3150 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3151 if (likely(skb)) {
3152 t->tid_tab[tid] = NULL;
3153 mk_tid_release(skb, chan, tid);
3154 t4_ofld_send(adap, skb);
3155 } else
3156 cxgb4_queue_tid_release(t, chan, tid);
3157 if (old)
3158 atomic_dec(&t->tids_in_use);
3159}
3160EXPORT_SYMBOL(cxgb4_remove_tid);
3161
3162/*
3163 * Allocate and initialize the TID tables. Returns 0 on success.
3164 */
3165static int tid_init(struct tid_info *t)
3166{
3167 size_t size;
f2b7e78d 3168 unsigned int stid_bmap_size;
b8ff05a9 3169 unsigned int natids = t->natids;
b6f8eaec 3170 struct adapter *adap = container_of(t, struct adapter, tids);
b8ff05a9 3171
dca4faeb 3172 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
f2b7e78d
VP		 3173 size = t->ntids * sizeof(*t->tid_tab) +
3174 natids * sizeof(*t->atid_tab) +
b8ff05a9 3175 t->nstids * sizeof(*t->stid_tab) +
dca4faeb 3176 t->nsftids * sizeof(*t->stid_tab) +
f2b7e78d 3177 stid_bmap_size * sizeof(long) +
dca4faeb
VP		 3178 t->nftids * sizeof(*t->ftid_tab) +
3179 t->nsftids * sizeof(*t->ftid_tab);
f2b7e78d 3180
b8ff05a9
DM		 3181 t->tid_tab = t4_alloc_mem(size);
3182 if (!t->tid_tab)
3183 return -ENOMEM;
3184
3185 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3186 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
dca4faeb 3187 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
f2b7e78d 3188 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
b8ff05a9
DM		 3189 spin_lock_init(&t->stid_lock);
3190 spin_lock_init(&t->atid_lock);
3191
3192 t->stids_in_use = 0;
3193 t->afree = NULL;
3194 t->atids_in_use = 0;
3195 atomic_set(&t->tids_in_use, 0);
3196
3197 /* Setup the free list for atid_tab and clear the stid bitmap. */
3198 if (natids) {
3199 while (--natids)
3200 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3201 t->afree = t->atid_tab;
3202 }
dca4faeb 3203 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
b6f8eaec
KS		 3204 /* Reserve stid 0 for T4/T5 adapters */
3205 if (!t->stid_base &&
3206 (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3207 __set_bit(0, t->stid_bmap);
3208
b8ff05a9
DM		 3209 return 0;
3210}
3211
01bcca68
VP		 3212static int cxgb4_clip_get(const struct net_device *dev,
3213 const struct in6_addr *lip)
3214{
3215 struct adapter *adap;
3216 struct fw_clip_cmd c;
3217
3218 adap = netdev2adap(dev);
3219 memset(&c, 0, sizeof(c));
3220 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3221 FW_CMD_REQUEST | FW_CMD_WRITE);
3222 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_ALLOC | FW_LEN16(c));
3223 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3224 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3225 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3226}
3227
3228static int cxgb4_clip_release(const struct net_device *dev,
3229 const struct in6_addr *lip)
3230{
3231 struct adapter *adap;
3232 struct fw_clip_cmd c;
3233
3234 adap = netdev2adap(dev);
3235 memset(&c, 0, sizeof(c));
3236 c.op_to_write = htonl(FW_CMD_OP(FW_CLIP_CMD) |
3237 FW_CMD_REQUEST | FW_CMD_READ);
3238 c.alloc_to_len16 = htonl(F_FW_CLIP_CMD_FREE | FW_LEN16(c));
3239 *(__be64 *)&c.ip_hi = *(__be64 *)(lip->s6_addr);
3240 *(__be64 *)&c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3241 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3242}
3243
b8ff05a9
DM		 3244/**
3245 * cxgb4_create_server - create an IP server
3246 * @dev: the device
3247 * @stid: the server TID
3248 * @sip: local IP address to bind server to
3249 * @sport: the server's TCP port
3250 * @queue: queue to direct messages from this server to
3251 *
3252 * Create an IP server for the given port and address.
3253 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3254 */
3255int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
793dad94
VP		 3256 __be32 sip, __be16 sport, __be16 vlan,
3257 unsigned int queue)
b8ff05a9
DM		 3258{
3259 unsigned int chan;
3260 struct sk_buff *skb;
3261 struct adapter *adap;
3262 struct cpl_pass_open_req *req;
80f40c1f 3263 int ret;
b8ff05a9
DM		 3264
3265 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3266 if (!skb)
3267 return -ENOMEM;
3268
3269 adap = netdev2adap(dev);
3270 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3271 INIT_TP_WR(req, 0);
3272 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3273 req->local_port = sport;
3274 req->peer_port = htons(0);
3275 req->local_ip = sip;
3276 req->peer_ip = htonl(0);
e46dab4d 3277 chan = rxq_to_chan(&adap->sge, queue);
b8ff05a9
DM		 3278 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3279 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3280 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
80f40c1f
VP		 3281 ret = t4_mgmt_tx(adap, skb);
3282 return net_xmit_eval(ret);
b8ff05a9
DM		 3283}
3284EXPORT_SYMBOL(cxgb4_create_server);
3285
80f40c1f
VP		 3286/* cxgb4_create_server6 - create an IPv6 server
3287 * @dev: the device
3288 * @stid: the server TID
3289 * @sip: local IPv6 address to bind server to
3290 * @sport: the server's TCP port
3291 * @queue: queue to direct messages from this server to
3292 *
3293 * Create an IPv6 server for the given port and address.
3294 * Returns <0 on error and one of the %NET_XMIT_* values on success.
3295 */
3296int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3297 const struct in6_addr *sip, __be16 sport,
3298 unsigned int queue)
3299{
3300 unsigned int chan;
3301 struct sk_buff *skb;
3302 struct adapter *adap;
3303 struct cpl_pass_open_req6 *req;
3304 int ret;
3305
3306 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3307 if (!skb)
3308 return -ENOMEM;
3309
3310 adap = netdev2adap(dev);
3311 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3312 INIT_TP_WR(req, 0);
3313 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3314 req->local_port = sport;
3315 req->peer_port = htons(0);
3316 req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3317 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3318 req->peer_ip_hi = cpu_to_be64(0);
3319 req->peer_ip_lo = cpu_to_be64(0);
3320 chan = rxq_to_chan(&adap->sge, queue);
3321 req->opt0 = cpu_to_be64(TX_CHAN(chan));
3322 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3323 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3324 ret = t4_mgmt_tx(adap, skb);
3325 return net_xmit_eval(ret);
3326}
3327EXPORT_SYMBOL(cxgb4_create_server6);
3328
3329int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3330 unsigned int queue, bool ipv6)
3331{
3332 struct sk_buff *skb;
3333 struct adapter *adap;
3334 struct cpl_close_listsvr_req *req;
3335 int ret;
3336
3337 adap = netdev2adap(dev);
3338
3339 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3340 if (!skb)
3341 return -ENOMEM;
3342
3343 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3344 INIT_TP_WR(req, 0);
3345 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3346 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3347 LISTSVR_IPV6(0)) | QUEUENO(queue));
3348 ret = t4_mgmt_tx(adap, skb);
3349 return net_xmit_eval(ret);
3350}
3351EXPORT_SYMBOL(cxgb4_remove_server);
3352
b8ff05a9
DM		 3353/**
3354 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3355 * @mtus: the HW MTU table
3356 * @mtu: the target MTU
3357 * @idx: index of selected entry in the MTU table
3358 *
3359 * Returns the index and the value in the HW MTU table that is closest to
3360 * but does not exceed @mtu, unless @mtu is smaller than any value in the
3361 * table, in which case that smallest available value is selected.
3362 */
3363unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3364 unsigned int *idx)
3365{
3366 unsigned int i = 0;
3367
3368 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3369 ++i;
3370 if (idx)
3371 *idx = i;
3372 return mtus[i];
3373}
3374EXPORT_SYMBOL(cxgb4_best_mtu);
3375
3376/**
3377 * cxgb4_port_chan - get the HW channel of a port
3378 * @dev: the net device for the port
3379 *
3380 * Return the HW Tx channel of the given port.
3381 */
3382unsigned int cxgb4_port_chan(const struct net_device *dev)
3383{
3384 return netdev2pinfo(dev)->tx_chan;
3385}
3386EXPORT_SYMBOL(cxgb4_port_chan);
3387
881806bc
VP		 3388unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3389{
3390 struct adapter *adap = netdev2adap(dev);
2cc301d2 3391 u32 v1, v2, lp_count, hp_count;
881806bc 3392
2cc301d2
SR		 3393 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3394 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3395 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3396 lp_count = G_LP_COUNT(v1);
3397 hp_count = G_HP_COUNT(v1);
3398 } else {
3399 lp_count = G_LP_COUNT_T5(v1);
3400 hp_count = G_HP_COUNT_T5(v2);
3401 }
3402 return lpfifo ? lp_count : hp_count;
881806bc
VP		 3403}
3404EXPORT_SYMBOL(cxgb4_dbfifo_count);
3405
b8ff05a9
DM		 3406/**
3407 * cxgb4_port_viid - get the VI id of a port
3408 * @dev: the net device for the port
3409 *
3410 * Return the VI id of the given port.
3411 */
3412unsigned int cxgb4_port_viid(const struct net_device *dev)
3413{
3414 return netdev2pinfo(dev)->viid;
3415}
3416EXPORT_SYMBOL(cxgb4_port_viid);
3417
3418/**
3419 * cxgb4_port_idx - get the index of a port
3420 * @dev: the net device for the port
3421 *
3422 * Return the index of the given port.
3423 */
3424unsigned int cxgb4_port_idx(const struct net_device *dev)
3425{
3426 return netdev2pinfo(dev)->port_id;
3427}
3428EXPORT_SYMBOL(cxgb4_port_idx);
3429
b8ff05a9
DM		 3430void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3431 struct tp_tcp_stats *v6)
3432{
3433 struct adapter *adap = pci_get_drvdata(pdev);
3434
3435 spin_lock(&adap->stats_lock);
3436 t4_tp_get_tcp_stats(adap, v4, v6);
3437 spin_unlock(&adap->stats_lock);
3438}
3439EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3440
3441void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3442 const unsigned int *pgsz_order)
3443{
3444 struct adapter *adap = netdev2adap(dev);
3445
3446 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3447 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3448 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3449 HPZ3(pgsz_order[3]));
3450}
3451EXPORT_SYMBOL(cxgb4_iscsi_init);
3452
3069ee9b
VP		 3453int cxgb4_flush_eq_cache(struct net_device *dev)
3454{
3455 struct adapter *adap = netdev2adap(dev);
3456 int ret;
3457
3458 ret = t4_fwaddrspace_write(adap, adap->mbox,
3459 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3460 return ret;
3461}
3462EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3463
3464static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3465{
3466 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3467 __be64 indices;
3468 int ret;
3469
3470 ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3471 if (!ret) {
404d9e3f
VP		 3472 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3473 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3069ee9b
VP		 3474 }
3475 return ret;
3476}
3477
3478int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3479 u16 size)
3480{
3481 struct adapter *adap = netdev2adap(dev);
3482 u16 hw_pidx, hw_cidx;
3483 int ret;
3484
3485 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3486 if (ret)
3487 goto out;
3488
3489 if (pidx != hw_pidx) {
3490 u16 delta;
3491
3492 if (pidx >= hw_pidx)
3493 delta = pidx - hw_pidx;
3494 else
3495 delta = size - hw_pidx + pidx;
3496 wmb();
840f3000
VP		 3497 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3498 QID(qid) | PIDX(delta));
3069ee9b
VP		 3499 }
3500out:
3501 return ret;
3502}
3503EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3504
3cbdb928
VP		 3505void cxgb4_disable_db_coalescing(struct net_device *dev)
3506{
3507 struct adapter *adap;
3508
3509 adap = netdev2adap(dev);
3510 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3511 F_NOCOALESCE);
3512}
3513EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3514
3515void cxgb4_enable_db_coalescing(struct net_device *dev)
3516{
3517 struct adapter *adap;
3518
3519 adap = netdev2adap(dev);
3520 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3521}
3522EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3523
b8ff05a9
DM		 3524static struct pci_driver cxgb4_driver;
3525
3526static void check_neigh_update(struct neighbour *neigh)
3527{
3528 const struct device *parent;
3529 const struct net_device *netdev = neigh->dev;
3530
3531 if (netdev->priv_flags & IFF_802_1Q_VLAN)
3532 netdev = vlan_dev_real_dev(netdev);
3533 parent = netdev->dev.parent;
3534 if (parent && parent->driver == &cxgb4_driver.driver)
3535 t4_l2t_update(dev_get_drvdata(parent), neigh);
3536}
3537
3538static int netevent_cb(struct notifier_block *nb, unsigned long event,
3539 void *data)
3540{
3541 switch (event) {
3542 case NETEVENT_NEIGH_UPDATE:
3543 check_neigh_update(data);
3544 break;
b8ff05a9
DM		 3545 case NETEVENT_REDIRECT:
3546 default:
3547 break;
3548 }
3549 return 0;
3550}
3551
3552static bool netevent_registered;
3553static struct notifier_block cxgb4_netevent_nb = {
3554 .notifier_call = netevent_cb
3555};
3556
3069ee9b
VP		 3557static void drain_db_fifo(struct adapter *adap, int usecs)
3558{
2cc301d2 3559 u32 v1, v2, lp_count, hp_count;
3069ee9b
VP		 3560
3561 do {
2cc301d2
SR		 3562 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3563 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
d14807dd 3564 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3565 lp_count = G_LP_COUNT(v1);
3566 hp_count = G_HP_COUNT(v1);
3567 } else {
3568 lp_count = G_LP_COUNT_T5(v1);
3569 hp_count = G_HP_COUNT_T5(v2);
3570 }
3571
3572 if (lp_count == 0 && hp_count == 0)
3573 break;
3069ee9b
VP		 3574 set_current_state(TASK_UNINTERRUPTIBLE);
3575 schedule_timeout(usecs_to_jiffies(usecs));
3069ee9b
VP		 3576 } while (1);
3577}
3578
3579static void disable_txq_db(struct sge_txq *q)
3580{
3581 spin_lock_irq(&q->db_lock);
3582 q->db_disabled = 1;
3583 spin_unlock_irq(&q->db_lock);
3584}
3585
3586static void enable_txq_db(struct sge_txq *q)
3587{
3588 spin_lock_irq(&q->db_lock);
3589 q->db_disabled = 0;
3590 spin_unlock_irq(&q->db_lock);
3591}
3592
3593static void disable_dbs(struct adapter *adap)
3594{
3595 int i;
3596
3597 for_each_ethrxq(&adap->sge, i)
3598 disable_txq_db(&adap->sge.ethtxq[i].q);
3599 for_each_ofldrxq(&adap->sge, i)
3600 disable_txq_db(&adap->sge.ofldtxq[i].q);
3601 for_each_port(adap, i)
3602 disable_txq_db(&adap->sge.ctrlq[i].q);
3603}
3604
3605static void enable_dbs(struct adapter *adap)
3606{
3607 int i;
3608
3609 for_each_ethrxq(&adap->sge, i)
3610 enable_txq_db(&adap->sge.ethtxq[i].q);
3611 for_each_ofldrxq(&adap->sge, i)
3612 enable_txq_db(&adap->sge.ofldtxq[i].q);
3613 for_each_port(adap, i)
3614 enable_txq_db(&adap->sge.ctrlq[i].q);
3615}
3616
3617static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3618{
3619 u16 hw_pidx, hw_cidx;
3620 int ret;
3621
3622 spin_lock_bh(&q->db_lock);
3623 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3624 if (ret)
3625 goto out;
3626 if (q->db_pidx != hw_pidx) {
3627 u16 delta;
3628
3629 if (q->db_pidx >= hw_pidx)
3630 delta = q->db_pidx - hw_pidx;
3631 else
3632 delta = q->size - hw_pidx + q->db_pidx;
3633 wmb();
840f3000
VP		 3634 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3635 QID(q->cntxt_id) | PIDX(delta));
3069ee9b
VP		 3636 }
3637out:
3638 q->db_disabled = 0;
3639 spin_unlock_bh(&q->db_lock);
3640 if (ret)
3641 CH_WARN(adap, "DB drop recovery failed.\n");
3642}
3643static void recover_all_queues(struct adapter *adap)
3644{
3645 int i;
3646
3647 for_each_ethrxq(&adap->sge, i)
3648 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3649 for_each_ofldrxq(&adap->sge, i)
3650 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3651 for_each_port(adap, i)
3652 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3653}
3654
881806bc
VP		 3655static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3656{
3657 mutex_lock(&uld_mutex);
3658 if (adap->uld_handle[CXGB4_ULD_RDMA])
3659 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3660 cmd);
3661 mutex_unlock(&uld_mutex);
3662}
3663
3664static void process_db_full(struct work_struct *work)
3665{
3666 struct adapter *adap;
881806bc
VP		 3667
3668 adap = container_of(work, struct adapter, db_full_task);
3669
881806bc 3670 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3069ee9b 3671 drain_db_fifo(adap, dbfifo_drain_delay);
840f3000
VP		 3672 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3673 DBFIFO_HP_INT | DBFIFO_LP_INT,
3674 DBFIFO_HP_INT | DBFIFO_LP_INT);
881806bc 3675 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
881806bc
VP		 3676}
3677
3678static void process_db_drop(struct work_struct *work)
3679{
3680 struct adapter *adap;
881806bc 3681
3069ee9b 3682 adap = container_of(work, struct adapter, db_drop_task);
881806bc 3683
d14807dd 3684 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3685 disable_dbs(adap);
3686 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3687 drain_db_fifo(adap, 1);
3688 recover_all_queues(adap);
3689 enable_dbs(adap);
3690 } else {
3691 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3692 u16 qid = (dropped_db >> 15) & 0x1ffff;
3693 u16 pidx_inc = dropped_db & 0x1fff;
3694 unsigned int s_qpp;
3695 unsigned short udb_density;
3696 unsigned long qpshift;
3697 int page;
3698 u32 udb;
3699
3700 dev_warn(adap->pdev_dev,
3701 "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3702 dropped_db, qid,
3703 (dropped_db >> 14) & 1,
3704 (dropped_db >> 13) & 1,
3705 pidx_inc);
3706
3707 drain_db_fifo(adap, 1);
3708
3709 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3710 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3711 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3712 qpshift = PAGE_SHIFT - ilog2(udb_density);
3713 udb = qid << qpshift;
3714 udb &= PAGE_MASK;
3715 page = udb / PAGE_SIZE;
3716 udb += (qid - (page * udb_density)) * 128;
3717
3718 writel(PIDX(pidx_inc), adap->bar2 + udb + 8);
3719
3720 /* Re-enable BAR2 WC */
3721 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3722 }
3723
3069ee9b 3724 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
881806bc
VP		 3725}
3726
3727void t4_db_full(struct adapter *adap)
3728{
d14807dd 3729 if (is_t4(adap->params.chip)) {
2cc301d2
SR		 3730 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3731 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3732 queue_work(workq, &adap->db_full_task);
3733 }
881806bc
VP		 3734}
3735
3736void t4_db_dropped(struct adapter *adap)
3737{
d14807dd 3738 if (is_t4(adap->params.chip))
2cc301d2 3739 queue_work(workq, &adap->db_drop_task);
881806bc
VP		 3740}
3741
b8ff05a9
DM		 3742static void uld_attach(struct adapter *adap, unsigned int uld)
3743{
3744 void *handle;
3745 struct cxgb4_lld_info lli;
dca4faeb 3746 unsigned short i;
b8ff05a9
DM		 3747
3748 lli.pdev = adap->pdev;
3749 lli.l2t = adap->l2t;
3750 lli.tids = &adap->tids;
3751 lli.ports = adap->port;
3752 lli.vr = &adap->vres;
3753 lli.mtus = adap->params.mtus;
3754 if (uld == CXGB4_ULD_RDMA) {
3755 lli.rxq_ids = adap->sge.rdma_rxq;
3756 lli.nrxq = adap->sge.rdmaqs;
3757 } else if (uld == CXGB4_ULD_ISCSI) {
3758 lli.rxq_ids = adap->sge.ofld_rxq;
3759 lli.nrxq = adap->sge.ofldqsets;
3760 }
3761 lli.ntxq = adap->sge.ofldqsets;
3762 lli.nchan = adap->params.nports;
3763 lli.nports = adap->params.nports;
3764 lli.wr_cred = adap->params.ofldq_wr_cred;
d14807dd 3765 lli.adapter_type = adap->params.chip;
b8ff05a9
DM		 3766 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3767 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3768 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3769 (adap->fn * 4));
b8ff05a9 3770 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
060e0c75
DM		 3771 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3772 (adap->fn * 4));
dcf7b6f5 3773 lli.filt_mode = adap->params.tp.vlan_pri_map;
dca4faeb
VP		 3774 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3775 for (i = 0; i < NCHAN; i++)
3776 lli.tx_modq[i] = i;
b8ff05a9
DM		 3777 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3778 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3779 lli.fw_vers = adap->params.fw_vers;
3069ee9b 3780 lli.dbfifo_int_thresh = dbfifo_int_thresh;
dca4faeb
VP		 3781 lli.sge_pktshift = adap->sge.pktshift;
3782 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
1ac0f095 3783 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
b8ff05a9
DM		 3784
3785 handle = ulds[uld].add(&lli);
3786 if (IS_ERR(handle)) {
3787 dev_warn(adap->pdev_dev,
3788 "could not attach to the %s driver, error %ld\n",
3789 uld_str[uld], PTR_ERR(handle));
3790 return;
3791 }
3792
3793 adap->uld_handle[uld] = handle;
3794
3795 if (!netevent_registered) {
3796 register_netevent_notifier(&cxgb4_netevent_nb);
3797 netevent_registered = true;
3798 }
e29f5dbc
DM		 3799
3800 if (adap->flags & FULL_INIT_DONE)
3801 ulds[uld].state_change(handle, CXGB4_STATE_UP);
b8ff05a9
DM		 3802}
3803
3804static void attach_ulds(struct adapter *adap)
3805{
3806 unsigned int i;
3807
01bcca68
VP		 3808 spin_lock(&adap_rcu_lock);
3809 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
3810 spin_unlock(&adap_rcu_lock);
3811
b8ff05a9
DM		 3812 mutex_lock(&uld_mutex);
3813 list_add_tail(&adap->list_node, &adapter_list);
3814 for (i = 0; i < CXGB4_ULD_MAX; i++)
3815 if (ulds[i].add)
3816 uld_attach(adap, i);
3817 mutex_unlock(&uld_mutex);
3818}
3819
3820static void detach_ulds(struct adapter *adap)
3821{
3822 unsigned int i;
3823
3824 mutex_lock(&uld_mutex);
3825 list_del(&adap->list_node);
3826 for (i = 0; i < CXGB4_ULD_MAX; i++)
3827 if (adap->uld_handle[i]) {
3828 ulds[i].state_change(adap->uld_handle[i],
3829 CXGB4_STATE_DETACH);
3830 adap->uld_handle[i] = NULL;
3831 }
3832 if (netevent_registered && list_empty(&adapter_list)) {
3833 unregister_netevent_notifier(&cxgb4_netevent_nb);
3834 netevent_registered = false;
3835 }
3836 mutex_unlock(&uld_mutex);
01bcca68
VP		 3837
3838 spin_lock(&adap_rcu_lock);
3839 list_del_rcu(&adap->rcu_node);
3840 spin_unlock(&adap_rcu_lock);
b8ff05a9
DM		 3841}
3842
3843static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3844{
3845 unsigned int i;
3846
3847 mutex_lock(&uld_mutex);
3848 for (i = 0; i < CXGB4_ULD_MAX; i++)
3849 if (adap->uld_handle[i])
3850 ulds[i].state_change(adap->uld_handle[i], new_state);
3851 mutex_unlock(&uld_mutex);
3852}
3853
3854/**
3855 * cxgb4_register_uld - register an upper-layer driver
3856 * @type: the ULD type
3857 * @p: the ULD methods
3858 *
3859 * Registers an upper-layer driver with this driver and notifies the ULD
3860 * about any presently available devices that support its type. Returns
3861 * %-EBUSY if a ULD of the same type is already registered.
3862 */
3863int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3864{
3865 int ret = 0;
3866 struct adapter *adap;
3867
3868 if (type >= CXGB4_ULD_MAX)
3869 return -EINVAL;
3870 mutex_lock(&uld_mutex);
3871 if (ulds[type].add) {
3872 ret = -EBUSY;
3873 goto out;
3874 }
3875 ulds[type] = *p;
3876 list_for_each_entry(adap, &adapter_list, list_node)
3877 uld_attach(adap, type);
3878out: mutex_unlock(&uld_mutex);
3879 return ret;
3880}
3881EXPORT_SYMBOL(cxgb4_register_uld);
3882
3883/**
3884 * cxgb4_unregister_uld - unregister an upper-layer driver
3885 * @type: the ULD type
3886 *
3887 * Unregisters an existing upper-layer driver.
3888 */
3889int cxgb4_unregister_uld(enum cxgb4_uld type)
3890{
3891 struct adapter *adap;
3892
3893 if (type >= CXGB4_ULD_MAX)
3894 return -EINVAL;
3895 mutex_lock(&uld_mutex);
3896 list_for_each_entry(adap, &adapter_list, list_node)
3897 adap->uld_handle[type] = NULL;
3898 ulds[type].add = NULL;
3899 mutex_unlock(&uld_mutex);
3900 return 0;
3901}
3902EXPORT_SYMBOL(cxgb4_unregister_uld);
3903
01bcca68
VP		 3904/* Check if netdev on which event is occured belongs to us or not. Return
3905 * suceess (1) if it belongs otherwise failure (0).
3906 */
3907static int cxgb4_netdev(struct net_device *netdev)
3908{
3909 struct adapter *adap;
3910 int i;
3911
3912 spin_lock(&adap_rcu_lock);
3913 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
3914 for (i = 0; i < MAX_NPORTS; i++)
3915 if (adap->port[i] == netdev) {
3916 spin_unlock(&adap_rcu_lock);
3917 return 1;
3918 }
3919 spin_unlock(&adap_rcu_lock);
3920 return 0;
3921}
3922
3923static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
3924 unsigned long event)
3925{
3926 int ret = NOTIFY_DONE;
3927
3928 rcu_read_lock();
3929 if (cxgb4_netdev(event_dev)) {
3930 switch (event) {
3931 case NETDEV_UP:
3932 ret = cxgb4_clip_get(event_dev,
3933 (const struct in6_addr *)ifa->addr.s6_addr);
3934 if (ret < 0) {
3935 rcu_read_unlock();
3936 return ret;
3937 }
3938 ret = NOTIFY_OK;
3939 break;
3940 case NETDEV_DOWN:
3941 cxgb4_clip_release(event_dev,
3942 (const struct in6_addr *)ifa->addr.s6_addr);
3943 ret = NOTIFY_OK;
3944 break;
3945 default:
3946 break;
3947 }
3948 }
3949 rcu_read_unlock();
3950 return ret;
3951}
3952
3953static int cxgb4_inet6addr_handler(struct notifier_block *this,
3954 unsigned long event, void *data)
3955{
3956 struct inet6_ifaddr *ifa = data;
3957 struct net_device *event_dev;
3958 int ret = NOTIFY_DONE;
01bcca68 3959 struct bonding *bond = netdev_priv(ifa->idev->dev);
9caff1e7 3960 struct list_head *iter;
01bcca68
VP		 3961 struct slave *slave;
3962 struct pci_dev *first_pdev = NULL;
3963
3964 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
3965 event_dev = vlan_dev_real_dev(ifa->idev->dev);
3966 ret = clip_add(event_dev, ifa, event);
3967 } else if (ifa->idev->dev->flags & IFF_MASTER) {
3968 /* It is possible that two different adapters are bonded in one
3969 * bond. We need to find such different adapters and add clip
3970 * in all of them only once.
3971 */
3972 read_lock(&bond->lock);
9caff1e7 3973 bond_for_each_slave(bond, slave, iter) {
01bcca68
VP		 3974 if (!first_pdev) {
3975 ret = clip_add(slave->dev, ifa, event);
3976 /* If clip_add is success then only initialize
3977 * first_pdev since it means it is our device
3978 */
3979 if (ret == NOTIFY_OK)
3980 first_pdev = to_pci_dev(
3981 slave->dev->dev.parent);
3982 } else if (first_pdev !=
3983 to_pci_dev(slave->dev->dev.parent))
3984 ret = clip_add(slave->dev, ifa, event);
3985 }
3986 read_unlock(&bond->lock);
3987 } else
3988 ret = clip_add(ifa->idev->dev, ifa, event);
3989
3990 return ret;
3991}
3992
3993static struct notifier_block cxgb4_inet6addr_notifier = {
3994 .notifier_call = cxgb4_inet6addr_handler
3995};
3996
3997/* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
3998 * a physical device.
3999 * The physical device reference is needed to send the actul CLIP command.
4000 */
4001static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4002{
4003 struct inet6_dev *idev = NULL;
4004 struct inet6_ifaddr *ifa;
4005 int ret = 0;
4006
4007 idev = __in6_dev_get(root_dev);
4008 if (!idev)
4009 return ret;
4010
4011 read_lock_bh(&idev->lock);
4012 list_for_each_entry(ifa, &idev->addr_list, if_list) {
4013 ret = cxgb4_clip_get(dev,
4014 (const struct in6_addr *)ifa->addr.s6_addr);
4015 if (ret < 0)
4016 break;
4017 }
4018 read_unlock_bh(&idev->lock);
4019
4020 return ret;
4021}
4022
4023static int update_root_dev_clip(struct net_device *dev)
4024{
4025 struct net_device *root_dev = NULL;
4026 int i, ret = 0;
4027
4028 /* First populate the real net device's IPv6 addresses */
4029 ret = update_dev_clip(dev, dev);
4030 if (ret)
4031 return ret;
4032
4033 /* Parse all bond and vlan devices layered on top of the physical dev */
4034 for (i = 0; i < VLAN_N_VID; i++) {
4035 root_dev = __vlan_find_dev_deep(dev, htons(ETH_P_8021Q), i);
4036 if (!root_dev)
4037 continue;
4038
4039 ret = update_dev_clip(root_dev, dev);
4040 if (ret)
4041 break;
4042 }
4043 return ret;
4044}
4045
4046static void update_clip(const struct adapter *adap)
4047{
4048 int i;
4049 struct net_device *dev;
4050 int ret;
4051
4052 rcu_read_lock();
4053
4054 for (i = 0; i < MAX_NPORTS; i++) {
4055 dev = adap->port[i];
4056 ret = 0;
4057
4058 if (dev)
4059 ret = update_root_dev_clip(dev);
4060
4061 if (ret < 0)
4062 break;
4063 }
4064 rcu_read_unlock();
4065}
4066
b8ff05a9
DM		 4067/**
4068 * cxgb_up - enable the adapter
4069 * @adap: adapter being enabled
4070 *
4071 * Called when the first port is enabled, this function performs the
4072 * actions necessary to make an adapter operational, such as completing
4073 * the initialization of HW modules, and enabling interrupts.
4074 *
4075 * Must be called with the rtnl lock held.
4076 */
4077static int cxgb_up(struct adapter *adap)
4078{
aaefae9b 4079 int err;
b8ff05a9 4080
aaefae9b
DM		 4081 err = setup_sge_queues(adap);
4082 if (err)
4083 goto out;
4084 err = setup_rss(adap);
4085 if (err)
4086 goto freeq;
b8ff05a9
DM		 4087
4088 if (adap->flags & USING_MSIX) {
aaefae9b 4089 name_msix_vecs(adap);
b8ff05a9
DM		 4090 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4091 adap->msix_info[0].desc, adap);
4092 if (err)
4093 goto irq_err;
4094
4095 err = request_msix_queue_irqs(adap);
4096 if (err) {
4097 free_irq(adap->msix_info[0].vec, adap);
4098 goto irq_err;
4099 }
4100 } else {
4101 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4102 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
b1a3c2b6 4103 adap->port[0]->name, adap);
b8ff05a9
DM		 4104 if (err)
4105 goto irq_err;
4106 }
4107 enable_rx(adap);
4108 t4_sge_start(adap);
4109 t4_intr_enable(adap);
aaefae9b 4110 adap->flags |= FULL_INIT_DONE;
b8ff05a9 4111 notify_ulds(adap, CXGB4_STATE_UP);
01bcca68 4112 update_clip(adap);
b8ff05a9
DM		 4113 out:
4114 return err;
4115 irq_err:
4116 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
aaefae9b
DM		 4117 freeq:
4118 t4_free_sge_resources(adap);
b8ff05a9
DM		 4119 goto out;
4120}
4121
4122static void cxgb_down(struct adapter *adapter)
4123{
4124 t4_intr_disable(adapter);
4125 cancel_work_sync(&adapter->tid_release_task);
881806bc
VP		 4126 cancel_work_sync(&adapter->db_full_task);
4127 cancel_work_sync(&adapter->db_drop_task);
b8ff05a9 4128 adapter->tid_release_task_busy = false;
204dc3c0 4129 adapter->tid_release_head = NULL;
b8ff05a9
DM		 4130
4131 if (adapter->flags & USING_MSIX) {
4132 free_msix_queue_irqs(adapter);
4133 free_irq(adapter->msix_info[0].vec, adapter);
4134 } else
4135 free_irq(adapter->pdev->irq, adapter);
4136 quiesce_rx(adapter);
aaefae9b
DM		 4137 t4_sge_stop(adapter);
4138 t4_free_sge_resources(adapter);
4139 adapter->flags &= ~FULL_INIT_DONE;
b8ff05a9
DM		 4140}
4141
4142/*
4143 * net_device operations
4144 */
4145static int cxgb_open(struct net_device *dev)
4146{
4147 int err;
4148 struct port_info *pi = netdev_priv(dev);
4149 struct adapter *adapter = pi->adapter;
4150
6a3c869a
DM		 4151 netif_carrier_off(dev);
4152
aaefae9b
DM		 4153 if (!(adapter->flags & FULL_INIT_DONE)) {
4154 err = cxgb_up(adapter);
4155 if (err < 0)
4156 return err;
4157 }
b8ff05a9 4158
f68707b8
DM		 4159 err = link_start(dev);
4160 if (!err)
4161 netif_tx_start_all_queues(dev);
4162 return err;
b8ff05a9
DM		 4163}
4164
4165static int cxgb_close(struct net_device *dev)
4166{
b8ff05a9
DM		 4167 struct port_info *pi = netdev_priv(dev);
4168 struct adapter *adapter = pi->adapter;
4169
4170 netif_tx_stop_all_queues(dev);
4171 netif_carrier_off(dev);
060e0c75 4172 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
b8ff05a9
DM		 4173}
4174
f2b7e78d
VP		 4175/* Return an error number if the indicated filter isn't writable ...
4176 */
4177static int writable_filter(struct filter_entry *f)
4178{
4179 if (f->locked)
4180 return -EPERM;
4181 if (f->pending)
4182 return -EBUSY;
4183
4184 return 0;
4185}
4186
4187/* Delete the filter at the specified index (if valid). The checks for all
4188 * the common problems with doing this like the filter being locked, currently
4189 * pending in another operation, etc.
4190 */
4191static int delete_filter(struct adapter *adapter, unsigned int fidx)
4192{
4193 struct filter_entry *f;
4194 int ret;
4195
dca4faeb 4196 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
f2b7e78d
VP		 4197 return -EINVAL;
4198
4199 f = &adapter->tids.ftid_tab[fidx];
4200 ret = writable_filter(f);
4201 if (ret)
4202 return ret;
4203 if (f->valid)
4204 return del_filter_wr(adapter, fidx);
4205
4206 return 0;
4207}
4208
dca4faeb 4209int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
793dad94
VP		 4210 __be32 sip, __be16 sport, __be16 vlan,
4211 unsigned int queue, unsigned char port, unsigned char mask)
dca4faeb
VP		 4212{
4213 int ret;
4214 struct filter_entry *f;
4215 struct adapter *adap;
4216 int i;
4217 u8 *val;
4218
4219 adap = netdev2adap(dev);
4220
1cab775c 4221 /* Adjust stid to correct filter index */
470c60c4 4222 stid -= adap->tids.sftid_base;
1cab775c
VP		 4223 stid += adap->tids.nftids;
4224
dca4faeb
VP		 4225 /* Check to make sure the filter requested is writable ...
4226 */
4227 f = &adap->tids.ftid_tab[stid];
4228 ret = writable_filter(f);
4229 if (ret)
4230 return ret;
4231
4232 /* Clear out any old resources being used by the filter before
4233 * we start constructing the new filter.
4234 */
4235 if (f->valid)
4236 clear_filter(adap, f);
4237
4238 /* Clear out filter specifications */
4239 memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4240 f->fs.val.lport = cpu_to_be16(sport);
4241 f->fs.mask.lport = ~0;
4242 val = (u8 *)&sip;
793dad94 4243 if ((val[0] | val[1] | val[2] | val[3]) != 0) {
dca4faeb
VP		 4244 for (i = 0; i < 4; i++) {
4245 f->fs.val.lip[i] = val[i];
4246 f->fs.mask.lip[i] = ~0;
4247 }
dcf7b6f5 4248 if (adap->params.tp.vlan_pri_map & F_PORT) {
793dad94
VP		 4249 f->fs.val.iport = port;
4250 f->fs.mask.iport = mask;
4251 }
4252 }
dca4faeb 4253
dcf7b6f5 4254 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
7c89e555
KS		 4255 f->fs.val.proto = IPPROTO_TCP;
4256 f->fs.mask.proto = ~0;
4257 }
4258
dca4faeb
VP		 4259 f->fs.dirsteer = 1;
4260 f->fs.iq = queue;
4261 /* Mark filter as locked */
4262 f->locked = 1;
4263 f->fs.rpttid = 1;
4264
4265 ret = set_filter_wr(adap, stid);
4266 if (ret) {
4267 clear_filter(adap, f);
4268 return ret;
4269 }
4270
4271 return 0;
4272}
4273EXPORT_SYMBOL(cxgb4_create_server_filter);
4274
4275int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4276 unsigned int queue, bool ipv6)
4277{
4278 int ret;
4279 struct filter_entry *f;
4280 struct adapter *adap;
4281
4282 adap = netdev2adap(dev);
1cab775c
VP		 4283
4284 /* Adjust stid to correct filter index */
470c60c4 4285 stid -= adap->tids.sftid_base;
1cab775c
VP		 4286 stid += adap->tids.nftids;
4287
dca4faeb
VP		 4288 f = &adap->tids.ftid_tab[stid];
4289 /* Unlock the filter */
4290 f->locked = 0;
4291
4292 ret = delete_filter(adap, stid);
4293 if (ret)
4294 return ret;
4295
4296 return 0;
4297}
4298EXPORT_SYMBOL(cxgb4_remove_server_filter);
4299
f5152c90
DM		 4300static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4301 struct rtnl_link_stats64 *ns)
b8ff05a9
DM		 4302{
4303 struct port_stats stats;
4304 struct port_info *p = netdev_priv(dev);
4305 struct adapter *adapter = p->adapter;
b8ff05a9 4306
9fe6cb58
GS		 4307 /* Block retrieving statistics during EEH error
4308 * recovery. Otherwise, the recovery might fail
4309 * and the PCI device will be removed permanently
4310 */
b8ff05a9 4311 spin_lock(&adapter->stats_lock);
9fe6cb58
GS		 4312 if (!netif_device_present(dev)) {
4313 spin_unlock(&adapter->stats_lock);
4314 return ns;
4315 }
b8ff05a9
DM		 4316 t4_get_port_stats(adapter, p->tx_chan, &stats);
4317 spin_unlock(&adapter->stats_lock);
4318
4319 ns->tx_bytes = stats.tx_octets;
4320 ns->tx_packets = stats.tx_frames;
4321 ns->rx_bytes = stats.rx_octets;
4322 ns->rx_packets = stats.rx_frames;
4323 ns->multicast = stats.rx_mcast_frames;
4324
4325 /* detailed rx_errors */
4326 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4327 stats.rx_runt;
4328 ns->rx_over_errors = 0;
4329 ns->rx_crc_errors = stats.rx_fcs_err;
4330 ns->rx_frame_errors = stats.rx_symbol_err;
4331 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
4332 stats.rx_ovflow2 + stats.rx_ovflow3 +
4333 stats.rx_trunc0 + stats.rx_trunc1 +
4334 stats.rx_trunc2 + stats.rx_trunc3;
4335 ns->rx_missed_errors = 0;
4336
4337 /* detailed tx_errors */
4338 ns->tx_aborted_errors = 0;
4339 ns->tx_carrier_errors = 0;
4340 ns->tx_fifo_errors = 0;
4341 ns->tx_heartbeat_errors = 0;
4342 ns->tx_window_errors = 0;
4343
4344 ns->tx_errors = stats.tx_error_frames;
4345 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4346 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4347 return ns;
4348}
4349
4350static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4351{
060e0c75 4352 unsigned int mbox;
b8ff05a9
DM		 4353 int ret = 0, prtad, devad;
4354 struct port_info *pi = netdev_priv(dev);
4355 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4356
4357 switch (cmd) {
4358 case SIOCGMIIPHY:
4359 if (pi->mdio_addr < 0)
4360 return -EOPNOTSUPP;
4361 data->phy_id = pi->mdio_addr;
4362 break;
4363 case SIOCGMIIREG:
4364 case SIOCSMIIREG:
4365 if (mdio_phy_id_is_c45(data->phy_id)) {
4366 prtad = mdio_phy_id_prtad(data->phy_id);
4367 devad = mdio_phy_id_devad(data->phy_id);
4368 } else if (data->phy_id < 32) {
4369 prtad = data->phy_id;
4370 devad = 0;
4371 data->reg_num &= 0x1f;
4372 } else
4373 return -EINVAL;
4374
060e0c75 4375 mbox = pi->adapter->fn;
b8ff05a9 4376 if (cmd == SIOCGMIIREG)
060e0c75 4377 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4378 data->reg_num, &data->val_out);
4379 else
060e0c75 4380 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
b8ff05a9
DM		 4381 data->reg_num, data->val_in);
4382 break;
4383 default:
4384 return -EOPNOTSUPP;
4385 }
4386 return ret;
4387}
4388
4389static void cxgb_set_rxmode(struct net_device *dev)
4390{
4391 /* unfortunately we can't return errors to the stack */
4392 set_rxmode(dev, -1, false);
4393}
4394
4395static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4396{
4397 int ret;
4398 struct port_info *pi = netdev_priv(dev);
4399
4400 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
4401 return -EINVAL;
060e0c75
DM		 4402 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4403 -1, -1, -1, true);
b8ff05a9
DM		 4404 if (!ret)
4405 dev->mtu = new_mtu;
4406 return ret;
4407}
4408
4409static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4410{
4411 int ret;
4412 struct sockaddr *addr = p;
4413 struct port_info *pi = netdev_priv(dev);
4414
4415 if (!is_valid_ether_addr(addr->sa_data))
504f9b5a 4416 return -EADDRNOTAVAIL;
b8ff05a9 4417
060e0c75
DM		 4418 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4419 pi->xact_addr_filt, addr->sa_data, true, true);
b8ff05a9
DM		 4420 if (ret < 0)
4421 return ret;
4422
4423 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4424 pi->xact_addr_filt = ret;
4425 return 0;
4426}
4427
b8ff05a9
DM		 4428#ifdef CONFIG_NET_POLL_CONTROLLER
4429static void cxgb_netpoll(struct net_device *dev)
4430{
4431 struct port_info *pi = netdev_priv(dev);
4432 struct adapter *adap = pi->adapter;
4433
4434 if (adap->flags & USING_MSIX) {
4435 int i;
4436 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4437
4438 for (i = pi->nqsets; i; i--, rx++)
4439 t4_sge_intr_msix(0, &rx->rspq);
4440 } else
4441 t4_intr_handler(adap)(0, adap);
4442}
4443#endif
4444
4445static const struct net_device_ops cxgb4_netdev_ops = {
4446 .ndo_open = cxgb_open,
4447 .ndo_stop = cxgb_close,
4448 .ndo_start_xmit = t4_eth_xmit,
9be793bf 4449 .ndo_get_stats64 = cxgb_get_stats,
b8ff05a9
DM		 4450 .ndo_set_rx_mode = cxgb_set_rxmode,
4451 .ndo_set_mac_address = cxgb_set_mac_addr,
2ed28baa 4452 .ndo_set_features = cxgb_set_features,
b8ff05a9
DM		 4453 .ndo_validate_addr = eth_validate_addr,
4454 .ndo_do_ioctl = cxgb_ioctl,
4455 .ndo_change_mtu = cxgb_change_mtu,
b8ff05a9
DM		 4456#ifdef CONFIG_NET_POLL_CONTROLLER
4457 .ndo_poll_controller = cxgb_netpoll,
4458#endif
4459};
4460
4461void t4_fatal_err(struct adapter *adap)
4462{
4463 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4464 t4_intr_disable(adap);
4465 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4466}
4467
4468static void setup_memwin(struct adapter *adap)
4469{
19dd37ba 4470 u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
b8ff05a9
DM		 4471
4472 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
d14807dd 4473 if (is_t4(adap->params.chip)) {
19dd37ba
SR		 4474 mem_win0_base = bar0 + MEMWIN0_BASE;
4475 mem_win1_base = bar0 + MEMWIN1_BASE;
4476 mem_win2_base = bar0 + MEMWIN2_BASE;
4477 } else {
4478 /* For T5, only relative offset inside the PCIe BAR is passed */
4479 mem_win0_base = MEMWIN0_BASE;
4480 mem_win1_base = MEMWIN1_BASE_T5;
4481 mem_win2_base = MEMWIN2_BASE_T5;
4482 }
b8ff05a9 4483 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
19dd37ba 4484 mem_win0_base | BIR(0) |
b8ff05a9
DM		 4485 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4486 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
19dd37ba 4487 mem_win1_base | BIR(0) |
b8ff05a9
DM		 4488 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4489 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
19dd37ba 4490 mem_win2_base | BIR(0) |
b8ff05a9 4491 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
636f9d37
VP		 4492}
4493
4494static void setup_memwin_rdma(struct adapter *adap)
4495{
1ae970e0
DM		 4496 if (adap->vres.ocq.size) {
4497 unsigned int start, sz_kb;
4498
4499 start = pci_resource_start(adap->pdev, 2) +
4500 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4501 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4502 t4_write_reg(adap,
4503 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4504 start | BIR(1) | WINDOW(ilog2(sz_kb)));
4505 t4_write_reg(adap,
4506 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4507 adap->vres.ocq.start);
4508 t4_read_reg(adap,
4509 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4510 }
b8ff05a9
DM		 4511}
4512
02b5fb8e
DM		 4513static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4514{
4515 u32 v;
4516 int ret;
4517
4518 /* get device capabilities */
4519 memset(c, 0, sizeof(*c));
4520 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4521 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 4522 c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
060e0c75 4523 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
02b5fb8e
DM		 4524 if (ret < 0)
4525 return ret;
4526
4527 /* select capabilities we'll be using */
4528 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4529 if (!vf_acls)
4530 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4531 else
4532 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4533 } else if (vf_acls) {
4534 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4535 return ret;
4536 }
4537 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4538 FW_CMD_REQUEST | FW_CMD_WRITE);
060e0c75 4539 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
02b5fb8e
DM		 4540 if (ret < 0)
4541 return ret;
4542
060e0c75 4543 ret = t4_config_glbl_rss(adap, adap->fn,
02b5fb8e
DM		 4544 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4545 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4546 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4547 if (ret < 0)
4548 return ret;
4549
060e0c75
DM		 4550 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4551 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
02b5fb8e
DM		 4552 if (ret < 0)
4553 return ret;
4554
4555 t4_sge_init(adap);
4556
02b5fb8e
DM		 4557 /* tweak some settings */
4558 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4559 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4560 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4561 v = t4_read_reg(adap, TP_PIO_DATA);
4562 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
060e0c75 4563
dca4faeb
VP		 4564 /* first 4 Tx modulation queues point to consecutive Tx channels */
4565 adap->params.tp.tx_modq_map = 0xE4;
4566 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4567 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4568
4569 /* associate each Tx modulation queue with consecutive Tx channels */
4570 v = 0x84218421;
4571 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4572 &v, 1, A_TP_TX_SCHED_HDR);
4573 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4574 &v, 1, A_TP_TX_SCHED_FIFO);
4575 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4576 &v, 1, A_TP_TX_SCHED_PCMD);
4577
4578#define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4579 if (is_offload(adap)) {
4580 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4581 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4582 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4583 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4584 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4585 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4586 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4587 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4588 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4589 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4590 }
4591
060e0c75
DM		 4592 /* get basic stuff going */
4593 return t4_early_init(adap, adap->fn);
02b5fb8e
DM		 4594}
4595
b8ff05a9
DM		 4596/*
4597 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
4598 */
4599#define MAX_ATIDS 8192U
4600
636f9d37
VP		 4601/*
4602 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4603 *
4604 * If the firmware we're dealing with has Configuration File support, then
4605 * we use that to perform all configuration
4606 */
4607
4608/*
4609 * Tweak configuration based on module parameters, etc. Most of these have
4610 * defaults assigned to them by Firmware Configuration Files (if we're using
4611 * them) but need to be explicitly set if we're using hard-coded
4612 * initialization. But even in the case of using Firmware Configuration
4613 * Files, we'd like to expose the ability to change these via module
4614 * parameters so these are essentially common tweaks/settings for
4615 * Configuration Files and hard-coded initialization ...
4616 */
4617static int adap_init0_tweaks(struct adapter *adapter)
4618{
4619 /*
4620 * Fix up various Host-Dependent Parameters like Page Size, Cache
4621 * Line Size, etc. The firmware default is for a 4KB Page Size and
4622 * 64B Cache Line Size ...
4623 */
4624 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4625
4626 /*
4627 * Process module parameters which affect early initialization.
4628 */
4629 if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4630 dev_err(&adapter->pdev->dev,
4631 "Ignoring illegal rx_dma_offset=%d, using 2\n",
4632 rx_dma_offset);
4633 rx_dma_offset = 2;
4634 }
4635 t4_set_reg_field(adapter, SGE_CONTROL,
4636 PKTSHIFT_MASK,
4637 PKTSHIFT(rx_dma_offset));
4638
4639 /*
4640 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4641 * adds the pseudo header itself.
4642 */
4643 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4644 CSUM_HAS_PSEUDO_HDR, 0);
4645
4646 return 0;
4647}
4648
4649/*
4650 * Attempt to initialize the adapter via a Firmware Configuration File.
4651 */
4652static int adap_init0_config(struct adapter *adapter, int reset)
4653{
4654 struct fw_caps_config_cmd caps_cmd;
4655 const struct firmware *cf;
4656 unsigned long mtype = 0, maddr = 0;
4657 u32 finiver, finicsum, cfcsum;
16e47624
HS		 4658 int ret;
4659 int config_issued = 0;
0a57a536 4660 char *fw_config_file, fw_config_file_path[256];
16e47624 4661 char *config_name = NULL;
636f9d37
VP		 4662
4663 /*
4664 * Reset device if necessary.
4665 */
4666 if (reset) {
4667 ret = t4_fw_reset(adapter, adapter->mbox,
4668 PIORSTMODE | PIORST);
4669 if (ret < 0)
4670 goto bye;
4671 }
4672
4673 /*
4674 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4675 * then use that. Otherwise, use the configuration file stored
4676 * in the adapter flash ...
4677 */
d14807dd 4678 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
0a57a536 4679 case CHELSIO_T4:
16e47624 4680 fw_config_file = FW4_CFNAME;
0a57a536
SR		 4681 break;
4682 case CHELSIO_T5:
4683 fw_config_file = FW5_CFNAME;
4684 break;
4685 default:
4686 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4687 adapter->pdev->device);
4688 ret = -EINVAL;
4689 goto bye;
4690 }
4691
4692 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
636f9d37 4693 if (ret < 0) {
16e47624 4694 config_name = "On FLASH";
636f9d37
VP		 4695 mtype = FW_MEMTYPE_CF_FLASH;
4696 maddr = t4_flash_cfg_addr(adapter);
4697 } else {
4698 u32 params[7], val[7];
4699
16e47624
HS		 4700 sprintf(fw_config_file_path,
4701 "/lib/firmware/%s", fw_config_file);
4702 config_name = fw_config_file_path;
4703
636f9d37
VP		 4704 if (cf->size >= FLASH_CFG_MAX_SIZE)
4705 ret = -ENOMEM;
4706 else {
4707 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4708 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4709 ret = t4_query_params(adapter, adapter->mbox,
4710 adapter->fn, 0, 1, params, val);
4711 if (ret == 0) {
4712 /*
4713 * For t4_memory_write() below addresses and
4714 * sizes have to be in terms of multiples of 4
4715 * bytes. So, if the Configuration File isn't
4716 * a multiple of 4 bytes in length we'll have
4717 * to write that out separately since we can't
4718 * guarantee that the bytes following the
4719 * residual byte in the buffer returned by
4720 * request_firmware() are zeroed out ...
4721 */
4722 size_t resid = cf->size & 0x3;
4723 size_t size = cf->size & ~0x3;
4724 __be32 *data = (__be32 *)cf->data;
4725
4726 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4727 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4728
4729 ret = t4_memory_write(adapter, mtype, maddr,
4730 size, data);
4731 if (ret == 0 && resid != 0) {
4732 union {
4733 __be32 word;
4734 char buf[4];
4735 } last;
4736 int i;
4737
4738 last.word = data[size >> 2];
4739 for (i = resid; i < 4; i++)
4740 last.buf[i] = 0;
4741 ret = t4_memory_write(adapter, mtype,
4742 maddr + size,
4743 4, &last.word);
4744 }
4745 }
4746 }
4747
4748 release_firmware(cf);
4749 if (ret)
4750 goto bye;
4751 }
4752
4753 /*
4754 * Issue a Capability Configuration command to the firmware to get it
4755 * to parse the Configuration File. We don't use t4_fw_config_file()
4756 * because we want the ability to modify various features after we've
4757 * processed the configuration file ...
4758 */
4759 memset(&caps_cmd, 0, sizeof(caps_cmd));
4760 caps_cmd.op_to_write =
4761 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4762 FW_CMD_REQUEST |
4763 FW_CMD_READ);
ce91a923 4764 caps_cmd.cfvalid_to_len16 =
636f9d37
VP		 4765 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4766 FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4767 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4768 FW_LEN16(caps_cmd));
4769 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4770 &caps_cmd);
16e47624
HS		 4771
4772 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4773 * Configuration File in FLASH), our last gasp effort is to use the
4774 * Firmware Configuration File which is embedded in the firmware. A
4775 * very few early versions of the firmware didn't have one embedded
4776 * but we can ignore those.
4777 */
4778 if (ret == -ENOENT) {
4779 memset(&caps_cmd, 0, sizeof(caps_cmd));
4780 caps_cmd.op_to_write =
4781 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4782 FW_CMD_REQUEST |
4783 FW_CMD_READ);
4784 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4785 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4786 sizeof(caps_cmd), &caps_cmd);
4787 config_name = "Firmware Default";
4788 }
4789
4790 config_issued = 1;
636f9d37
VP		 4791 if (ret < 0)
4792 goto bye;
4793
4794 finiver = ntohl(caps_cmd.finiver);
4795 finicsum = ntohl(caps_cmd.finicsum);
4796 cfcsum = ntohl(caps_cmd.cfcsum);
4797 if (finicsum != cfcsum)
4798 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4799 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4800 finicsum, cfcsum);
4801
636f9d37
VP		 4802 /*
4803 * And now tell the firmware to use the configuration we just loaded.
4804 */
4805 caps_cmd.op_to_write =
4806 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4807 FW_CMD_REQUEST |
4808 FW_CMD_WRITE);
ce91a923 4809 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 4810 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4811 NULL);
4812 if (ret < 0)
4813 goto bye;
4814
4815 /*
4816 * Tweak configuration based on system architecture, module
4817 * parameters, etc.
4818 */
4819 ret = adap_init0_tweaks(adapter);
4820 if (ret < 0)
4821 goto bye;
4822
4823 /*
4824 * And finally tell the firmware to initialize itself using the
4825 * parameters from the Configuration File.
4826 */
4827 ret = t4_fw_initialize(adapter, adapter->mbox);
4828 if (ret < 0)
4829 goto bye;
4830
4831 /*
4832 * Return successfully and note that we're operating with parameters
4833 * not supplied by the driver, rather than from hard-wired
4834 * initialization constants burried in the driver.
4835 */
4836 adapter->flags |= USING_SOFT_PARAMS;
4837 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
16e47624
HS		 4838 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
4839 config_name, finiver, cfcsum);
636f9d37
VP		 4840 return 0;
4841
4842 /*
4843 * Something bad happened. Return the error ... (If the "error"
4844 * is that there's no Configuration File on the adapter we don't
4845 * want to issue a warning since this is fairly common.)
4846 */
4847bye:
16e47624
HS		 4848 if (config_issued && ret != -ENOENT)
4849 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
4850 config_name, -ret);
636f9d37
VP		 4851 return ret;
4852}
4853
13ee15d3
VP		 4854/*
4855 * Attempt to initialize the adapter via hard-coded, driver supplied
4856 * parameters ...
4857 */
4858static int adap_init0_no_config(struct adapter *adapter, int reset)
4859{
4860 struct sge *s = &adapter->sge;
4861 struct fw_caps_config_cmd caps_cmd;
4862 u32 v;
4863 int i, ret;
4864
4865 /*
4866 * Reset device if necessary
4867 */
4868 if (reset) {
4869 ret = t4_fw_reset(adapter, adapter->mbox,
4870 PIORSTMODE | PIORST);
4871 if (ret < 0)
4872 goto bye;
4873 }
4874
4875 /*
4876 * Get device capabilities and select which we'll be using.
4877 */
4878 memset(&caps_cmd, 0, sizeof(caps_cmd));
4879 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4880 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 4881 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
13ee15d3
VP		 4882 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4883 &caps_cmd);
4884 if (ret < 0)
4885 goto bye;
4886
13ee15d3
VP		 4887 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4888 if (!vf_acls)
4889 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4890 else
4891 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4892 } else if (vf_acls) {
4893 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
4894 goto bye;
4895 }
4896 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4897 FW_CMD_REQUEST | FW_CMD_WRITE);
4898 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4899 NULL);
4900 if (ret < 0)
4901 goto bye;
4902
4903 /*
4904 * Tweak configuration based on system architecture, module
4905 * parameters, etc.
4906 */
4907 ret = adap_init0_tweaks(adapter);
4908 if (ret < 0)
4909 goto bye;
4910
4911 /*
4912 * Select RSS Global Mode we want to use. We use "Basic Virtual"
4913 * mode which maps each Virtual Interface to its own section of
4914 * the RSS Table and we turn on all map and hash enables ...
4915 */
4916 adapter->flags |= RSS_TNLALLLOOKUP;
4917 ret = t4_config_glbl_rss(adapter, adapter->mbox,
4918 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4919 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4920 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4921 ((adapter->flags & RSS_TNLALLLOOKUP) ?
4922 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4923 if (ret < 0)
4924 goto bye;
4925
4926 /*
4927 * Set up our own fundamental resource provisioning ...
4928 */
4929 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4930 PFRES_NEQ, PFRES_NETHCTRL,
4931 PFRES_NIQFLINT, PFRES_NIQ,
4932 PFRES_TC, PFRES_NVI,
4933 FW_PFVF_CMD_CMASK_MASK,
4934 pfvfres_pmask(adapter, adapter->fn, 0),
4935 PFRES_NEXACTF,
4936 PFRES_R_CAPS, PFRES_WX_CAPS);
4937 if (ret < 0)
4938 goto bye;
4939
4940 /*
4941 * Perform low level SGE initialization. We need to do this before we
4942 * send the firmware the INITIALIZE command because that will cause
4943 * any other PF Drivers which are waiting for the Master
4944 * Initialization to proceed forward.
4945 */
4946 for (i = 0; i < SGE_NTIMERS - 1; i++)
4947 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4948 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4949 s->counter_val[0] = 1;
4950 for (i = 1; i < SGE_NCOUNTERS; i++)
4951 s->counter_val[i] = min(intr_cnt[i - 1],
4952 THRESHOLD_0_GET(THRESHOLD_0_MASK));
4953 t4_sge_init(adapter);
4954
4955#ifdef CONFIG_PCI_IOV
4956 /*
4957 * Provision resource limits for Virtual Functions. We currently
4958 * grant them all the same static resource limits except for the Port
4959 * Access Rights Mask which we're assigning based on the PF. All of
4960 * the static provisioning stuff for both the PF and VF really needs
4961 * to be managed in a persistent manner for each device which the
4962 * firmware controls.
4963 */
4964 {
4965 int pf, vf;
4966
7d6727cf 4967 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
13ee15d3
VP		 4968 if (num_vf[pf] <= 0)
4969 continue;
4970
4971 /* VF numbering starts at 1! */
4972 for (vf = 1; vf <= num_vf[pf]; vf++) {
4973 ret = t4_cfg_pfvf(adapter, adapter->mbox,
4974 pf, vf,
4975 VFRES_NEQ, VFRES_NETHCTRL,
4976 VFRES_NIQFLINT, VFRES_NIQ,
4977 VFRES_TC, VFRES_NVI,
1f1e4958 4978 FW_PFVF_CMD_CMASK_MASK,
13ee15d3
VP		 4979 pfvfres_pmask(
4980 adapter, pf, vf),
4981 VFRES_NEXACTF,
4982 VFRES_R_CAPS, VFRES_WX_CAPS);
4983 if (ret < 0)
4984 dev_warn(adapter->pdev_dev,
4985 "failed to "\
4986 "provision pf/vf=%d/%d; "
4987 "err=%d\n", pf, vf, ret);
4988 }
4989 }
4990 }
4991#endif
4992
4993 /*
4994 * Set up the default filter mode. Later we'll want to implement this
4995 * via a firmware command, etc. ... This needs to be done before the
4996 * firmare initialization command ... If the selected set of fields
4997 * isn't equal to the default value, we'll need to make sure that the
4998 * field selections will fit in the 36-bit budget.
4999 */
5000 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
404d9e3f 5001 int j, bits = 0;
13ee15d3 5002
404d9e3f
VP		 5003 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5004 switch (tp_vlan_pri_map & (1 << j)) {
13ee15d3
VP		 5005 case 0:
5006 /* compressed filter field not enabled */
5007 break;
5008 case FCOE_MASK:
5009 bits += 1;
5010 break;
5011 case PORT_MASK:
5012 bits += 3;
5013 break;
5014 case VNIC_ID_MASK:
5015 bits += 17;
5016 break;
5017 case VLAN_MASK:
5018 bits += 17;
5019 break;
5020 case TOS_MASK:
5021 bits += 8;
5022 break;
5023 case PROTOCOL_MASK:
5024 bits += 8;
5025 break;
5026 case ETHERTYPE_MASK:
5027 bits += 16;
5028 break;
5029 case MACMATCH_MASK:
5030 bits += 9;
5031 break;
5032 case MPSHITTYPE_MASK:
5033 bits += 3;
5034 break;
5035 case FRAGMENTATION_MASK:
5036 bits += 1;
5037 break;
5038 }
5039
5040 if (bits > 36) {
5041 dev_err(adapter->pdev_dev,
5042 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
5043 " using %#x\n", tp_vlan_pri_map, bits,
5044 TP_VLAN_PRI_MAP_DEFAULT);
5045 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5046 }
5047 }
5048 v = tp_vlan_pri_map;
5049 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5050 &v, 1, TP_VLAN_PRI_MAP);
5051
5052 /*
5053 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5054 * to support any of the compressed filter fields above. Newer
5055 * versions of the firmware do this automatically but it doesn't hurt
5056 * to set it here. Meanwhile, we do _not_ need to set Lookup Every
5057 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5058 * since the firmware automatically turns this on and off when we have
5059 * a non-zero number of filters active (since it does have a
5060 * performance impact).
5061 */
5062 if (tp_vlan_pri_map)
5063 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5064 FIVETUPLELOOKUP_MASK,
5065 FIVETUPLELOOKUP_MASK);
5066
5067 /*
5068 * Tweak some settings.
5069 */
5070 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5071 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5072 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5073 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5074
5075 /*
5076 * Get basic stuff going by issuing the Firmware Initialize command.
5077 * Note that this _must_ be after all PFVF commands ...
5078 */
5079 ret = t4_fw_initialize(adapter, adapter->mbox);
5080 if (ret < 0)
5081 goto bye;
5082
5083 /*
5084 * Return successfully!
5085 */
5086 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5087 "driver parameters\n");
5088 return 0;
5089
5090 /*
5091 * Something bad happened. Return the error ...
5092 */
5093bye:
5094 return ret;
5095}
5096
16e47624
HS		 5097static struct fw_info fw_info_array[] = {
5098 {
5099 .chip = CHELSIO_T4,
5100 .fs_name = FW4_CFNAME,
5101 .fw_mod_name = FW4_FNAME,
5102 .fw_hdr = {
5103 .chip = FW_HDR_CHIP_T4,
5104 .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5105 .intfver_nic = FW_INTFVER(T4, NIC),
5106 .intfver_vnic = FW_INTFVER(T4, VNIC),
5107 .intfver_ri = FW_INTFVER(T4, RI),
5108 .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5109 .intfver_fcoe = FW_INTFVER(T4, FCOE),
5110 },
5111 }, {
5112 .chip = CHELSIO_T5,
5113 .fs_name = FW5_CFNAME,
5114 .fw_mod_name = FW5_FNAME,
5115 .fw_hdr = {
5116 .chip = FW_HDR_CHIP_T5,
5117 .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5118 .intfver_nic = FW_INTFVER(T5, NIC),
5119 .intfver_vnic = FW_INTFVER(T5, VNIC),
5120 .intfver_ri = FW_INTFVER(T5, RI),
5121 .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5122 .intfver_fcoe = FW_INTFVER(T5, FCOE),
5123 },
5124 }
5125};
5126
5127static struct fw_info *find_fw_info(int chip)
5128{
5129 int i;
5130
5131 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5132 if (fw_info_array[i].chip == chip)
5133 return &fw_info_array[i];
5134 }
5135 return NULL;
5136}
5137
b8ff05a9
DM		 5138/*
5139 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5140 */
5141static int adap_init0(struct adapter *adap)
5142{
5143 int ret;
5144 u32 v, port_vec;
5145 enum dev_state state;
5146 u32 params[7], val[7];
9a4da2cd 5147 struct fw_caps_config_cmd caps_cmd;
dcf7b6f5 5148 int reset = 1;
b8ff05a9 5149
636f9d37
VP		 5150 /*
5151 * Contact FW, advertising Master capability (and potentially forcing
5152 * ourselves as the Master PF if our module parameter force_init is
5153 * set).
5154 */
5155 ret = t4_fw_hello(adap, adap->mbox, adap->fn,
5156 force_init ? MASTER_MUST : MASTER_MAY,
5157 &state);
b8ff05a9
DM		 5158 if (ret < 0) {
5159 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5160 ret);
5161 return ret;
5162 }
636f9d37
VP		 5163 if (ret == adap->mbox)
5164 adap->flags |= MASTER_PF;
5165 if (force_init && state == DEV_STATE_INIT)
5166 state = DEV_STATE_UNINIT;
b8ff05a9 5167
636f9d37
VP		 5168 /*
5169 * If we're the Master PF Driver and the device is uninitialized,
5170 * then let's consider upgrading the firmware ... (We always want
5171 * to check the firmware version number in order to A. get it for
5172 * later reporting and B. to warn if the currently loaded firmware
5173 * is excessively mismatched relative to the driver.)
5174 */
16e47624
HS		 5175 t4_get_fw_version(adap, &adap->params.fw_vers);
5176 t4_get_tp_version(adap, &adap->params.tp_vers);
636f9d37 5177 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
16e47624
HS		 5178 struct fw_info *fw_info;
5179 struct fw_hdr *card_fw;
5180 const struct firmware *fw;
5181 const u8 *fw_data = NULL;
5182 unsigned int fw_size = 0;
5183
5184 /* This is the firmware whose headers the driver was compiled
5185 * against
5186 */
5187 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5188 if (fw_info == NULL) {
5189 dev_err(adap->pdev_dev,
5190 "unable to get firmware info for chip %d.\n",
5191 CHELSIO_CHIP_VERSION(adap->params.chip));
5192 return -EINVAL;
636f9d37 5193 }
16e47624
HS		 5194
5195 /* allocate memory to read the header of the firmware on the
5196 * card
5197 */
5198 card_fw = t4_alloc_mem(sizeof(*card_fw));
5199
5200 /* Get FW from from /lib/firmware/ */
5201 ret = request_firmware(&fw, fw_info->fw_mod_name,
5202 adap->pdev_dev);
5203 if (ret < 0) {
5204 dev_err(adap->pdev_dev,
5205 "unable to load firmware image %s, error %d\n",
5206 fw_info->fw_mod_name, ret);
5207 } else {
5208 fw_data = fw->data;
5209 fw_size = fw->size;
5210 }
5211
5212 /* upgrade FW logic */
5213 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5214 state, &reset);
5215
5216 /* Cleaning up */
5217 if (fw != NULL)
5218 release_firmware(fw);
5219 t4_free_mem(card_fw);
5220
636f9d37 5221 if (ret < 0)
16e47624 5222 goto bye;
636f9d37 5223 }
b8ff05a9 5224
636f9d37
VP		 5225 /*
5226 * Grab VPD parameters. This should be done after we establish a
5227 * connection to the firmware since some of the VPD parameters
5228 * (notably the Core Clock frequency) are retrieved via requests to
5229 * the firmware. On the other hand, we need these fairly early on
5230 * so we do this right after getting ahold of the firmware.
5231 */
5232 ret = get_vpd_params(adap, &adap->params.vpd);
a0881cab
DM		 5233 if (ret < 0)
5234 goto bye;
a0881cab 5235
636f9d37 5236 /*
13ee15d3
VP		 5237 * Find out what ports are available to us. Note that we need to do
5238 * this before calling adap_init0_no_config() since it needs nports
5239 * and portvec ...
636f9d37
VP		 5240 */
5241 v =
5242 FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5243 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
5244 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
a0881cab
DM		 5245 if (ret < 0)
5246 goto bye;
5247
636f9d37
VP		 5248 adap->params.nports = hweight32(port_vec);
5249 adap->params.portvec = port_vec;
5250
5251 /*
5252 * If the firmware is initialized already (and we're not forcing a
5253 * master initialization), note that we're living with existing
5254 * adapter parameters. Otherwise, it's time to try initializing the
5255 * adapter ...
5256 */
5257 if (state == DEV_STATE_INIT) {
5258 dev_info(adap->pdev_dev, "Coming up as %s: "\
5259 "Adapter already initialized\n",
5260 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5261 adap->flags |= USING_SOFT_PARAMS;
5262 } else {
5263 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5264 "Initializing adapter\n");
636f9d37
VP		 5265
5266 /*
5267 * If the firmware doesn't support Configuration
5268 * Files warn user and exit,
5269 */
5270 if (ret < 0)
13ee15d3 5271 dev_warn(adap->pdev_dev, "Firmware doesn't support "
636f9d37 5272 "configuration file.\n");
13ee15d3
VP		 5273 if (force_old_init)
5274 ret = adap_init0_no_config(adap, reset);
636f9d37
VP		 5275 else {
5276 /*
13ee15d3
VP		 5277 * Find out whether we're dealing with a version of
5278 * the firmware which has configuration file support.
636f9d37 5279 */
13ee15d3
VP		 5280 params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
5281 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
5282 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5283 params, val);
636f9d37 5284
13ee15d3
VP		 5285 /*
5286 * If the firmware doesn't support Configuration
5287 * Files, use the old Driver-based, hard-wired
5288 * initialization. Otherwise, try using the
5289 * Configuration File support and fall back to the
5290 * Driver-based initialization if there's no
5291 * Configuration File found.
5292 */
5293 if (ret < 0)
5294 ret = adap_init0_no_config(adap, reset);
5295 else {
5296 /*
5297 * The firmware provides us with a memory
5298 * buffer where we can load a Configuration
5299 * File from the host if we want to override
5300 * the Configuration File in flash.
5301 */
5302
5303 ret = adap_init0_config(adap, reset);
5304 if (ret == -ENOENT) {
5305 dev_info(adap->pdev_dev,
5306 "No Configuration File present "
16e47624 5307 "on adapter. Using hard-wired "
13ee15d3
VP		 5308 "configuration parameters.\n");
5309 ret = adap_init0_no_config(adap, reset);
5310 }
636f9d37
VP		 5311 }
5312 }
5313 if (ret < 0) {
5314 dev_err(adap->pdev_dev,
5315 "could not initialize adapter, error %d\n",
5316 -ret);
5317 goto bye;
5318 }
5319 }
5320
5321 /*
5322 * If we're living with non-hard-coded parameters (either from a
5323 * Firmware Configuration File or values programmed by a different PF
5324 * Driver), give the SGE code a chance to pull in anything that it
5325 * needs ... Note that this must be called after we retrieve our VPD
5326 * parameters in order to know how to convert core ticks to seconds.
5327 */
5328 if (adap->flags & USING_SOFT_PARAMS) {
5329 ret = t4_sge_init(adap);
5330 if (ret < 0)
5331 goto bye;
5332 }
5333
9a4da2cd
VP		 5334 if (is_bypass_device(adap->pdev->device))
5335 adap->params.bypass = 1;
5336
636f9d37
VP		 5337 /*
5338 * Grab some of our basic fundamental operating parameters.
5339 */
5340#define FW_PARAM_DEV(param) \
5341 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5342 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5343
b8ff05a9 5344#define FW_PARAM_PFVF(param) \
636f9d37
VP		 5345 FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5346 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)| \
5347 FW_PARAMS_PARAM_Y(0) | \
5348 FW_PARAMS_PARAM_Z(0)
b8ff05a9 5349
636f9d37 5350 params[0] = FW_PARAM_PFVF(EQ_START);
b8ff05a9
DM		 5351 params[1] = FW_PARAM_PFVF(L2T_START);
5352 params[2] = FW_PARAM_PFVF(L2T_END);
5353 params[3] = FW_PARAM_PFVF(FILTER_START);
5354 params[4] = FW_PARAM_PFVF(FILTER_END);
e46dab4d 5355 params[5] = FW_PARAM_PFVF(IQFLINT_START);
636f9d37 5356 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
b8ff05a9
DM		 5357 if (ret < 0)
5358 goto bye;
636f9d37
VP		 5359 adap->sge.egr_start = val[0];
5360 adap->l2t_start = val[1];
5361 adap->l2t_end = val[2];
b8ff05a9
DM		 5362 adap->tids.ftid_base = val[3];
5363 adap->tids.nftids = val[4] - val[3] + 1;
e46dab4d 5364 adap->sge.ingr_start = val[5];
b8ff05a9 5365
636f9d37
VP		 5366 /* query params related to active filter region */
5367 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5368 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5369 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5370 /* If Active filter size is set we enable establishing
5371 * offload connection through firmware work request
5372 */
5373 if ((val[0] != val[1]) && (ret >= 0)) {
5374 adap->flags |= FW_OFLD_CONN;
5375 adap->tids.aftid_base = val[0];
5376 adap->tids.aftid_end = val[1];
5377 }
5378
b407a4a9
VP		 5379 /* If we're running on newer firmware, let it know that we're
5380 * prepared to deal with encapsulated CPL messages. Older
5381 * firmware won't understand this and we'll just get
5382 * unencapsulated messages ...
5383 */
5384 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5385 val[0] = 1;
5386 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5387
1ac0f095
KS		 5388 /*
5389 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5390 * capability. Earlier versions of the firmware didn't have the
5391 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5392 * permission to use ULPTX MEMWRITE DSGL.
5393 */
5394 if (is_t4(adap->params.chip)) {
5395 adap->params.ulptx_memwrite_dsgl = false;
5396 } else {
5397 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5398 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5399 1, params, val);
5400 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5401 }
5402
636f9d37
VP		 5403 /*
5404 * Get device capabilities so we can determine what resources we need
5405 * to manage.
5406 */
5407 memset(&caps_cmd, 0, sizeof(caps_cmd));
9a4da2cd 5408 caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
13ee15d3 5409 FW_CMD_REQUEST | FW_CMD_READ);
ce91a923 5410 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
636f9d37
VP		 5411 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5412 &caps_cmd);
5413 if (ret < 0)
5414 goto bye;
5415
13ee15d3 5416 if (caps_cmd.ofldcaps) {
b8ff05a9
DM		 5417 /* query offload-related parameters */
5418 params[0] = FW_PARAM_DEV(NTID);
5419 params[1] = FW_PARAM_PFVF(SERVER_START);
5420 params[2] = FW_PARAM_PFVF(SERVER_END);
5421 params[3] = FW_PARAM_PFVF(TDDP_START);
5422 params[4] = FW_PARAM_PFVF(TDDP_END);
5423 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
636f9d37
VP		 5424 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5425 params, val);
b8ff05a9
DM		 5426 if (ret < 0)
5427 goto bye;
5428 adap->tids.ntids = val[0];
5429 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5430 adap->tids.stid_base = val[1];
5431 adap->tids.nstids = val[2] - val[1] + 1;
636f9d37
VP		 5432 /*
5433 * Setup server filter region. Divide the availble filter
5434 * region into two parts. Regular filters get 1/3rd and server
5435 * filters get 2/3rd part. This is only enabled if workarond
5436 * path is enabled.
5437 * 1. For regular filters.
5438 * 2. Server filter: This are special filters which are used
5439 * to redirect SYN packets to offload queue.
5440 */
5441 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5442 adap->tids.sftid_base = adap->tids.ftid_base +
5443 DIV_ROUND_UP(adap->tids.nftids, 3);
5444 adap->tids.nsftids = adap->tids.nftids -
5445 DIV_ROUND_UP(adap->tids.nftids, 3);
5446 adap->tids.nftids = adap->tids.sftid_base -
5447 adap->tids.ftid_base;
5448 }
b8ff05a9
DM		 5449 adap->vres.ddp.start = val[3];
5450 adap->vres.ddp.size = val[4] - val[3] + 1;
5451 adap->params.ofldq_wr_cred = val[5];
636f9d37 5452
b8ff05a9
DM		 5453 adap->params.offload = 1;
5454 }
636f9d37 5455 if (caps_cmd.rdmacaps) {
b8ff05a9
DM		 5456 params[0] = FW_PARAM_PFVF(STAG_START);
5457 params[1] = FW_PARAM_PFVF(STAG_END);
5458 params[2] = FW_PARAM_PFVF(RQ_START);
5459 params[3] = FW_PARAM_PFVF(RQ_END);
5460 params[4] = FW_PARAM_PFVF(PBL_START);
5461 params[5] = FW_PARAM_PFVF(PBL_END);
636f9d37
VP		 5462 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5463 params, val);
b8ff05a9
DM		 5464 if (ret < 0)
5465 goto bye;
5466 adap->vres.stag.start = val[0];
5467 adap->vres.stag.size = val[1] - val[0] + 1;
5468 adap->vres.rq.start = val[2];
5469 adap->vres.rq.size = val[3] - val[2] + 1;
5470 adap->vres.pbl.start = val[4];
5471 adap->vres.pbl.size = val[5] - val[4] + 1;
a0881cab
DM		 5472
5473 params[0] = FW_PARAM_PFVF(SQRQ_START);
5474 params[1] = FW_PARAM_PFVF(SQRQ_END);
5475 params[2] = FW_PARAM_PFVF(CQ_START);
5476 params[3] = FW_PARAM_PFVF(CQ_END);
1ae970e0
DM		 5477 params[4] = FW_PARAM_PFVF(OCQ_START);
5478 params[5] = FW_PARAM_PFVF(OCQ_END);
636f9d37 5479 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
a0881cab
DM		 5480 if (ret < 0)
5481 goto bye;
5482 adap->vres.qp.start = val[0];
5483 adap->vres.qp.size = val[1] - val[0] + 1;
5484 adap->vres.cq.start = val[2];
5485 adap->vres.cq.size = val[3] - val[2] + 1;
1ae970e0
DM		 5486 adap->vres.ocq.start = val[4];
5487 adap->vres.ocq.size = val[5] - val[4] + 1;
b8ff05a9 5488 }
636f9d37 5489 if (caps_cmd.iscsicaps) {
b8ff05a9
DM		 5490 params[0] = FW_PARAM_PFVF(ISCSI_START);
5491 params[1] = FW_PARAM_PFVF(ISCSI_END);
636f9d37
VP		 5492 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5493 params, val);
b8ff05a9
DM		 5494 if (ret < 0)
5495 goto bye;
5496 adap->vres.iscsi.start = val[0];
5497 adap->vres.iscsi.size = val[1] - val[0] + 1;
5498 }
5499#undef FW_PARAM_PFVF
5500#undef FW_PARAM_DEV
5501
636f9d37
VP		 5502 /*
5503 * These are finalized by FW initialization, load their values now.
5504 */
b8ff05a9
DM		 5505 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5506 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5507 adap->params.b_wnd);
7ee9ff94 5508
dcf7b6f5 5509 t4_init_tp_params(adap);
636f9d37 5510 adap->flags |= FW_OK;
b8ff05a9
DM		 5511 return 0;
5512
5513 /*
636f9d37
VP		 5514 * Something bad happened. If a command timed out or failed with EIO
5515 * FW does not operate within its spec or something catastrophic
5516 * happened to HW/FW, stop issuing commands.
b8ff05a9 5517 */
636f9d37
VP		 5518bye:
5519 if (ret != -ETIMEDOUT && ret != -EIO)
5520 t4_fw_bye(adap, adap->mbox);
b8ff05a9
DM		 5521 return ret;
5522}
5523
204dc3c0
DM		 5524/* EEH callbacks */
5525
5526static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5527 pci_channel_state_t state)
5528{
5529 int i;
5530 struct adapter *adap = pci_get_drvdata(pdev);
5531
5532 if (!adap)
5533 goto out;
5534
5535 rtnl_lock();
5536 adap->flags &= ~FW_OK;
5537 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
9fe6cb58 5538 spin_lock(&adap->stats_lock);
204dc3c0
DM		 5539 for_each_port(adap, i) {
5540 struct net_device *dev = adap->port[i];
5541
5542 netif_device_detach(dev);
5543 netif_carrier_off(dev);
5544 }
9fe6cb58 5545 spin_unlock(&adap->stats_lock);
204dc3c0
DM		 5546 if (adap->flags & FULL_INIT_DONE)
5547 cxgb_down(adap);
5548 rtnl_unlock();
144be3d9
GS		 5549 if ((adap->flags & DEV_ENABLED)) {
5550 pci_disable_device(pdev);
5551 adap->flags &= ~DEV_ENABLED;
5552 }
204dc3c0
DM		 5553out: return state == pci_channel_io_perm_failure ?
5554 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5555}
5556
5557static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5558{
5559 int i, ret;
5560 struct fw_caps_config_cmd c;
5561 struct adapter *adap = pci_get_drvdata(pdev);
5562
5563 if (!adap) {
5564 pci_restore_state(pdev);
5565 pci_save_state(pdev);
5566 return PCI_ERS_RESULT_RECOVERED;
5567 }
5568
144be3d9
GS		 5569 if (!(adap->flags & DEV_ENABLED)) {
5570 if (pci_enable_device(pdev)) {
5571 dev_err(&pdev->dev, "Cannot reenable PCI "
5572 "device after reset\n");
5573 return PCI_ERS_RESULT_DISCONNECT;
5574 }
5575 adap->flags |= DEV_ENABLED;
204dc3c0
DM		 5576 }
5577
5578 pci_set_master(pdev);
5579 pci_restore_state(pdev);
5580 pci_save_state(pdev);
5581 pci_cleanup_aer_uncorrect_error_status(pdev);
5582
5583 if (t4_wait_dev_ready(adap) < 0)
5584 return PCI_ERS_RESULT_DISCONNECT;
777c2300 5585 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
204dc3c0
DM		 5586 return PCI_ERS_RESULT_DISCONNECT;
5587 adap->flags |= FW_OK;
5588 if (adap_init1(adap, &c))
5589 return PCI_ERS_RESULT_DISCONNECT;
5590
5591 for_each_port(adap, i) {
5592 struct port_info *p = adap2pinfo(adap, i);
5593
060e0c75
DM		 5594 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5595 NULL, NULL);
204dc3c0
DM		 5596 if (ret < 0)
5597 return PCI_ERS_RESULT_DISCONNECT;
5598 p->viid = ret;
5599 p->xact_addr_filt = -1;
5600 }
5601
5602 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5603 adap->params.b_wnd);
1ae970e0 5604 setup_memwin(adap);
204dc3c0
DM		 5605 if (cxgb_up(adap))
5606 return PCI_ERS_RESULT_DISCONNECT;
5607 return PCI_ERS_RESULT_RECOVERED;
5608}
5609
5610static void eeh_resume(struct pci_dev *pdev)
5611{
5612 int i;
5613 struct adapter *adap = pci_get_drvdata(pdev);
5614
5615 if (!adap)
5616 return;
5617
5618 rtnl_lock();
5619 for_each_port(adap, i) {
5620 struct net_device *dev = adap->port[i];
5621
5622 if (netif_running(dev)) {
5623 link_start(dev);
5624 cxgb_set_rxmode(dev);
5625 }
5626 netif_device_attach(dev);
5627 }
5628 rtnl_unlock();
5629}
5630
3646f0e5 5631static const struct pci_error_handlers cxgb4_eeh = {
204dc3c0
DM		 5632 .error_detected = eeh_err_detected,
5633 .slot_reset = eeh_slot_reset,
5634 .resume = eeh_resume,
5635};
5636
57d8b764 5637static inline bool is_x_10g_port(const struct link_config *lc)
b8ff05a9 5638{
57d8b764
KS		 5639 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5640 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
b8ff05a9
DM		 5641}
5642
5643static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5644 unsigned int size, unsigned int iqe_size)
5645{
5646 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5647 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5648 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5649 q->iqe_len = iqe_size;
5650 q->size = size;
5651}
5652
5653/*
5654 * Perform default configuration of DMA queues depending on the number and type
5655 * of ports we found and the number of available CPUs. Most settings can be
5656 * modified by the admin prior to actual use.
5657 */
91744948 5658static void cfg_queues(struct adapter *adap)
b8ff05a9
DM		 5659{
5660 struct sge *s = &adap->sge;
5661 int i, q10g = 0, n10g = 0, qidx = 0;
5662
5663 for_each_port(adap, i)
57d8b764 5664 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
b8ff05a9
DM		 5665
5666 /*
5667 * We default to 1 queue per non-10G port and up to # of cores queues
5668 * per 10G port.
5669 */
5670 if (n10g)
5671 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5952dde7
YM		 5672 if (q10g > netif_get_num_default_rss_queues())
5673 q10g = netif_get_num_default_rss_queues();
b8ff05a9
DM		 5674
5675 for_each_port(adap, i) {
5676 struct port_info *pi = adap2pinfo(adap, i);
5677
5678 pi->first_qset = qidx;
57d8b764 5679 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
b8ff05a9
DM		 5680 qidx += pi->nqsets;
5681 }
5682
5683 s->ethqsets = qidx;
5684 s->max_ethqsets = qidx; /* MSI-X may lower it later */
5685
5686 if (is_offload(adap)) {
5687 /*
5688 * For offload we use 1 queue/channel if all ports are up to 1G,
5689 * otherwise we divide all available queues amongst the channels
5690 * capped by the number of available cores.
5691 */
5692 if (n10g) {
5693 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5694 num_online_cpus());
5695 s->ofldqsets = roundup(i, adap->params.nports);
5696 } else
5697 s->ofldqsets = adap->params.nports;
5698 /* For RDMA one Rx queue per channel suffices */
5699 s->rdmaqs = adap->params.nports;
5700 }
5701
5702 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5703 struct sge_eth_rxq *r = &s->ethrxq[i];
5704
5705 init_rspq(&r->rspq, 0, 0, 1024, 64);
5706 r->fl.size = 72;
5707 }
5708
5709 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5710 s->ethtxq[i].q.size = 1024;
5711
5712 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5713 s->ctrlq[i].q.size = 512;
5714
5715 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5716 s->ofldtxq[i].q.size = 1024;
5717
5718 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5719 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5720
5721 init_rspq(&r->rspq, 0, 0, 1024, 64);
5722 r->rspq.uld = CXGB4_ULD_ISCSI;
5723 r->fl.size = 72;
5724 }
5725
5726 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5727 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5728
5729 init_rspq(&r->rspq, 0, 0, 511, 64);
5730 r->rspq.uld = CXGB4_ULD_RDMA;
5731 r->fl.size = 72;
5732 }
5733
5734 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5735 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5736}
5737
5738/*
5739 * Reduce the number of Ethernet queues across all ports to at most n.
5740 * n provides at least one queue per port.
5741 */
91744948 5742static void reduce_ethqs(struct adapter *adap, int n)
b8ff05a9
DM		 5743{
5744 int i;
5745 struct port_info *pi;
5746
5747 while (n < adap->sge.ethqsets)
5748 for_each_port(adap, i) {
5749 pi = adap2pinfo(adap, i);
5750 if (pi->nqsets > 1) {
5751 pi->nqsets--;
5752 adap->sge.ethqsets--;
5753 if (adap->sge.ethqsets <= n)
5754 break;
5755 }
5756 }
5757
5758 n = 0;
5759 for_each_port(adap, i) {
5760 pi = adap2pinfo(adap, i);
5761 pi->first_qset = n;
5762 n += pi->nqsets;
5763 }
5764}
5765
5766/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5767#define EXTRA_VECS 2
5768
91744948 5769static int enable_msix(struct adapter *adap)
b8ff05a9
DM		 5770{
5771 int ofld_need = 0;
c32ad224 5772 int i, want, need;
b8ff05a9
DM		 5773 struct sge *s = &adap->sge;
5774 unsigned int nchan = adap->params.nports;
5775 struct msix_entry entries[MAX_INGQ + 1];
5776
5777 for (i = 0; i < ARRAY_SIZE(entries); ++i)
5778 entries[i].entry = i;
5779
5780 want = s->max_ethqsets + EXTRA_VECS;
5781 if (is_offload(adap)) {
5782 want += s->rdmaqs + s->ofldqsets;
5783 /* need nchan for each possible ULD */
5784 ofld_need = 2 * nchan;
5785 }
5786 need = adap->params.nports + EXTRA_VECS + ofld_need;
5787
c32ad224
AG		 5788 want = pci_enable_msix_range(adap->pdev, entries, need, want);
5789 if (want < 0)
5790 return want;
b8ff05a9 5791
c32ad224
AG		 5792 /*
5793 * Distribute available vectors to the various queue groups.
5794 * Every group gets its minimum requirement and NIC gets top
5795 * priority for leftovers.
5796 */
5797 i = want - EXTRA_VECS - ofld_need;
5798 if (i < s->max_ethqsets) {
5799 s->max_ethqsets = i;
5800 if (i < s->ethqsets)
5801 reduce_ethqs(adap, i);
5802 }
5803 if (is_offload(adap)) {
5804 i = want - EXTRA_VECS - s->max_ethqsets;
5805 i -= ofld_need - nchan;
5806 s->ofldqsets = (i / nchan) * nchan; /* round down */
5807 }
5808 for (i = 0; i < want; ++i)
5809 adap->msix_info[i].vec = entries[i].vector;
5810
5811 return 0;
b8ff05a9
DM		 5812}
5813
5814#undef EXTRA_VECS
5815
91744948 5816static int init_rss(struct adapter *adap)
671b0060
DM		 5817{
5818 unsigned int i, j;
5819
5820 for_each_port(adap, i) {
5821 struct port_info *pi = adap2pinfo(adap, i);
5822
5823 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5824 if (!pi->rss)
5825 return -ENOMEM;
5826 for (j = 0; j < pi->rss_size; j++)
278bc429 5827 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
671b0060
DM		 5828 }
5829 return 0;
5830}
5831
91744948 5832static void print_port_info(const struct net_device *dev)
b8ff05a9 5833{
b8ff05a9 5834 char buf[80];
118969ed 5835 char *bufp = buf;
f1a051b9 5836 const char *spd = "";
118969ed
DM		 5837 const struct port_info *pi = netdev_priv(dev);
5838 const struct adapter *adap = pi->adapter;
f1a051b9
DM		 5839
5840 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5841 spd = " 2.5 GT/s";
5842 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5843 spd = " 5 GT/s";
b8ff05a9 5844
118969ed
DM		 5845 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5846 bufp += sprintf(bufp, "100/");
5847 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5848 bufp += sprintf(bufp, "1000/");
5849 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5850 bufp += sprintf(bufp, "10G/");
72aca4bf
KS		 5851 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5852 bufp += sprintf(bufp, "40G/");
118969ed
DM		 5853 if (bufp != buf)
5854 --bufp;
72aca4bf 5855 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
118969ed
DM		 5856
5857 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
0a57a536 5858 adap->params.vpd.id,
d14807dd 5859 CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
118969ed
DM		 5860 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5861 (adap->flags & USING_MSIX) ? " MSI-X" :
5862 (adap->flags & USING_MSI) ? " MSI" : "");
a94cd705
KS		 5863 netdev_info(dev, "S/N: %s, P/N: %s\n",
5864 adap->params.vpd.sn, adap->params.vpd.pn);
b8ff05a9
DM		 5865}
5866
91744948 5867static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
ef306b50 5868{
e5c8ae5f 5869 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
ef306b50
DM		 5870}
5871
06546391
DM		 5872/*
5873 * Free the following resources:
5874 * - memory used for tables
5875 * - MSI/MSI-X
5876 * - net devices
5877 * - resources FW is holding for us
5878 */
5879static void free_some_resources(struct adapter *adapter)
5880{
5881 unsigned int i;
5882
5883 t4_free_mem(adapter->l2t);
5884 t4_free_mem(adapter->tids.tid_tab);
5885 disable_msi(adapter);
5886
5887 for_each_port(adapter, i)
671b0060
DM		 5888 if (adapter->port[i]) {
5889 kfree(adap2pinfo(adapter, i)->rss);
06546391 5890 free_netdev(adapter->port[i]);
671b0060 5891 }
06546391 5892 if (adapter->flags & FW_OK)
060e0c75 5893 t4_fw_bye(adapter, adapter->fn);
06546391
DM		 5894}
5895
2ed28baa 5896#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
35d35682 5897#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
b8ff05a9 5898 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
22adfe0a 5899#define SEGMENT_SIZE 128
b8ff05a9 5900
1dd06ae8 5901static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
b8ff05a9 5902{
22adfe0a 5903 int func, i, err, s_qpp, qpp, num_seg;
b8ff05a9 5904 struct port_info *pi;
c8f44aff 5905 bool highdma = false;
b8ff05a9
DM		 5906 struct adapter *adapter = NULL;
5907
5908 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5909
5910 err = pci_request_regions(pdev, KBUILD_MODNAME);
5911 if (err) {
5912 /* Just info, some other driver may have claimed the device. */
5913 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5914 return err;
5915 }
5916
060e0c75 5917 /* We control everything through one PF */
b8ff05a9 5918 func = PCI_FUNC(pdev->devfn);
060e0c75 5919 if (func != ent->driver_data) {
204dc3c0 5920 pci_save_state(pdev); /* to restore SR-IOV later */
b8ff05a9 5921 goto sriov;
204dc3c0 5922 }
b8ff05a9
DM		 5923
5924 err = pci_enable_device(pdev);
5925 if (err) {
5926 dev_err(&pdev->dev, "cannot enable PCI device\n");
5927 goto out_release_regions;
5928 }
5929
5930 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
c8f44aff 5931 highdma = true;
b8ff05a9
DM		 5932 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5933 if (err) {
5934 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5935 "coherent allocations\n");
5936 goto out_disable_device;
5937 }
5938 } else {
5939 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5940 if (err) {
5941 dev_err(&pdev->dev, "no usable DMA configuration\n");
5942 goto out_disable_device;
5943 }
5944 }
5945
5946 pci_enable_pcie_error_reporting(pdev);
ef306b50 5947 enable_pcie_relaxed_ordering(pdev);
b8ff05a9
DM		 5948 pci_set_master(pdev);
5949 pci_save_state(pdev);
5950
5951 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5952 if (!adapter) {
5953 err = -ENOMEM;
5954 goto out_disable_device;
5955 }
5956
144be3d9
GS		 5957 /* PCI device has been enabled */
5958 adapter->flags |= DEV_ENABLED;
5959
b8ff05a9
DM		 5960 adapter->regs = pci_ioremap_bar(pdev, 0);
5961 if (!adapter->regs) {
5962 dev_err(&pdev->dev, "cannot map device registers\n");
5963 err = -ENOMEM;
5964 goto out_free_adapter;
5965 }
5966
5967 adapter->pdev = pdev;
5968 adapter->pdev_dev = &pdev->dev;
3069ee9b 5969 adapter->mbox = func;
060e0c75 5970 adapter->fn = func;
b8ff05a9
DM		 5971 adapter->msg_enable = dflt_msg_enable;
5972 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
5973
5974 spin_lock_init(&adapter->stats_lock);
5975 spin_lock_init(&adapter->tid_release_lock);
5976
5977 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
881806bc
VP		 5978 INIT_WORK(&adapter->db_full_task, process_db_full);
5979 INIT_WORK(&adapter->db_drop_task, process_db_drop);
b8ff05a9
DM		 5980
5981 err = t4_prep_adapter(adapter);
5982 if (err)
22adfe0a
SR		 5983 goto out_unmap_bar0;
5984
d14807dd 5985 if (!is_t4(adapter->params.chip)) {
22adfe0a
SR		 5986 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
5987 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
5988 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
5989 num_seg = PAGE_SIZE / SEGMENT_SIZE;
5990
5991 /* Each segment size is 128B. Write coalescing is enabled only
5992 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
5993 * queue is less no of segments that can be accommodated in
5994 * a page size.
5995 */
5996 if (qpp > num_seg) {
5997 dev_err(&pdev->dev,
5998 "Incorrect number of egress queues per page\n");
5999 err = -EINVAL;
6000 goto out_unmap_bar0;
6001 }
6002 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6003 pci_resource_len(pdev, 2));
6004 if (!adapter->bar2) {
6005 dev_err(&pdev->dev, "cannot map device bar2 region\n");
6006 err = -ENOMEM;
6007 goto out_unmap_bar0;
6008 }
6009 }
6010
636f9d37 6011 setup_memwin(adapter);
b8ff05a9 6012 err = adap_init0(adapter);
636f9d37 6013 setup_memwin_rdma(adapter);
b8ff05a9
DM		 6014 if (err)
6015 goto out_unmap_bar;
6016
6017 for_each_port(adapter, i) {
6018 struct net_device *netdev;
6019
6020 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6021 MAX_ETH_QSETS);
6022 if (!netdev) {
6023 err = -ENOMEM;
6024 goto out_free_dev;
6025 }
6026
6027 SET_NETDEV_DEV(netdev, &pdev->dev);
6028
6029 adapter->port[i] = netdev;
6030 pi = netdev_priv(netdev);
6031 pi->adapter = adapter;
6032 pi->xact_addr_filt = -1;
b8ff05a9 6033 pi->port_id = i;
b8ff05a9
DM		 6034 netdev->irq = pdev->irq;
6035
2ed28baa
MM		 6036 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6037 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6038 NETIF_F_RXCSUM | NETIF_F_RXHASH |
f646968f 6039 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
c8f44aff
MM		 6040 if (highdma)
6041 netdev->hw_features |= NETIF_F_HIGHDMA;
6042 netdev->features |= netdev->hw_features;
b8ff05a9
DM		 6043 netdev->vlan_features = netdev->features & VLAN_FEAT;
6044
01789349
JP		 6045 netdev->priv_flags |= IFF_UNICAST_FLT;
6046
b8ff05a9
DM		 6047 netdev->netdev_ops = &cxgb4_netdev_ops;
6048 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
6049 }
6050
6051 pci_set_drvdata(pdev, adapter);
6052
6053 if (adapter->flags & FW_OK) {
060e0c75 6054 err = t4_port_init(adapter, func, func, 0);
b8ff05a9
DM		 6055 if (err)
6056 goto out_free_dev;
6057 }
6058
6059 /*
6060 * Configure queues and allocate tables now, they can be needed as
6061 * soon as the first register_netdev completes.
6062 */
6063 cfg_queues(adapter);
6064
6065 adapter->l2t = t4_init_l2t();
6066 if (!adapter->l2t) {
6067 /* We tolerate a lack of L2T, giving up some functionality */
6068 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6069 adapter->params.offload = 0;
6070 }
6071
6072 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6073 dev_warn(&pdev->dev, "could not allocate TID table, "
6074 "continuing\n");
6075 adapter->params.offload = 0;
6076 }
6077
f7cabcdd
DM		 6078 /* See what interrupts we'll be using */
6079 if (msi > 1 && enable_msix(adapter) == 0)
6080 adapter->flags |= USING_MSIX;
6081 else if (msi > 0 && pci_enable_msi(pdev) == 0)
6082 adapter->flags |= USING_MSI;
6083
671b0060
DM		 6084 err = init_rss(adapter);
6085 if (err)
6086 goto out_free_dev;
6087
b8ff05a9
DM		 6088 /*
6089 * The card is now ready to go. If any errors occur during device
6090 * registration we do not fail the whole card but rather proceed only
6091 * with the ports we manage to register successfully. However we must
6092 * register at least one net device.
6093 */
6094 for_each_port(adapter, i) {
a57cabe0
DM		 6095 pi = adap2pinfo(adapter, i);
6096 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6097 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6098
b8ff05a9
DM		 6099 err = register_netdev(adapter->port[i]);
6100 if (err)
b1a3c2b6 6101 break;
b1a3c2b6
DM		 6102 adapter->chan_map[pi->tx_chan] = i;
6103 print_port_info(adapter->port[i]);
b8ff05a9 6104 }
b1a3c2b6 6105 if (i == 0) {
b8ff05a9
DM		 6106 dev_err(&pdev->dev, "could not register any net devices\n");
6107 goto out_free_dev;
6108 }
b1a3c2b6
DM		 6109 if (err) {
6110 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6111 err = 0;
6403eab1 6112 }
b8ff05a9
DM		 6113
6114 if (cxgb4_debugfs_root) {
6115 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6116 cxgb4_debugfs_root);
6117 setup_debugfs(adapter);
6118 }
6119
6482aa7c
DLR		 6120 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6121 pdev->needs_freset = 1;
6122
b8ff05a9
DM		 6123 if (is_offload(adapter))
6124 attach_ulds(adapter);
6125
b8ff05a9
DM		 6126sriov:
6127#ifdef CONFIG_PCI_IOV
7d6727cf 6128 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
b8ff05a9
DM		 6129 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6130 dev_info(&pdev->dev,
6131 "instantiated %u virtual functions\n",
6132 num_vf[func]);
6133#endif
6134 return 0;
6135
6136 out_free_dev:
06546391 6137 free_some_resources(adapter);
b8ff05a9 6138 out_unmap_bar:
d14807dd 6139 if (!is_t4(adapter->params.chip))
22adfe0a
SR		 6140 iounmap(adapter->bar2);
6141 out_unmap_bar0:
b8ff05a9
DM		 6142 iounmap(adapter->regs);
6143 out_free_adapter:
6144 kfree(adapter);
6145 out_disable_device:
6146 pci_disable_pcie_error_reporting(pdev);
6147 pci_disable_device(pdev);
6148 out_release_regions:
6149 pci_release_regions(pdev);
b8ff05a9
DM		 6150 return err;
6151}
6152
91744948 6153static void remove_one(struct pci_dev *pdev)
b8ff05a9
DM		 6154{
6155 struct adapter *adapter = pci_get_drvdata(pdev);
6156
636f9d37 6157#ifdef CONFIG_PCI_IOV
b8ff05a9
DM		 6158 pci_disable_sriov(pdev);
6159
636f9d37
VP		 6160#endif
6161
b8ff05a9
DM		 6162 if (adapter) {
6163 int i;
6164
6165 if (is_offload(adapter))
6166 detach_ulds(adapter);
6167
6168 for_each_port(adapter, i)
8f3a7676 6169 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
b8ff05a9
DM		 6170 unregister_netdev(adapter->port[i]);
6171
6172 if (adapter->debugfs_root)
6173 debugfs_remove_recursive(adapter->debugfs_root);
6174
f2b7e78d
VP		 6175 /* If we allocated filters, free up state associated with any
6176 * valid filters ...
6177 */
6178 if (adapter->tids.ftid_tab) {
6179 struct filter_entry *f = &adapter->tids.ftid_tab[0];
dca4faeb
VP		 6180 for (i = 0; i < (adapter->tids.nftids +
6181 adapter->tids.nsftids); i++, f++)
f2b7e78d
VP		 6182 if (f->valid)
6183 clear_filter(adapter, f);
6184 }
6185
aaefae9b
DM		 6186 if (adapter->flags & FULL_INIT_DONE)
6187 cxgb_down(adapter);
b8ff05a9 6188
06546391 6189 free_some_resources(adapter);
b8ff05a9 6190 iounmap(adapter->regs);
d14807dd 6191 if (!is_t4(adapter->params.chip))
22adfe0a 6192 iounmap(adapter->bar2);
b8ff05a9 6193 pci_disable_pcie_error_reporting(pdev);
144be3d9
GS		 6194 if ((adapter->flags & DEV_ENABLED)) {
6195 pci_disable_device(pdev);
6196 adapter->flags &= ~DEV_ENABLED;
6197 }
b8ff05a9 6198 pci_release_regions(pdev);
8b662fe7 6199 kfree(adapter);
a069ec91 6200 } else
b8ff05a9
DM		 6201 pci_release_regions(pdev);
6202}
6203
6204static struct pci_driver cxgb4_driver = {
6205 .name = KBUILD_MODNAME,
6206 .id_table = cxgb4_pci_tbl,
6207 .probe = init_one,
91744948 6208 .remove = remove_one,
204dc3c0 6209 .err_handler = &cxgb4_eeh,
b8ff05a9
DM		 6210};
6211
6212static int __init cxgb4_init_module(void)
6213{
6214 int ret;
6215
3069ee9b
VP		 6216 workq = create_singlethread_workqueue("cxgb4");
6217 if (!workq)
6218 return -ENOMEM;
6219
b8ff05a9
DM		 6220 /* Debugfs support is optional, just warn if this fails */
6221 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6222 if (!cxgb4_debugfs_root)
428ac43f 6223 pr_warn("could not create debugfs entry, continuing\n");
b8ff05a9
DM		 6224
6225 ret = pci_register_driver(&cxgb4_driver);
73a695f8 6226 if (ret < 0) {
b8ff05a9 6227 debugfs_remove(cxgb4_debugfs_root);
73a695f8
WY		 6228 destroy_workqueue(workq);
6229 }
01bcca68
VP		 6230
6231 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6232
b8ff05a9
DM		 6233 return ret;
6234}
6235
6236static void __exit cxgb4_cleanup_module(void)
6237{
01bcca68 6238 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
b8ff05a9
DM		 6239 pci_unregister_driver(&cxgb4_driver);
6240 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3069ee9b
VP		 6241 flush_workqueue(workq);
6242 destroy_workqueue(workq);
b8ff05a9
DM		 6243}
6244
6245module_init(cxgb4_init_module);
6246module_exit(cxgb4_cleanup_module);
Linux kernel - Deliverables
This page took 0.746247 seconds and 5 git commands to generate.