projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'iov_iter' into for-davem-2
[deliverable/linux.git] / drivers / net / ethernet / mellanox / mlx4 / main.c
CommitLineData
225c7b1f
RD		 1/*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
51a379d0 4 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
225c7b1f
RD		 5 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
6 *
7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
12 *
13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
16 *
17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
20 *
21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
25 *
26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
34 */
35
36#include <linux/module.h>
37#include <linux/init.h>
38#include <linux/errno.h>
39#include <linux/pci.h>
40#include <linux/dma-mapping.h>
5a0e3ad6 41#include <linux/slab.h>
c1b43dca 42#include <linux/io-mapping.h>
ab9c17a0 43#include <linux/delay.h>
b046ffe5 44#include <linux/kmod.h>
225c7b1f
RD		 45
46#include <linux/mlx4/device.h>
47#include <linux/mlx4/doorbell.h>
48
49#include "mlx4.h"
50#include "fw.h"
51#include "icm.h"
52
53MODULE_AUTHOR("Roland Dreier");
54MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
55MODULE_LICENSE("Dual BSD/GPL");
56MODULE_VERSION(DRV_VERSION);
57
27bf91d6
YP		 58struct workqueue_struct *mlx4_wq;
59
225c7b1f
RD		 60#ifdef CONFIG_MLX4_DEBUG
61
62int mlx4_debug_level = 0;
63module_param_named(debug_level, mlx4_debug_level, int, 0644);
64MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
65
66#endif /* CONFIG_MLX4_DEBUG */
67
68#ifdef CONFIG_PCI_MSI
69
08fb1055 70static int msi_x = 1;
225c7b1f
RD		 71module_param(msi_x, int, 0444);
72MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
73
74#else /* CONFIG_PCI_MSI */
75
76#define msi_x (0)
77
78#endif /* CONFIG_PCI_MSI */
79
dd41cc3b 80static uint8_t num_vfs[3] = {0, 0, 0};
effa4bc4 81static int num_vfs_argc;
dd41cc3b
MB		 82module_param_array(num_vfs, byte , &num_vfs_argc, 0444);
83MODULE_PARM_DESC(num_vfs, "enable #num_vfs functions if num_vfs > 0\n"
84 "num_vfs=port1,port2,port1+2");
85
86static uint8_t probe_vf[3] = {0, 0, 0};
effa4bc4 87static int probe_vfs_argc;
dd41cc3b
MB		 88module_param_array(probe_vf, byte, &probe_vfs_argc, 0444);
89MODULE_PARM_DESC(probe_vf, "number of vfs to probe by pf driver (num_vfs > 0)\n"
90 "probe_vf=port1,port2,port1+2");
ab9c17a0 91
3c439b55 92int mlx4_log_num_mgm_entry_size = MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
0ec2c0f8
EE		 93module_param_named(log_num_mgm_entry_size,
94 mlx4_log_num_mgm_entry_size, int, 0444);
95MODULE_PARM_DESC(log_num_mgm_entry_size, "log mgm size, that defines the num"
96 " of qp per mcg, for example:"
3c439b55 97 " 10 gives 248.range: 7 <="
0ff1fb65 98 " log_num_mgm_entry_size <= 12."
3c439b55
JM		 99 " To activate device managed"
100 " flow steering when available, set to -1");
0ec2c0f8 101
be902ab1 102static bool enable_64b_cqe_eqe = true;
08ff3235
OG		 103module_param(enable_64b_cqe_eqe, bool, 0444);
104MODULE_PARM_DESC(enable_64b_cqe_eqe,
be902ab1 105 "Enable 64 byte CQEs/EQEs when the FW supports this (default: True)");
08ff3235 106
77507aa2
IS		 107#define PF_CONTEXT_BEHAVIOUR_MASK (MLX4_FUNC_CAP_64B_EQE_CQE | \
108 MLX4_FUNC_CAP_EQE_CQE_STRIDE)
ab9c17a0 109
f57e6848 110static char mlx4_version[] =
225c7b1f
RD		 111 DRV_NAME ": Mellanox ConnectX core driver v"
112 DRV_VERSION " (" DRV_RELDATE ")\n";
113
114static struct mlx4_profile default_profile = {
ab9c17a0 115 .num_qp = 1 << 18,
225c7b1f 116 .num_srq = 1 << 16,
c9f2ba5e 117 .rdmarc_per_qp = 1 << 4,
225c7b1f
RD		 118 .num_cq = 1 << 16,
119 .num_mcg = 1 << 13,
ab9c17a0 120 .num_mpt = 1 << 19,
9fd7a1e1 121 .num_mtt = 1 << 20, /* It is really num mtt segements */
225c7b1f
RD		 122};
123
2599d858
AV		 124static struct mlx4_profile low_mem_profile = {
125 .num_qp = 1 << 17,
126 .num_srq = 1 << 6,
127 .rdmarc_per_qp = 1 << 4,
128 .num_cq = 1 << 8,
129 .num_mcg = 1 << 8,
130 .num_mpt = 1 << 9,
131 .num_mtt = 1 << 7,
132};
133
ab9c17a0 134static int log_num_mac = 7;
93fc9e1b
YP		 135module_param_named(log_num_mac, log_num_mac, int, 0444);
136MODULE_PARM_DESC(log_num_mac, "Log2 max number of MACs per ETH port (1-7)");
137
138static int log_num_vlan;
139module_param_named(log_num_vlan, log_num_vlan, int, 0444);
140MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)");
cb29688a
OG		 141/* Log2 max number of VLANs per ETH port (0-7) */
142#define MLX4_LOG_NUM_VLANS 7
2599d858
AV		 143#define MLX4_MIN_LOG_NUM_VLANS 0
144#define MLX4_MIN_LOG_NUM_MAC 1
93fc9e1b 145
eb939922 146static bool use_prio;
93fc9e1b 147module_param_named(use_prio, use_prio, bool, 0444);
ecc8fb11 148MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)");
93fc9e1b 149
2b8fb286 150int log_mtts_per_seg = ilog2(MLX4_MTT_ENTRY_PER_SEG);
ab6bf42e 151module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444);
0498628f 152MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-7)");
ab6bf42e 153
8d0fc7b6 154static int port_type_array[2] = {MLX4_PORT_TYPE_NONE, MLX4_PORT_TYPE_NONE};
ab9c17a0
JM		 155static int arr_argc = 2;
156module_param_array(port_type_array, int, &arr_argc, 0444);
8d0fc7b6
YP		 157MODULE_PARM_DESC(port_type_array, "Array of port types: HW_DEFAULT (0) is default "
158 "1 for IB, 2 for Ethernet");
ab9c17a0
JM		 159
160struct mlx4_port_config {
161 struct list_head list;
162 enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
163 struct pci_dev *pdev;
164};
165
97989356
AV		 166static atomic_t pf_loading = ATOMIC_INIT(0);
167
27bf91d6
YP		 168int mlx4_check_port_params(struct mlx4_dev *dev,
169 enum mlx4_port_type *port_type)
7ff93f8b
YP		 170{
171 int i;
172
173 for (i = 0; i < dev->caps.num_ports - 1; i++) {
27bf91d6
YP		 174 if (port_type[i] != port_type[i + 1]) {
175 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
1a91de28 176 mlx4_err(dev, "Only same port types supported on this HCA, aborting\n");
27bf91d6
YP		 177 return -EINVAL;
178 }
7ff93f8b
YP		 179 }
180 }
7ff93f8b
YP		 181
182 for (i = 0; i < dev->caps.num_ports; i++) {
183 if (!(port_type[i] & dev->caps.supported_type[i+1])) {
1a91de28
JP		 184 mlx4_err(dev, "Requested port type for port %d is not supported on this HCA\n",
185 i + 1);
7ff93f8b
YP		 186 return -EINVAL;
187 }
188 }
189 return 0;
190}
191
192static void mlx4_set_port_mask(struct mlx4_dev *dev)
193{
194 int i;
195
7ff93f8b 196 for (i = 1; i <= dev->caps.num_ports; ++i)
65dab25d 197 dev->caps.port_mask[i] = dev->caps.port_type[i];
7ff93f8b 198}
f2a3f6a3 199
7ae0e400
MB		 200enum {
201 MLX4_QUERY_FUNC_NUM_SYS_EQS = 1 << 0,
202};
203
204static int mlx4_query_func(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
205{
206 int err = 0;
207 struct mlx4_func func;
208
209 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
210 err = mlx4_QUERY_FUNC(dev, &func, 0);
211 if (err) {
212 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
213 return err;
214 }
215 dev_cap->max_eqs = func.max_eq;
216 dev_cap->reserved_eqs = func.rsvd_eqs;
217 dev_cap->reserved_uars = func.rsvd_uars;
218 err |= MLX4_QUERY_FUNC_NUM_SYS_EQS;
219 }
220 return err;
221}
222
77507aa2
IS		 223static void mlx4_enable_cqe_eqe_stride(struct mlx4_dev *dev)
224{
225 struct mlx4_caps *dev_cap = &dev->caps;
226
227 /* FW not supporting or cancelled by user */
228 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_EQE_STRIDE) ||
229 !(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_CQE_STRIDE))
230 return;
231
232 /* Must have 64B CQE_EQE enabled by FW to use bigger stride
233 * When FW has NCSI it may decide not to report 64B CQE/EQEs
234 */
235 if (!(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_EQE) ||
236 !(dev_cap->flags & MLX4_DEV_CAP_FLAG_64B_CQE)) {
237 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
238 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
239 return;
240 }
241
242 if (cache_line_size() == 128 || cache_line_size() == 256) {
243 mlx4_dbg(dev, "Enabling CQE stride cacheLine supported\n");
244 /* Changing the real data inside CQE size to 32B */
245 dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
246 dev_cap->flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
247
248 if (mlx4_is_master(dev))
249 dev_cap->function_caps |= MLX4_FUNC_CAP_EQE_CQE_STRIDE;
250 } else {
251 mlx4_dbg(dev, "Disabling CQE stride cacheLine unsupported\n");
252 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
253 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
254 }
255}
256
3d73c288 257static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
225c7b1f
RD		 258{
259 int err;
5ae2a7a8 260 int i;
225c7b1f
RD		 261
262 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
263 if (err) {
1a91de28 264 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
225c7b1f
RD		 265 return err;
266 }
267
268 if (dev_cap->min_page_sz > PAGE_SIZE) {
1a91de28 269 mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
225c7b1f
RD		 270 dev_cap->min_page_sz, PAGE_SIZE);
271 return -ENODEV;
272 }
273 if (dev_cap->num_ports > MLX4_MAX_PORTS) {
1a91de28 274 mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
225c7b1f
RD		 275 dev_cap->num_ports, MLX4_MAX_PORTS);
276 return -ENODEV;
277 }
278
279 if (dev_cap->uar_size > pci_resource_len(dev->pdev, 2)) {
1a91de28 280 mlx4_err(dev, "HCA reported UAR size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
225c7b1f
RD		 281 dev_cap->uar_size,
282 (unsigned long long) pci_resource_len(dev->pdev, 2));
283 return -ENODEV;
284 }
285
286 dev->caps.num_ports = dev_cap->num_ports;
7ae0e400
MB		 287 dev->caps.num_sys_eqs = dev_cap->num_sys_eqs;
288 dev->phys_caps.num_phys_eqs = dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS ?
289 dev->caps.num_sys_eqs :
290 MLX4_MAX_EQ_NUM;
5ae2a7a8
RD		 291 for (i = 1; i <= dev->caps.num_ports; ++i) {
292 dev->caps.vl_cap[i] = dev_cap->max_vl[i];
b79acb49 293 dev->caps.ib_mtu_cap[i] = dev_cap->ib_mtu[i];
6634961c
JM		 294 dev->phys_caps.gid_phys_table_len[i] = dev_cap->max_gids[i];
295 dev->phys_caps.pkey_phys_table_len[i] = dev_cap->max_pkeys[i];
296 /* set gid and pkey table operating lengths by default
297 * to non-sriov values */
5ae2a7a8
RD		 298 dev->caps.gid_table_len[i] = dev_cap->max_gids[i];
299 dev->caps.pkey_table_len[i] = dev_cap->max_pkeys[i];
300 dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
b79acb49
YP		 301 dev->caps.eth_mtu_cap[i] = dev_cap->eth_mtu[i];
302 dev->caps.def_mac[i] = dev_cap->def_mac[i];
7ff93f8b 303 dev->caps.supported_type[i] = dev_cap->supported_port_types[i];
8d0fc7b6
YP		 304 dev->caps.suggested_type[i] = dev_cap->suggested_type[i];
305 dev->caps.default_sense[i] = dev_cap->default_sense[i];
7699517d
YP		 306 dev->caps.trans_type[i] = dev_cap->trans_type[i];
307 dev->caps.vendor_oui[i] = dev_cap->vendor_oui[i];
308 dev->caps.wavelength[i] = dev_cap->wavelength[i];
309 dev->caps.trans_code[i] = dev_cap->trans_code[i];
5ae2a7a8
RD		 310 }
311
ab9c17a0 312 dev->caps.uar_page_size = PAGE_SIZE;
225c7b1f 313 dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE;
225c7b1f
RD		 314 dev->caps.local_ca_ack_delay = dev_cap->local_ca_ack_delay;
315 dev->caps.bf_reg_size = dev_cap->bf_reg_size;
316 dev->caps.bf_regs_per_page = dev_cap->bf_regs_per_page;
317 dev->caps.max_sq_sg = dev_cap->max_sq_sg;
318 dev->caps.max_rq_sg = dev_cap->max_rq_sg;
319 dev->caps.max_wqes = dev_cap->max_qp_sz;
320 dev->caps.max_qp_init_rdma = dev_cap->max_requester_per_qp;
225c7b1f
RD		 321 dev->caps.max_srq_wqes = dev_cap->max_srq_sz;
322 dev->caps.max_srq_sge = dev_cap->max_rq_sg - 1;
323 dev->caps.reserved_srqs = dev_cap->reserved_srqs;
324 dev->caps.max_sq_desc_sz = dev_cap->max_sq_desc_sz;
325 dev->caps.max_rq_desc_sz = dev_cap->max_rq_desc_sz;
225c7b1f
RD		 326 /*
327 * Subtract 1 from the limit because we need to allocate a
328 * spare CQE so the HCA HW can tell the difference between an
329 * empty CQ and a full CQ.
330 */
331 dev->caps.max_cqes = dev_cap->max_cq_sz - 1;
332 dev->caps.reserved_cqs = dev_cap->reserved_cqs;
333 dev->caps.reserved_eqs = dev_cap->reserved_eqs;
2b8fb286 334 dev->caps.reserved_mtts = dev_cap->reserved_mtts;
225c7b1f 335 dev->caps.reserved_mrws = dev_cap->reserved_mrws;
ab9c17a0
JM		 336
337 /* The first 128 UARs are used for EQ doorbells */
338 dev->caps.reserved_uars = max_t(int, 128, dev_cap->reserved_uars);
225c7b1f 339 dev->caps.reserved_pds = dev_cap->reserved_pds;
012a8ff5
SH		 340 dev->caps.reserved_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
341 dev_cap->reserved_xrcds : 0;
342 dev->caps.max_xrcds = (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) ?
343 dev_cap->max_xrcds : 0;
2b8fb286
MA		 344 dev->caps.mtt_entry_sz = dev_cap->mtt_entry_sz;
345
149983af 346 dev->caps.max_msg_sz = dev_cap->max_msg_sz;
225c7b1f
RD		 347 dev->caps.page_size_cap = ~(u32) (dev_cap->min_page_sz - 1);
348 dev->caps.flags = dev_cap->flags;
b3416f44 349 dev->caps.flags2 = dev_cap->flags2;
95d04f07
RD		 350 dev->caps.bmme_flags = dev_cap->bmme_flags;
351 dev->caps.reserved_lkey = dev_cap->reserved_lkey;
225c7b1f 352 dev->caps.stat_rate_support = dev_cap->stat_rate_support;
b832be1e 353 dev->caps.max_gso_sz = dev_cap->max_gso_sz;
b3416f44 354 dev->caps.max_rss_tbl_sz = dev_cap->max_rss_tbl_sz;
225c7b1f 355
ca3e57a5
RD		 356 /* Sense port always allowed on supported devices for ConnectX-1 and -2 */
357 if (mlx4_priv(dev)->pci_dev_data & MLX4_PCI_DEV_FORCE_SENSE_PORT)
58a60168 358 dev->caps.flags |= MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
aadf4f3f
RD		 359 /* Don't do sense port on multifunction devices (for now at least) */
360 if (mlx4_is_mfunc(dev))
361 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_SENSE_SUPPORT;
58a60168 362
2599d858
AV		 363 if (mlx4_low_memory_profile()) {
364 dev->caps.log_num_macs = MLX4_MIN_LOG_NUM_MAC;
365 dev->caps.log_num_vlans = MLX4_MIN_LOG_NUM_VLANS;
366 } else {
367 dev->caps.log_num_macs = log_num_mac;
368 dev->caps.log_num_vlans = MLX4_LOG_NUM_VLANS;
369 }
93fc9e1b
YP		 370
371 for (i = 1; i <= dev->caps.num_ports; ++i) {
ab9c17a0
JM		 372 dev->caps.port_type[i] = MLX4_PORT_TYPE_NONE;
373 if (dev->caps.supported_type[i]) {
374 /* if only ETH is supported - assign ETH */
375 if (dev->caps.supported_type[i] == MLX4_PORT_TYPE_ETH)
376 dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
105c320f 377 /* if only IB is supported, assign IB */
ab9c17a0 378 else if (dev->caps.supported_type[i] ==
105c320f
JM		 379 MLX4_PORT_TYPE_IB)
380 dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
ab9c17a0 381 else {
105c320f
JM		 382 /* if IB and ETH are supported, we set the port
383 * type according to user selection of port type;
384 * if user selected none, take the FW hint */
385 if (port_type_array[i - 1] == MLX4_PORT_TYPE_NONE)
8d0fc7b6
YP		 386 dev->caps.port_type[i] = dev->caps.suggested_type[i] ?
387 MLX4_PORT_TYPE_ETH : MLX4_PORT_TYPE_IB;
ab9c17a0 388 else
105c320f 389 dev->caps.port_type[i] = port_type_array[i - 1];
ab9c17a0
JM		 390 }
391 }
8d0fc7b6
YP		 392 /*
393 * Link sensing is allowed on the port if 3 conditions are true:
394 * 1. Both protocols are supported on the port.
395 * 2. Different types are supported on the port
396 * 3. FW declared that it supports link sensing
397 */
27bf91d6 398 mlx4_priv(dev)->sense.sense_allowed[i] =
58a60168 399 ((dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO) &&
8d0fc7b6 400 (dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
58a60168 401 (dev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT));
7ff93f8b 402
8d0fc7b6
YP		 403 /*
404 * If "default_sense" bit is set, we move the port to "AUTO" mode
405 * and perform sense_port FW command to try and set the correct
406 * port type from beginning
407 */
46c46747 408 if (mlx4_priv(dev)->sense.sense_allowed[i] && dev->caps.default_sense[i]) {
8d0fc7b6
YP		 409 enum mlx4_port_type sensed_port = MLX4_PORT_TYPE_NONE;
410 dev->caps.possible_type[i] = MLX4_PORT_TYPE_AUTO;
411 mlx4_SENSE_PORT(dev, i, &sensed_port);
412 if (sensed_port != MLX4_PORT_TYPE_NONE)
413 dev->caps.port_type[i] = sensed_port;
414 } else {
415 dev->caps.possible_type[i] = dev->caps.port_type[i];
416 }
417
93fc9e1b
YP		 418 if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
419 dev->caps.log_num_macs = dev_cap->log_max_macs[i];
1a91de28 420 mlx4_warn(dev, "Requested number of MACs is too much for port %d, reducing to %d\n",
93fc9e1b
YP		 421 i, 1 << dev->caps.log_num_macs);
422 }
423 if (dev->caps.log_num_vlans > dev_cap->log_max_vlans[i]) {
424 dev->caps.log_num_vlans = dev_cap->log_max_vlans[i];
1a91de28 425 mlx4_warn(dev, "Requested number of VLANs is too much for port %d, reducing to %d\n",
93fc9e1b
YP		 426 i, 1 << dev->caps.log_num_vlans);
427 }
428 }
429
f2a3f6a3
OG		 430 dev->caps.max_counters = 1 << ilog2(dev_cap->max_counters);
431
93fc9e1b
YP		 432 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
433 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
434 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
435 (1 << dev->caps.log_num_macs) *
436 (1 << dev->caps.log_num_vlans) *
93fc9e1b
YP		 437 dev->caps.num_ports;
438 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH] = MLX4_NUM_FEXCH;
439
440 dev->caps.reserved_qps = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] +
441 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] +
442 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] +
443 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_EXCH];
444
e2c76824 445 dev->caps.sqp_demux = (mlx4_is_master(dev)) ? MLX4_MAX_NUM_SLAVES : 0;
08ff3235 446
b3051320 447 if (!enable_64b_cqe_eqe && !mlx4_is_slave(dev)) {
08ff3235
OG		 448 if (dev_cap->flags &
449 (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) {
450 mlx4_warn(dev, "64B EQEs/CQEs supported by the device but not enabled\n");
451 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_CQE;
452 dev->caps.flags &= ~MLX4_DEV_CAP_FLAG_64B_EQE;
453 }
77507aa2
IS		 454
455 if (dev_cap->flags2 &
456 (MLX4_DEV_CAP_FLAG2_CQE_STRIDE |
457 MLX4_DEV_CAP_FLAG2_EQE_STRIDE)) {
458 mlx4_warn(dev, "Disabling EQE/CQE stride per user request\n");
459 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_CQE_STRIDE;
460 dev_cap->flags2 &= ~MLX4_DEV_CAP_FLAG2_EQE_STRIDE;
461 }
08ff3235
OG		 462 }
463
f97b4b5d 464 if ((dev->caps.flags &
08ff3235
OG		 465 (MLX4_DEV_CAP_FLAG_64B_CQE | MLX4_DEV_CAP_FLAG_64B_EQE)) &&
466 mlx4_is_master(dev))
467 dev->caps.function_caps |= MLX4_FUNC_CAP_64B_EQE_CQE;
468
77507aa2
IS		 469 if (!mlx4_is_slave(dev))
470 mlx4_enable_cqe_eqe_stride(dev);
471
225c7b1f
RD		 472 return 0;
473}
b912b2f8
EP		 474
475static int mlx4_get_pcie_dev_link_caps(struct mlx4_dev *dev,
476 enum pci_bus_speed *speed,
477 enum pcie_link_width *width)
478{
479 u32 lnkcap1, lnkcap2;
480 int err1, err2;
481
482#define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */
483
484 *speed = PCI_SPEED_UNKNOWN;
485 *width = PCIE_LNK_WIDTH_UNKNOWN;
486
487 err1 = pcie_capability_read_dword(dev->pdev, PCI_EXP_LNKCAP, &lnkcap1);
488 err2 = pcie_capability_read_dword(dev->pdev, PCI_EXP_LNKCAP2, &lnkcap2);
489 if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */
490 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB)
491 *speed = PCIE_SPEED_8_0GT;
492 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB)
493 *speed = PCIE_SPEED_5_0GT;
494 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB)
495 *speed = PCIE_SPEED_2_5GT;
496 }
497 if (!err1) {
498 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT;
499 if (!lnkcap2) { /* pre-r3.0 */
500 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB)
501 *speed = PCIE_SPEED_5_0GT;
502 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB)
503 *speed = PCIE_SPEED_2_5GT;
504 }
505 }
506
507 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) {
508 return err1 ? err1 :
509 err2 ? err2 : -EINVAL;
510 }
511 return 0;
512}
513
514static void mlx4_check_pcie_caps(struct mlx4_dev *dev)
515{
516 enum pcie_link_width width, width_cap;
517 enum pci_bus_speed speed, speed_cap;
518 int err;
519
520#define PCIE_SPEED_STR(speed) \
521 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \
522 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \
523 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \
524 "Unknown")
525
526 err = mlx4_get_pcie_dev_link_caps(dev, &speed_cap, &width_cap);
527 if (err) {
528 mlx4_warn(dev,
529 "Unable to determine PCIe device BW capabilities\n");
530 return;
531 }
532
533 err = pcie_get_minimum_link(dev->pdev, &speed, &width);
534 if (err || speed == PCI_SPEED_UNKNOWN ||
535 width == PCIE_LNK_WIDTH_UNKNOWN) {
536 mlx4_warn(dev,
537 "Unable to determine PCI device chain minimum BW\n");
538 return;
539 }
540
541 if (width != width_cap || speed != speed_cap)
542 mlx4_warn(dev,
543 "PCIe BW is different than device's capability\n");
544
545 mlx4_info(dev, "PCIe link speed is %s, device supports %s\n",
546 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap));
547 mlx4_info(dev, "PCIe link width is x%d, device supports x%d\n",
548 width, width_cap);
549 return;
550}
551
ab9c17a0
JM		 552/*The function checks if there are live vf, return the num of them*/
553static int mlx4_how_many_lives_vf(struct mlx4_dev *dev)
554{
555 struct mlx4_priv *priv = mlx4_priv(dev);
556 struct mlx4_slave_state *s_state;
557 int i;
558 int ret = 0;
559
560 for (i = 1/*the ppf is 0*/; i < dev->num_slaves; ++i) {
561 s_state = &priv->mfunc.master.slave_state[i];
562 if (s_state->active && s_state->last_cmd !=
563 MLX4_COMM_CMD_RESET) {
564 mlx4_warn(dev, "%s: slave: %d is still active\n",
565 __func__, i);
566 ret++;
567 }
568 }
569 return ret;
570}
571
396f2feb
JM		 572int mlx4_get_parav_qkey(struct mlx4_dev *dev, u32 qpn, u32 *qkey)
573{
574 u32 qk = MLX4_RESERVED_QKEY_BASE;
47605df9
JM		 575
576 if (qpn >= dev->phys_caps.base_tunnel_sqpn + 8 * MLX4_MFUNC_MAX ||
577 qpn < dev->phys_caps.base_proxy_sqpn)
396f2feb
JM		 578 return -EINVAL;
579
47605df9 580 if (qpn >= dev->phys_caps.base_tunnel_sqpn)
396f2feb 581 /* tunnel qp */
47605df9 582 qk += qpn - dev->phys_caps.base_tunnel_sqpn;
396f2feb 583 else
47605df9 584 qk += qpn - dev->phys_caps.base_proxy_sqpn;
396f2feb
JM		 585 *qkey = qk;
586 return 0;
587}
588EXPORT_SYMBOL(mlx4_get_parav_qkey);
589
54679e14
JM		 590void mlx4_sync_pkey_table(struct mlx4_dev *dev, int slave, int port, int i, int val)
591{
592 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
593
594 if (!mlx4_is_master(dev))
595 return;
596
597 priv->virt2phys_pkey[slave][port - 1][i] = val;
598}
599EXPORT_SYMBOL(mlx4_sync_pkey_table);
600
afa8fd1d
JM		 601void mlx4_put_slave_node_guid(struct mlx4_dev *dev, int slave, __be64 guid)
602{
603 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
604
605 if (!mlx4_is_master(dev))
606 return;
607
608 priv->slave_node_guids[slave] = guid;
609}
610EXPORT_SYMBOL(mlx4_put_slave_node_guid);
611
612__be64 mlx4_get_slave_node_guid(struct mlx4_dev *dev, int slave)
613{
614 struct mlx4_priv *priv = container_of(dev, struct mlx4_priv, dev);
615
616 if (!mlx4_is_master(dev))
617 return 0;
618
619 return priv->slave_node_guids[slave];
620}
621EXPORT_SYMBOL(mlx4_get_slave_node_guid);
622
e10903b0 623int mlx4_is_slave_active(struct mlx4_dev *dev, int slave)
ab9c17a0
JM		 624{
625 struct mlx4_priv *priv = mlx4_priv(dev);
626 struct mlx4_slave_state *s_slave;
627
628 if (!mlx4_is_master(dev))
629 return 0;
630
631 s_slave = &priv->mfunc.master.slave_state[slave];
632 return !!s_slave->active;
633}
634EXPORT_SYMBOL(mlx4_is_slave_active);
635
7b8157be
JM		 636static void slave_adjust_steering_mode(struct mlx4_dev *dev,
637 struct mlx4_dev_cap *dev_cap,
638 struct mlx4_init_hca_param *hca_param)
639{
640 dev->caps.steering_mode = hca_param->steering_mode;
641 if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
642 dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
643 dev->caps.fs_log_max_ucast_qp_range_size =
644 dev_cap->fs_log_max_ucast_qp_range_size;
645 } else
646 dev->caps.num_qp_per_mgm =
647 4 * ((1 << hca_param->log_mc_entry_sz)/16 - 2);
648
649 mlx4_dbg(dev, "Steering mode is: %s\n",
650 mlx4_steering_mode_str(dev->caps.steering_mode));
651}
652
ab9c17a0
JM		 653static int mlx4_slave_cap(struct mlx4_dev *dev)
654{
655 int err;
656 u32 page_size;
657 struct mlx4_dev_cap dev_cap;
658 struct mlx4_func_cap func_cap;
659 struct mlx4_init_hca_param hca_param;
225c6c8c 660 u8 i;
ab9c17a0
JM		 661
662 memset(&hca_param, 0, sizeof(hca_param));
663 err = mlx4_QUERY_HCA(dev, &hca_param);
664 if (err) {
1a91de28 665 mlx4_err(dev, "QUERY_HCA command failed, aborting\n");
ab9c17a0
JM		 666 return err;
667 }
668
483e0132
EP		 669 /* fail if the hca has an unknown global capability
670 * at this time global_caps should be always zeroed
671 */
672 if (hca_param.global_caps) {
ab9c17a0
JM		 673 mlx4_err(dev, "Unknown hca global capabilities\n");
674 return -ENOSYS;
675 }
676
677 mlx4_log_num_mgm_entry_size = hca_param.log_mc_entry_sz;
678
ddd8a6c1
EE		 679 dev->caps.hca_core_clock = hca_param.hca_core_clock;
680
ab9c17a0 681 memset(&dev_cap, 0, sizeof(dev_cap));
b91cb3eb 682 dev->caps.max_qp_dest_rdma = 1 << hca_param.log_rd_per_qp;
ab9c17a0
JM		 683 err = mlx4_dev_cap(dev, &dev_cap);
684 if (err) {
1a91de28 685 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
ab9c17a0
JM		 686 return err;
687 }
688
b91cb3eb
JM		 689 err = mlx4_QUERY_FW(dev);
690 if (err)
1a91de28 691 mlx4_err(dev, "QUERY_FW command failed: could not get FW version\n");
b91cb3eb 692
ab9c17a0
JM		 693 page_size = ~dev->caps.page_size_cap + 1;
694 mlx4_warn(dev, "HCA minimum page size:%d\n", page_size);
695 if (page_size > PAGE_SIZE) {
1a91de28 696 mlx4_err(dev, "HCA minimum page size of %d bigger than kernel PAGE_SIZE of %ld, aborting\n",
ab9c17a0
JM		 697 page_size, PAGE_SIZE);
698 return -ENODEV;
699 }
700
701 /* slave gets uar page size from QUERY_HCA fw command */
702 dev->caps.uar_page_size = 1 << (hca_param.uar_page_sz + 12);
703
704 /* TODO: relax this assumption */
705 if (dev->caps.uar_page_size != PAGE_SIZE) {
706 mlx4_err(dev, "UAR size:%d != kernel PAGE_SIZE of %ld\n",
707 dev->caps.uar_page_size, PAGE_SIZE);
708 return -ENODEV;
709 }
710
711 memset(&func_cap, 0, sizeof(func_cap));
47605df9 712 err = mlx4_QUERY_FUNC_CAP(dev, 0, &func_cap);
ab9c17a0 713 if (err) {
1a91de28
JP		 714 mlx4_err(dev, "QUERY_FUNC_CAP general command failed, aborting (%d)\n",
715 err);
ab9c17a0
JM		 716 return err;
717 }
718
719 if ((func_cap.pf_context_behaviour | PF_CONTEXT_BEHAVIOUR_MASK) !=
720 PF_CONTEXT_BEHAVIOUR_MASK) {
721 mlx4_err(dev, "Unknown pf context behaviour\n");
722 return -ENOSYS;
723 }
724
ab9c17a0 725 dev->caps.num_ports = func_cap.num_ports;
5a0d0a61
JM		 726 dev->quotas.qp = func_cap.qp_quota;
727 dev->quotas.srq = func_cap.srq_quota;
728 dev->quotas.cq = func_cap.cq_quota;
729 dev->quotas.mpt = func_cap.mpt_quota;
730 dev->quotas.mtt = func_cap.mtt_quota;
731 dev->caps.num_qps = 1 << hca_param.log_num_qps;
732 dev->caps.num_srqs = 1 << hca_param.log_num_srqs;
733 dev->caps.num_cqs = 1 << hca_param.log_num_cqs;
734 dev->caps.num_mpts = 1 << hca_param.log_mpt_sz;
735 dev->caps.num_eqs = func_cap.max_eq;
736 dev->caps.reserved_eqs = func_cap.reserved_eq;
ab9c17a0
JM		 737 dev->caps.num_pds = MLX4_NUM_PDS;
738 dev->caps.num_mgms = 0;
739 dev->caps.num_amgms = 0;
740
ab9c17a0 741 if (dev->caps.num_ports > MLX4_MAX_PORTS) {
1a91de28
JP		 742 mlx4_err(dev, "HCA has %d ports, but we only support %d, aborting\n",
743 dev->caps.num_ports, MLX4_MAX_PORTS);
ab9c17a0
JM		 744 return -ENODEV;
745 }
746
99ec41d0 747 dev->caps.qp0_qkey = kcalloc(dev->caps.num_ports, sizeof(u32), GFP_KERNEL);
47605df9
JM		 748 dev->caps.qp0_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
749 dev->caps.qp0_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
750 dev->caps.qp1_tunnel = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
751 dev->caps.qp1_proxy = kcalloc(dev->caps.num_ports, sizeof (u32), GFP_KERNEL);
752
753 if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
99ec41d0
JM		 754 !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy ||
755 !dev->caps.qp0_qkey) {
47605df9
JM		 756 err = -ENOMEM;
757 goto err_mem;
758 }
759
6634961c 760 for (i = 1; i <= dev->caps.num_ports; ++i) {
225c6c8c 761 err = mlx4_QUERY_FUNC_CAP(dev, i, &func_cap);
47605df9 762 if (err) {
1a91de28
JP		 763 mlx4_err(dev, "QUERY_FUNC_CAP port command failed for port %d, aborting (%d)\n",
764 i, err);
47605df9
JM		 765 goto err_mem;
766 }
99ec41d0 767 dev->caps.qp0_qkey[i - 1] = func_cap.qp0_qkey;
47605df9
JM		 768 dev->caps.qp0_tunnel[i - 1] = func_cap.qp0_tunnel_qpn;
769 dev->caps.qp0_proxy[i - 1] = func_cap.qp0_proxy_qpn;
770 dev->caps.qp1_tunnel[i - 1] = func_cap.qp1_tunnel_qpn;
771 dev->caps.qp1_proxy[i - 1] = func_cap.qp1_proxy_qpn;
6230bb23 772 dev->caps.port_mask[i] = dev->caps.port_type[i];
8e1a28e8 773 dev->caps.phys_port_id[i] = func_cap.phys_port_id;
6634961c
JM		 774 if (mlx4_get_slave_pkey_gid_tbl_len(dev, i,
775 &dev->caps.gid_table_len[i],
776 &dev->caps.pkey_table_len[i]))
47605df9 777 goto err_mem;
6634961c 778 }
6230bb23 779
ab9c17a0
JM		 780 if (dev->caps.uar_page_size * (dev->caps.num_uars -
781 dev->caps.reserved_uars) >
782 pci_resource_len(dev->pdev, 2)) {
1a91de28 783 mlx4_err(dev, "HCA reported UAR region size of 0x%x bigger than PCI resource 2 size of 0x%llx, aborting\n",
ab9c17a0
JM		 784 dev->caps.uar_page_size * dev->caps.num_uars,
785 (unsigned long long) pci_resource_len(dev->pdev, 2));
47605df9 786 goto err_mem;
ab9c17a0
JM		 787 }
788
08ff3235
OG		 789 if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_EQE_ENABLED) {
790 dev->caps.eqe_size = 64;
791 dev->caps.eqe_factor = 1;
792 } else {
793 dev->caps.eqe_size = 32;
794 dev->caps.eqe_factor = 0;
795 }
796
797 if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_64B_CQE_ENABLED) {
798 dev->caps.cqe_size = 64;
77507aa2 799 dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
08ff3235
OG		 800 } else {
801 dev->caps.cqe_size = 32;
802 }
803
77507aa2
IS		 804 if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_EQE_STRIDE_ENABLED) {
805 dev->caps.eqe_size = hca_param.eqe_size;
806 dev->caps.eqe_factor = 0;
807 }
808
809 if (hca_param.dev_cap_enabled & MLX4_DEV_CAP_CQE_STRIDE_ENABLED) {
810 dev->caps.cqe_size = hca_param.cqe_size;
811 /* User still need to know when CQE > 32B */
812 dev->caps.userspace_caps |= MLX4_USER_DEV_CAP_LARGE_CQE;
813 }
814
f9bd2d7f 815 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1a91de28 816 mlx4_warn(dev, "Timestamping is not supported in slave mode\n");
f9bd2d7f 817
7b8157be
JM		 818 slave_adjust_steering_mode(dev, &dev_cap, &hca_param);
819
ab9c17a0 820 return 0;
47605df9
JM		 821
822err_mem:
99ec41d0 823 kfree(dev->caps.qp0_qkey);
47605df9
JM		 824 kfree(dev->caps.qp0_tunnel);
825 kfree(dev->caps.qp0_proxy);
826 kfree(dev->caps.qp1_tunnel);
827 kfree(dev->caps.qp1_proxy);
99ec41d0
JM		 828 dev->caps.qp0_qkey = NULL;
829 dev->caps.qp0_tunnel = NULL;
830 dev->caps.qp0_proxy = NULL;
831 dev->caps.qp1_tunnel = NULL;
832 dev->caps.qp1_proxy = NULL;
47605df9
JM		 833
834 return err;
ab9c17a0 835}
225c7b1f 836
b046ffe5
EP		 837static void mlx4_request_modules(struct mlx4_dev *dev)
838{
839 int port;
840 int has_ib_port = false;
841 int has_eth_port = false;
842#define EN_DRV_NAME "mlx4_en"
843#define IB_DRV_NAME "mlx4_ib"
844
845 for (port = 1; port <= dev->caps.num_ports; port++) {
846 if (dev->caps.port_type[port] == MLX4_PORT_TYPE_IB)
847 has_ib_port = true;
848 else if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
849 has_eth_port = true;
850 }
851
b046ffe5
EP		 852 if (has_eth_port)
853 request_module_nowait(EN_DRV_NAME);
f24f790f
OG		 854 if (has_ib_port || (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE))
855 request_module_nowait(IB_DRV_NAME);
b046ffe5
EP		 856}
857
7ff93f8b
YP		 858/*
859 * Change the port configuration of the device.
860 * Every user of this function must hold the port mutex.
861 */
27bf91d6
YP		 862int mlx4_change_port_types(struct mlx4_dev *dev,
863 enum mlx4_port_type *port_types)
7ff93f8b
YP		 864{
865 int err = 0;
866 int change = 0;
867 int port;
868
869 for (port = 0; port < dev->caps.num_ports; port++) {
27bf91d6
YP		 870 /* Change the port type only if the new type is different
871 * from the current, and not set to Auto */
3d8f9308 872 if (port_types[port] != dev->caps.port_type[port + 1])
7ff93f8b 873 change = 1;
7ff93f8b
YP		 874 }
875 if (change) {
876 mlx4_unregister_device(dev);
877 for (port = 1; port <= dev->caps.num_ports; port++) {
878 mlx4_CLOSE_PORT(dev, port);
1e0f03d5 879 dev->caps.port_type[port] = port_types[port - 1];
6634961c 880 err = mlx4_SET_PORT(dev, port, -1);
7ff93f8b 881 if (err) {
1a91de28
JP		 882 mlx4_err(dev, "Failed to set port %d, aborting\n",
883 port);
7ff93f8b
YP		 884 goto out;
885 }
886 }
887 mlx4_set_port_mask(dev);
888 err = mlx4_register_device(dev);
b046ffe5
EP		 889 if (err) {
890 mlx4_err(dev, "Failed to register device\n");
891 goto out;
892 }
893 mlx4_request_modules(dev);
7ff93f8b
YP		 894 }
895
896out:
897 return err;
898}
899
900static ssize_t show_port_type(struct device *dev,
901 struct device_attribute *attr,
902 char *buf)
903{
904 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
905 port_attr);
906 struct mlx4_dev *mdev = info->dev;
27bf91d6
YP		 907 char type[8];
908
909 sprintf(type, "%s",
910 (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
911 "ib" : "eth");
912 if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
913 sprintf(buf, "auto (%s)\n", type);
914 else
915 sprintf(buf, "%s\n", type);
7ff93f8b 916
27bf91d6 917 return strlen(buf);
7ff93f8b
YP		 918}
919
920static ssize_t set_port_type(struct device *dev,
921 struct device_attribute *attr,
922 const char *buf, size_t count)
923{
924 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
925 port_attr);
926 struct mlx4_dev *mdev = info->dev;
927 struct mlx4_priv *priv = mlx4_priv(mdev);
928 enum mlx4_port_type types[MLX4_MAX_PORTS];
27bf91d6 929 enum mlx4_port_type new_types[MLX4_MAX_PORTS];
0a984556 930 static DEFINE_MUTEX(set_port_type_mutex);
7ff93f8b
YP		 931 int i;
932 int err = 0;
933
0a984556
AV		 934 mutex_lock(&set_port_type_mutex);
935
7ff93f8b
YP		 936 if (!strcmp(buf, "ib\n"))
937 info->tmp_type = MLX4_PORT_TYPE_IB;
938 else if (!strcmp(buf, "eth\n"))
939 info->tmp_type = MLX4_PORT_TYPE_ETH;
27bf91d6
YP		 940 else if (!strcmp(buf, "auto\n"))
941 info->tmp_type = MLX4_PORT_TYPE_AUTO;
7ff93f8b
YP		 942 else {
943 mlx4_err(mdev, "%s is not supported port type\n", buf);
0a984556
AV		 944 err = -EINVAL;
945 goto err_out;
7ff93f8b
YP		 946 }
947
27bf91d6 948 mlx4_stop_sense(mdev);
7ff93f8b 949 mutex_lock(&priv->port_mutex);
27bf91d6
YP		 950 /* Possible type is always the one that was delivered */
951 mdev->caps.possible_type[info->port] = info->tmp_type;
952
953 for (i = 0; i < mdev->caps.num_ports; i++) {
7ff93f8b 954 types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
27bf91d6
YP		 955 mdev->caps.possible_type[i+1];
956 if (types[i] == MLX4_PORT_TYPE_AUTO)
957 types[i] = mdev->caps.port_type[i+1];
958 }
7ff93f8b 959
58a60168
YP		 960 if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP) &&
961 !(mdev->caps.flags & MLX4_DEV_CAP_FLAG_SENSE_SUPPORT)) {
27bf91d6
YP		 962 for (i = 1; i <= mdev->caps.num_ports; i++) {
963 if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
964 mdev->caps.possible_type[i] = mdev->caps.port_type[i];
965 err = -EINVAL;
966 }
967 }
968 }
969 if (err) {
1a91de28 970 mlx4_err(mdev, "Auto sensing is not supported on this HCA. Set only 'eth' or 'ib' for both ports (should be the same)\n");
27bf91d6
YP		 971 goto out;
972 }
973
974 mlx4_do_sense_ports(mdev, new_types, types);
975
976 err = mlx4_check_port_params(mdev, new_types);
7ff93f8b
YP		 977 if (err)
978 goto out;
979
27bf91d6
YP		 980 /* We are about to apply the changes after the configuration
981 * was verified, no need to remember the temporary types
982 * any more */
983 for (i = 0; i < mdev->caps.num_ports; i++)
984 priv->port[i + 1].tmp_type = 0;
7ff93f8b 985
27bf91d6 986 err = mlx4_change_port_types(mdev, new_types);
7ff93f8b
YP		 987
988out:
27bf91d6 989 mlx4_start_sense(mdev);
7ff93f8b 990 mutex_unlock(&priv->port_mutex);
0a984556
AV		 991err_out:
992 mutex_unlock(&set_port_type_mutex);
993
7ff93f8b
YP		 994 return err ? err : count;
995}
996
096335b3
OG		 997enum ibta_mtu {
998 IB_MTU_256 = 1,
999 IB_MTU_512 = 2,
1000 IB_MTU_1024 = 3,
1001 IB_MTU_2048 = 4,
1002 IB_MTU_4096 = 5
1003};
1004
1005static inline int int_to_ibta_mtu(int mtu)
1006{
1007 switch (mtu) {
1008 case 256: return IB_MTU_256;
1009 case 512: return IB_MTU_512;
1010 case 1024: return IB_MTU_1024;
1011 case 2048: return IB_MTU_2048;
1012 case 4096: return IB_MTU_4096;
1013 default: return -1;
1014 }
1015}
1016
1017static inline int ibta_mtu_to_int(enum ibta_mtu mtu)
1018{
1019 switch (mtu) {
1020 case IB_MTU_256: return 256;
1021 case IB_MTU_512: return 512;
1022 case IB_MTU_1024: return 1024;
1023 case IB_MTU_2048: return 2048;
1024 case IB_MTU_4096: return 4096;
1025 default: return -1;
1026 }
1027}
1028
1029static ssize_t show_port_ib_mtu(struct device *dev,
1030 struct device_attribute *attr,
1031 char *buf)
1032{
1033 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1034 port_mtu_attr);
1035 struct mlx4_dev *mdev = info->dev;
1036
1037 if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH)
1038 mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1039
1040 sprintf(buf, "%d\n",
1041 ibta_mtu_to_int(mdev->caps.port_ib_mtu[info->port]));
1042 return strlen(buf);
1043}
1044
1045static ssize_t set_port_ib_mtu(struct device *dev,
1046 struct device_attribute *attr,
1047 const char *buf, size_t count)
1048{
1049 struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
1050 port_mtu_attr);
1051 struct mlx4_dev *mdev = info->dev;
1052 struct mlx4_priv *priv = mlx4_priv(mdev);
1053 int err, port, mtu, ibta_mtu = -1;
1054
1055 if (mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_ETH) {
1056 mlx4_warn(mdev, "port level mtu is only used for IB ports\n");
1057 return -EINVAL;
1058 }
1059
618fad95
DB		 1060 err = kstrtoint(buf, 0, &mtu);
1061 if (!err)
096335b3
OG		 1062 ibta_mtu = int_to_ibta_mtu(mtu);
1063
618fad95 1064 if (err || ibta_mtu < 0) {
096335b3
OG		 1065 mlx4_err(mdev, "%s is invalid IBTA mtu\n", buf);
1066 return -EINVAL;
1067 }
1068
1069 mdev->caps.port_ib_mtu[info->port] = ibta_mtu;
1070
1071 mlx4_stop_sense(mdev);
1072 mutex_lock(&priv->port_mutex);
1073 mlx4_unregister_device(mdev);
1074 for (port = 1; port <= mdev->caps.num_ports; port++) {
1075 mlx4_CLOSE_PORT(mdev, port);
6634961c 1076 err = mlx4_SET_PORT(mdev, port, -1);
096335b3 1077 if (err) {
1a91de28
JP		 1078 mlx4_err(mdev, "Failed to set port %d, aborting\n",
1079 port);
096335b3
OG		 1080 goto err_set_port;
1081 }
1082 }
1083 err = mlx4_register_device(mdev);
1084err_set_port:
1085 mutex_unlock(&priv->port_mutex);
1086 mlx4_start_sense(mdev);
1087 return err ? err : count;
1088}
1089
e8f9b2ed 1090static int mlx4_load_fw(struct mlx4_dev *dev)
225c7b1f
RD		 1091{
1092 struct mlx4_priv *priv = mlx4_priv(dev);
1093 int err;
1094
1095 priv->fw.fw_icm = mlx4_alloc_icm(dev, priv->fw.fw_pages,
5b0bf5e2 1096 GFP_HIGHUSER | __GFP_NOWARN, 0);
225c7b1f 1097 if (!priv->fw.fw_icm) {
1a91de28 1098 mlx4_err(dev, "Couldn't allocate FW area, aborting\n");
225c7b1f
RD		 1099 return -ENOMEM;
1100 }
1101
1102 err = mlx4_MAP_FA(dev, priv->fw.fw_icm);
1103 if (err) {
1a91de28 1104 mlx4_err(dev, "MAP_FA command failed, aborting\n");
225c7b1f
RD		 1105 goto err_free;
1106 }
1107
1108 err = mlx4_RUN_FW(dev);
1109 if (err) {
1a91de28 1110 mlx4_err(dev, "RUN_FW command failed, aborting\n");
225c7b1f
RD		 1111 goto err_unmap_fa;
1112 }
1113
1114 return 0;
1115
1116err_unmap_fa:
1117 mlx4_UNMAP_FA(dev);
1118
1119err_free:
5b0bf5e2 1120 mlx4_free_icm(dev, priv->fw.fw_icm, 0);
225c7b1f
RD		 1121 return err;
1122}
1123
e8f9b2ed
RD		 1124static int mlx4_init_cmpt_table(struct mlx4_dev *dev, u64 cmpt_base,
1125 int cmpt_entry_sz)
225c7b1f
RD		 1126{
1127 struct mlx4_priv *priv = mlx4_priv(dev);
1128 int err;
ab9c17a0 1129 int num_eqs;
225c7b1f
RD		 1130
1131 err = mlx4_init_icm_table(dev, &priv->qp_table.cmpt_table,
1132 cmpt_base +
1133 ((u64) (MLX4_CMPT_TYPE_QP *
1134 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1135 cmpt_entry_sz, dev->caps.num_qps,
93fc9e1b
YP		 1136 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1137 0, 0);
225c7b1f
RD		 1138 if (err)
1139 goto err;
1140
1141 err = mlx4_init_icm_table(dev, &priv->srq_table.cmpt_table,
1142 cmpt_base +
1143 ((u64) (MLX4_CMPT_TYPE_SRQ *
1144 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1145 cmpt_entry_sz, dev->caps.num_srqs,
5b0bf5e2 1146 dev->caps.reserved_srqs, 0, 0);
225c7b1f
RD		 1147 if (err)
1148 goto err_qp;
1149
1150 err = mlx4_init_icm_table(dev, &priv->cq_table.cmpt_table,
1151 cmpt_base +
1152 ((u64) (MLX4_CMPT_TYPE_CQ *
1153 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
1154 cmpt_entry_sz, dev->caps.num_cqs,
5b0bf5e2 1155 dev->caps.reserved_cqs, 0, 0);
225c7b1f
RD		 1156 if (err)
1157 goto err_srq;
1158
7ae0e400 1159 num_eqs = dev->phys_caps.num_phys_eqs;
225c7b1f
RD		 1160 err = mlx4_init_icm_table(dev, &priv->eq_table.cmpt_table,
1161 cmpt_base +
1162 ((u64) (MLX4_CMPT_TYPE_EQ *
1163 cmpt_entry_sz) << MLX4_CMPT_SHIFT),
ab9c17a0 1164 cmpt_entry_sz, num_eqs, num_eqs, 0, 0);
225c7b1f
RD		 1165 if (err)
1166 goto err_cq;
1167
1168 return 0;
1169
1170err_cq:
1171 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1172
1173err_srq:
1174 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1175
1176err_qp:
1177 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
1178
1179err:
1180 return err;
1181}
1182
3d73c288
RD		 1183static int mlx4_init_icm(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
1184 struct mlx4_init_hca_param *init_hca, u64 icm_size)
225c7b1f
RD		 1185{
1186 struct mlx4_priv *priv = mlx4_priv(dev);
1187 u64 aux_pages;
ab9c17a0 1188 int num_eqs;
225c7b1f
RD		 1189 int err;
1190
1191 err = mlx4_SET_ICM_SIZE(dev, icm_size, &aux_pages);
1192 if (err) {
1a91de28 1193 mlx4_err(dev, "SET_ICM_SIZE command failed, aborting\n");
225c7b1f
RD		 1194 return err;
1195 }
1196
1a91de28 1197 mlx4_dbg(dev, "%lld KB of HCA context requires %lld KB aux memory\n",
225c7b1f
RD		 1198 (unsigned long long) icm_size >> 10,
1199 (unsigned long long) aux_pages << 2);
1200
1201 priv->fw.aux_icm = mlx4_alloc_icm(dev, aux_pages,
5b0bf5e2 1202 GFP_HIGHUSER | __GFP_NOWARN, 0);
225c7b1f 1203 if (!priv->fw.aux_icm) {
1a91de28 1204 mlx4_err(dev, "Couldn't allocate aux memory, aborting\n");
225c7b1f
RD		 1205 return -ENOMEM;
1206 }
1207
1208 err = mlx4_MAP_ICM_AUX(dev, priv->fw.aux_icm);
1209 if (err) {
1a91de28 1210 mlx4_err(dev, "MAP_ICM_AUX command failed, aborting\n");
225c7b1f
RD		 1211 goto err_free_aux;
1212 }
1213
1214 err = mlx4_init_cmpt_table(dev, init_hca->cmpt_base, dev_cap->cmpt_entry_sz);
1215 if (err) {
1a91de28 1216 mlx4_err(dev, "Failed to map cMPT context memory, aborting\n");
225c7b1f
RD		 1217 goto err_unmap_aux;
1218 }
1219
ab9c17a0 1220
7ae0e400 1221 num_eqs = dev->phys_caps.num_phys_eqs;
fa0681d2
RD		 1222 err = mlx4_init_icm_table(dev, &priv->eq_table.table,
1223 init_hca->eqc_base, dev_cap->eqc_entry_sz,
ab9c17a0 1224 num_eqs, num_eqs, 0, 0);
225c7b1f 1225 if (err) {
1a91de28 1226 mlx4_err(dev, "Failed to map EQ context memory, aborting\n");
225c7b1f
RD		 1227 goto err_unmap_cmpt;
1228 }
1229
d7bb58fb
JM		 1230 /*
1231 * Reserved MTT entries must be aligned up to a cacheline
1232 * boundary, since the FW will write to them, while the driver
1233 * writes to all other MTT entries. (The variable
1234 * dev->caps.mtt_entry_sz below is really the MTT segment
1235 * size, not the raw entry size)
1236 */
1237 dev->caps.reserved_mtts =
1238 ALIGN(dev->caps.reserved_mtts * dev->caps.mtt_entry_sz,
1239 dma_get_cache_alignment()) / dev->caps.mtt_entry_sz;
1240
225c7b1f
RD		 1241 err = mlx4_init_icm_table(dev, &priv->mr_table.mtt_table,
1242 init_hca->mtt_base,
1243 dev->caps.mtt_entry_sz,
2b8fb286 1244 dev->caps.num_mtts,
5b0bf5e2 1245 dev->caps.reserved_mtts, 1, 0);
225c7b1f 1246 if (err) {
1a91de28 1247 mlx4_err(dev, "Failed to map MTT context memory, aborting\n");
225c7b1f
RD		 1248 goto err_unmap_eq;
1249 }
1250
1251 err = mlx4_init_icm_table(dev, &priv->mr_table.dmpt_table,
1252 init_hca->dmpt_base,
1253 dev_cap->dmpt_entry_sz,
1254 dev->caps.num_mpts,
5b0bf5e2 1255 dev->caps.reserved_mrws, 1, 1);
225c7b1f 1256 if (err) {
1a91de28 1257 mlx4_err(dev, "Failed to map dMPT context memory, aborting\n");
225c7b1f
RD		 1258 goto err_unmap_mtt;
1259 }
1260
1261 err = mlx4_init_icm_table(dev, &priv->qp_table.qp_table,
1262 init_hca->qpc_base,
1263 dev_cap->qpc_entry_sz,
1264 dev->caps.num_qps,
93fc9e1b
YP		 1265 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1266 0, 0);
225c7b1f 1267 if (err) {
1a91de28 1268 mlx4_err(dev, "Failed to map QP context memory, aborting\n");
225c7b1f
RD		 1269 goto err_unmap_dmpt;
1270 }
1271
1272 err = mlx4_init_icm_table(dev, &priv->qp_table.auxc_table,
1273 init_hca->auxc_base,
1274 dev_cap->aux_entry_sz,
1275 dev->caps.num_qps,
93fc9e1b
YP		 1276 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1277 0, 0);
225c7b1f 1278 if (err) {
1a91de28 1279 mlx4_err(dev, "Failed to map AUXC context memory, aborting\n");
225c7b1f
RD		 1280 goto err_unmap_qp;
1281 }
1282
1283 err = mlx4_init_icm_table(dev, &priv->qp_table.altc_table,
1284 init_hca->altc_base,
1285 dev_cap->altc_entry_sz,
1286 dev->caps.num_qps,
93fc9e1b
YP		 1287 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1288 0, 0);
225c7b1f 1289 if (err) {
1a91de28 1290 mlx4_err(dev, "Failed to map ALTC context memory, aborting\n");
225c7b1f
RD		 1291 goto err_unmap_auxc;
1292 }
1293
1294 err = mlx4_init_icm_table(dev, &priv->qp_table.rdmarc_table,
1295 init_hca->rdmarc_base,
1296 dev_cap->rdmarc_entry_sz << priv->qp_table.rdmarc_shift,
1297 dev->caps.num_qps,
93fc9e1b
YP		 1298 dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
1299 0, 0);
225c7b1f
RD		 1300 if (err) {
1301 mlx4_err(dev, "Failed to map RDMARC context memory, aborting\n");
1302 goto err_unmap_altc;
1303 }
1304
1305 err = mlx4_init_icm_table(dev, &priv->cq_table.table,
1306 init_hca->cqc_base,
1307 dev_cap->cqc_entry_sz,
1308 dev->caps.num_cqs,
5b0bf5e2 1309 dev->caps.reserved_cqs, 0, 0);
225c7b1f 1310 if (err) {
1a91de28 1311 mlx4_err(dev, "Failed to map CQ context memory, aborting\n");
225c7b1f
RD		 1312 goto err_unmap_rdmarc;
1313 }
1314
1315 err = mlx4_init_icm_table(dev, &priv->srq_table.table,
1316 init_hca->srqc_base,
1317 dev_cap->srq_entry_sz,
1318 dev->caps.num_srqs,
5b0bf5e2 1319 dev->caps.reserved_srqs, 0, 0);
225c7b1f 1320 if (err) {
1a91de28 1321 mlx4_err(dev, "Failed to map SRQ context memory, aborting\n");
225c7b1f
RD		 1322 goto err_unmap_cq;
1323 }
1324
1325 /*
0ff1fb65
HHZ		 1326 * For flow steering device managed mode it is required to use
1327 * mlx4_init_icm_table. For B0 steering mode it's not strictly
1328 * required, but for simplicity just map the whole multicast
1329 * group table now. The table isn't very big and it's a lot
1330 * easier than trying to track ref counts.
225c7b1f
RD		 1331 */
1332 err = mlx4_init_icm_table(dev, &priv->mcg_table.table,
0ec2c0f8
EE		 1333 init_hca->mc_base,
1334 mlx4_get_mgm_entry_size(dev),
225c7b1f
RD		 1335 dev->caps.num_mgms + dev->caps.num_amgms,
1336 dev->caps.num_mgms + dev->caps.num_amgms,
5b0bf5e2 1337 0, 0);
225c7b1f 1338 if (err) {
1a91de28 1339 mlx4_err(dev, "Failed to map MCG context memory, aborting\n");
225c7b1f
RD		 1340 goto err_unmap_srq;
1341 }
1342
1343 return 0;
1344
1345err_unmap_srq:
1346 mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
1347
1348err_unmap_cq:
1349 mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
1350
1351err_unmap_rdmarc:
1352 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
1353
1354err_unmap_altc:
1355 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
1356
1357err_unmap_auxc:
1358 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
1359
1360err_unmap_qp:
1361 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
1362
1363err_unmap_dmpt:
1364 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
1365
1366err_unmap_mtt:
1367 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
1368
1369err_unmap_eq:
fa0681d2 1370 mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
225c7b1f
RD		 1371
1372err_unmap_cmpt:
1373 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
1374 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1375 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1376 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
1377
1378err_unmap_aux:
1379 mlx4_UNMAP_ICM_AUX(dev);
1380
1381err_free_aux:
5b0bf5e2 1382 mlx4_free_icm(dev, priv->fw.aux_icm, 0);
225c7b1f
RD		 1383
1384 return err;
1385}
1386
1387static void mlx4_free_icms(struct mlx4_dev *dev)
1388{
1389 struct mlx4_priv *priv = mlx4_priv(dev);
1390
1391 mlx4_cleanup_icm_table(dev, &priv->mcg_table.table);
1392 mlx4_cleanup_icm_table(dev, &priv->srq_table.table);
1393 mlx4_cleanup_icm_table(dev, &priv->cq_table.table);
1394 mlx4_cleanup_icm_table(dev, &priv->qp_table.rdmarc_table);
1395 mlx4_cleanup_icm_table(dev, &priv->qp_table.altc_table);
1396 mlx4_cleanup_icm_table(dev, &priv->qp_table.auxc_table);
1397 mlx4_cleanup_icm_table(dev, &priv->qp_table.qp_table);
1398 mlx4_cleanup_icm_table(dev, &priv->mr_table.dmpt_table);
1399 mlx4_cleanup_icm_table(dev, &priv->mr_table.mtt_table);
fa0681d2 1400 mlx4_cleanup_icm_table(dev, &priv->eq_table.table);
225c7b1f
RD		 1401 mlx4_cleanup_icm_table(dev, &priv->eq_table.cmpt_table);
1402 mlx4_cleanup_icm_table(dev, &priv->cq_table.cmpt_table);
1403 mlx4_cleanup_icm_table(dev, &priv->srq_table.cmpt_table);
1404 mlx4_cleanup_icm_table(dev, &priv->qp_table.cmpt_table);
225c7b1f
RD		 1405
1406 mlx4_UNMAP_ICM_AUX(dev);
5b0bf5e2 1407 mlx4_free_icm(dev, priv->fw.aux_icm, 0);
225c7b1f
RD		 1408}
1409
ab9c17a0
JM		 1410static void mlx4_slave_exit(struct mlx4_dev *dev)
1411{
1412 struct mlx4_priv *priv = mlx4_priv(dev);
1413
f3d4c89e 1414 mutex_lock(&priv->cmd.slave_cmd_mutex);
ab9c17a0 1415 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, MLX4_COMM_TIME))
1a91de28 1416 mlx4_warn(dev, "Failed to close slave function\n");
f3d4c89e 1417 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0
JM		 1418}
1419
c1b43dca
EC		 1420static int map_bf_area(struct mlx4_dev *dev)
1421{
1422 struct mlx4_priv *priv = mlx4_priv(dev);
1423 resource_size_t bf_start;
1424 resource_size_t bf_len;
1425 int err = 0;
1426
3d747473
JM		 1427 if (!dev->caps.bf_reg_size)
1428 return -ENXIO;
1429
ab9c17a0
JM		 1430 bf_start = pci_resource_start(dev->pdev, 2) +
1431 (dev->caps.num_uars << PAGE_SHIFT);
1432 bf_len = pci_resource_len(dev->pdev, 2) -
1433 (dev->caps.num_uars << PAGE_SHIFT);
c1b43dca
EC		 1434 priv->bf_mapping = io_mapping_create_wc(bf_start, bf_len);
1435 if (!priv->bf_mapping)
1436 err = -ENOMEM;
1437
1438 return err;
1439}
1440
1441static void unmap_bf_area(struct mlx4_dev *dev)
1442{
1443 if (mlx4_priv(dev)->bf_mapping)
1444 io_mapping_free(mlx4_priv(dev)->bf_mapping);
1445}
1446
ec693d47
AV		 1447cycle_t mlx4_read_clock(struct mlx4_dev *dev)
1448{
1449 u32 clockhi, clocklo, clockhi1;
1450 cycle_t cycles;
1451 int i;
1452 struct mlx4_priv *priv = mlx4_priv(dev);
1453
1454 for (i = 0; i < 10; i++) {
1455 clockhi = swab32(readl(priv->clock_mapping));
1456 clocklo = swab32(readl(priv->clock_mapping + 4));
1457 clockhi1 = swab32(readl(priv->clock_mapping));
1458 if (clockhi == clockhi1)
1459 break;
1460 }
1461
1462 cycles = (u64) clockhi << 32 | (u64) clocklo;
1463
1464 return cycles;
1465}
1466EXPORT_SYMBOL_GPL(mlx4_read_clock);
1467
1468
ddd8a6c1
EE		 1469static int map_internal_clock(struct mlx4_dev *dev)
1470{
1471 struct mlx4_priv *priv = mlx4_priv(dev);
1472
1473 priv->clock_mapping =
1474 ioremap(pci_resource_start(dev->pdev, priv->fw.clock_bar) +
1475 priv->fw.clock_offset, MLX4_CLOCK_SIZE);
1476
1477 if (!priv->clock_mapping)
1478 return -ENOMEM;
1479
1480 return 0;
1481}
1482
1483static void unmap_internal_clock(struct mlx4_dev *dev)
1484{
1485 struct mlx4_priv *priv = mlx4_priv(dev);
1486
1487 if (priv->clock_mapping)
1488 iounmap(priv->clock_mapping);
1489}
1490
225c7b1f
RD		 1491static void mlx4_close_hca(struct mlx4_dev *dev)
1492{
ddd8a6c1 1493 unmap_internal_clock(dev);
c1b43dca 1494 unmap_bf_area(dev);
ab9c17a0
JM		 1495 if (mlx4_is_slave(dev))
1496 mlx4_slave_exit(dev);
1497 else {
1498 mlx4_CLOSE_HCA(dev, 0);
1499 mlx4_free_icms(dev);
a0eacca9
MB		 1500 }
1501}
1502
1503static void mlx4_close_fw(struct mlx4_dev *dev)
1504{
1505 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 1506 mlx4_UNMAP_FA(dev);
1507 mlx4_free_icm(dev, mlx4_priv(dev)->fw.fw_icm, 0);
1508 }
1509}
1510
1511static int mlx4_init_slave(struct mlx4_dev *dev)
1512{
1513 struct mlx4_priv *priv = mlx4_priv(dev);
1514 u64 dma = (u64) priv->mfunc.vhcr_dma;
ab9c17a0
JM		 1515 int ret_from_reset = 0;
1516 u32 slave_read;
1517 u32 cmd_channel_ver;
1518
97989356 1519 if (atomic_read(&pf_loading)) {
1a91de28 1520 mlx4_warn(dev, "PF is not ready - Deferring probe\n");
97989356
AV		 1521 return -EPROBE_DEFER;
1522 }
1523
f3d4c89e 1524 mutex_lock(&priv->cmd.slave_cmd_mutex);
ab9c17a0
JM		 1525 priv->cmd.max_cmds = 1;
1526 mlx4_warn(dev, "Sending reset\n");
1527 ret_from_reset = mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0,
1528 MLX4_COMM_TIME);
1529 /* if we are in the middle of flr the slave will try
1530 * NUM_OF_RESET_RETRIES times before leaving.*/
1531 if (ret_from_reset) {
1532 if (MLX4_DELAY_RESET_SLAVE == ret_from_reset) {
1a91de28 1533 mlx4_warn(dev, "slave is currently in the middle of FLR - Deferring probe\n");
5efe5355
JM		 1534 mutex_unlock(&priv->cmd.slave_cmd_mutex);
1535 return -EPROBE_DEFER;
ab9c17a0
JM		 1536 } else
1537 goto err;
1538 }
1539
1540 /* check the driver version - the slave I/F revision
1541 * must match the master's */
1542 slave_read = swab32(readl(&priv->mfunc.comm->slave_read));
1543 cmd_channel_ver = mlx4_comm_get_version();
1544
1545 if (MLX4_COMM_GET_IF_REV(cmd_channel_ver) !=
1546 MLX4_COMM_GET_IF_REV(slave_read)) {
1a91de28 1547 mlx4_err(dev, "slave driver version is not supported by the master\n");
ab9c17a0
JM		 1548 goto err;
1549 }
1550
1551 mlx4_warn(dev, "Sending vhcr0\n");
1552 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR0, dma >> 48,
1553 MLX4_COMM_TIME))
1554 goto err;
1555 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR1, dma >> 32,
1556 MLX4_COMM_TIME))
1557 goto err;
1558 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR2, dma >> 16,
1559 MLX4_COMM_TIME))
1560 goto err;
1561 if (mlx4_comm_cmd(dev, MLX4_COMM_CMD_VHCR_EN, dma, MLX4_COMM_TIME))
1562 goto err;
f3d4c89e
RD		 1563
1564 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0
JM		 1565 return 0;
1566
1567err:
1568 mlx4_comm_cmd(dev, MLX4_COMM_CMD_RESET, 0, 0);
f3d4c89e 1569 mutex_unlock(&priv->cmd.slave_cmd_mutex);
ab9c17a0 1570 return -EIO;
225c7b1f
RD		 1571}
1572
6634961c
JM		 1573static void mlx4_parav_master_pf_caps(struct mlx4_dev *dev)
1574{
1575 int i;
1576
1577 for (i = 1; i <= dev->caps.num_ports; i++) {
b6ffaeff
JM		 1578 if (dev->caps.port_type[i] == MLX4_PORT_TYPE_ETH)
1579 dev->caps.gid_table_len[i] =
449fc488 1580 mlx4_get_slave_num_gids(dev, 0, i);
b6ffaeff
JM		 1581 else
1582 dev->caps.gid_table_len[i] = 1;
6634961c
JM		 1583 dev->caps.pkey_table_len[i] =
1584 dev->phys_caps.pkey_phys_table_len[i] - 1;
1585 }
1586}
1587
3c439b55
JM		 1588static int choose_log_fs_mgm_entry_size(int qp_per_entry)
1589{
1590 int i = MLX4_MIN_MGM_LOG_ENTRY_SIZE;
1591
1592 for (i = MLX4_MIN_MGM_LOG_ENTRY_SIZE; i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE;
1593 i++) {
1594 if (qp_per_entry <= 4 * ((1 << i) / 16 - 2))
1595 break;
1596 }
1597
1598 return (i <= MLX4_MAX_MGM_LOG_ENTRY_SIZE) ? i : -1;
1599}
1600
7b8157be
JM		 1601static void choose_steering_mode(struct mlx4_dev *dev,
1602 struct mlx4_dev_cap *dev_cap)
1603{
3c439b55
JM		 1604 if (mlx4_log_num_mgm_entry_size == -1 &&
1605 dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_FS_EN &&
7b8157be 1606 (!mlx4_is_mfunc(dev) ||
449fc488 1607 (dev_cap->fs_max_num_qp_per_entry >= (dev->num_vfs + 1))) &&
3c439b55
JM		 1608 choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry) >=
1609 MLX4_MIN_MGM_LOG_ENTRY_SIZE) {
1610 dev->oper_log_mgm_entry_size =
1611 choose_log_fs_mgm_entry_size(dev_cap->fs_max_num_qp_per_entry);
7b8157be
JM		 1612 dev->caps.steering_mode = MLX4_STEERING_MODE_DEVICE_MANAGED;
1613 dev->caps.num_qp_per_mgm = dev_cap->fs_max_num_qp_per_entry;
1614 dev->caps.fs_log_max_ucast_qp_range_size =
1615 dev_cap->fs_log_max_ucast_qp_range_size;
1616 } else {
1617 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER &&
1618 dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
1619 dev->caps.steering_mode = MLX4_STEERING_MODE_B0;
1620 else {
1621 dev->caps.steering_mode = MLX4_STEERING_MODE_A0;
1622
1623 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_UC_STEER ||
1624 dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER)
1a91de28 1625 mlx4_warn(dev, "Must have both UC_STEER and MC_STEER flags set to use B0 steering - falling back to A0 steering mode\n");
7b8157be 1626 }
3c439b55
JM		 1627 dev->oper_log_mgm_entry_size =
1628 mlx4_log_num_mgm_entry_size > 0 ?
1629 mlx4_log_num_mgm_entry_size :
1630 MLX4_DEFAULT_MGM_LOG_ENTRY_SIZE;
7b8157be
JM		 1631 dev->caps.num_qp_per_mgm = mlx4_get_qp_per_mgm(dev);
1632 }
1a91de28 1633 mlx4_dbg(dev, "Steering mode is: %s, oper_log_mgm_entry_size = %d, modparam log_num_mgm_entry_size = %d\n",
3c439b55
JM		 1634 mlx4_steering_mode_str(dev->caps.steering_mode),
1635 dev->oper_log_mgm_entry_size,
1636 mlx4_log_num_mgm_entry_size);
7b8157be
JM		 1637}
1638
7ffdf726
OG		 1639static void choose_tunnel_offload_mode(struct mlx4_dev *dev,
1640 struct mlx4_dev_cap *dev_cap)
1641{
1642 if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED &&
1643 dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_VXLAN_OFFLOADS)
1644 dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_VXLAN;
1645 else
1646 dev->caps.tunnel_offload_mode = MLX4_TUNNEL_OFFLOAD_MODE_NONE;
1647
1648 mlx4_dbg(dev, "Tunneling offload mode is: %s\n", (dev->caps.tunnel_offload_mode
1649 == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) ? "vxlan" : "none");
1650}
1651
a0eacca9 1652static int mlx4_init_fw(struct mlx4_dev *dev)
225c7b1f 1653{
2d928651 1654 struct mlx4_mod_stat_cfg mlx4_cfg;
a0eacca9 1655 int err = 0;
225c7b1f 1656
ab9c17a0
JM		 1657 if (!mlx4_is_slave(dev)) {
1658 err = mlx4_QUERY_FW(dev);
1659 if (err) {
1660 if (err == -EACCES)
1a91de28 1661 mlx4_info(dev, "non-primary physical function, skipping\n");
ab9c17a0 1662 else
1a91de28 1663 mlx4_err(dev, "QUERY_FW command failed, aborting\n");
bef772eb 1664 return err;
ab9c17a0 1665 }
225c7b1f 1666
ab9c17a0
JM		 1667 err = mlx4_load_fw(dev);
1668 if (err) {
1a91de28 1669 mlx4_err(dev, "Failed to start FW, aborting\n");
bef772eb 1670 return err;
ab9c17a0 1671 }
225c7b1f 1672
ab9c17a0
JM		 1673 mlx4_cfg.log_pg_sz_m = 1;
1674 mlx4_cfg.log_pg_sz = 0;
1675 err = mlx4_MOD_STAT_CFG(dev, &mlx4_cfg);
1676 if (err)
1677 mlx4_warn(dev, "Failed to override log_pg_sz parameter\n");
a0eacca9 1678 }
2d928651 1679
a0eacca9
MB		 1680 return err;
1681}
1682
1683static int mlx4_init_hca(struct mlx4_dev *dev)
1684{
1685 struct mlx4_priv *priv = mlx4_priv(dev);
1686 struct mlx4_adapter adapter;
1687 struct mlx4_dev_cap dev_cap;
1688 struct mlx4_profile profile;
1689 struct mlx4_init_hca_param init_hca;
1690 u64 icm_size;
1691 struct mlx4_config_dev_params params;
1692 int err;
1693
1694 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 1695 err = mlx4_dev_cap(dev, &dev_cap);
1696 if (err) {
1a91de28 1697 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting\n");
ab9c17a0
JM		 1698 goto err_stop_fw;
1699 }
225c7b1f 1700
7b8157be 1701 choose_steering_mode(dev, &dev_cap);
7ffdf726 1702 choose_tunnel_offload_mode(dev, &dev_cap);
7b8157be 1703
8e1a28e8
HHZ		 1704 err = mlx4_get_phys_port_id(dev);
1705 if (err)
1706 mlx4_err(dev, "Fail to get physical port id\n");
1707
6634961c
JM		 1708 if (mlx4_is_master(dev))
1709 mlx4_parav_master_pf_caps(dev);
1710
2599d858
AV		 1711 if (mlx4_low_memory_profile()) {
1712 mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");
1713 profile = low_mem_profile;
1714 } else {
1715 profile = default_profile;
1716 }
0ff1fb65
HHZ		 1717 if (dev->caps.steering_mode ==
1718 MLX4_STEERING_MODE_DEVICE_MANAGED)
1719 profile.num_mcg = MLX4_FS_NUM_MCG;
225c7b1f 1720
ab9c17a0
JM		 1721 icm_size = mlx4_make_profile(dev, &profile, &dev_cap,
1722 &init_hca);
1723 if ((long long) icm_size < 0) {
1724 err = icm_size;
1725 goto err_stop_fw;
1726 }
225c7b1f 1727
a5bbe892
EC		 1728 dev->caps.max_fmr_maps = (1 << (32 - ilog2(dev->caps.num_mpts))) - 1;
1729
ab9c17a0
JM		 1730 init_hca.log_uar_sz = ilog2(dev->caps.num_uars);
1731 init_hca.uar_page_sz = PAGE_SHIFT - 12;
e448834e
SM		 1732 init_hca.mw_enabled = 0;
1733 if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
1734 dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN)
1735 init_hca.mw_enabled = INIT_HCA_TPT_MW_ENABLE;
c1b43dca 1736
ab9c17a0
JM		 1737 err = mlx4_init_icm(dev, &dev_cap, &init_hca, icm_size);
1738 if (err)
1739 goto err_stop_fw;
225c7b1f 1740
ab9c17a0
JM		 1741 err = mlx4_INIT_HCA(dev, &init_hca);
1742 if (err) {
1a91de28 1743 mlx4_err(dev, "INIT_HCA command failed, aborting\n");
ab9c17a0
JM		 1744 goto err_free_icm;
1745 }
7ae0e400
MB		 1746
1747 if (dev_cap.flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS) {
1748 err = mlx4_query_func(dev, &dev_cap);
1749 if (err < 0) {
1750 mlx4_err(dev, "QUERY_FUNC command failed, aborting.\n");
1751 goto err_stop_fw;
1752 } else if (err & MLX4_QUERY_FUNC_NUM_SYS_EQS) {
1753 dev->caps.num_eqs = dev_cap.max_eqs;
1754 dev->caps.reserved_eqs = dev_cap.reserved_eqs;
1755 dev->caps.reserved_uars = dev_cap.reserved_uars;
1756 }
1757 }
1758
ddd8a6c1
EE		 1759 /*
1760 * If TS is supported by FW
1761 * read HCA frequency by QUERY_HCA command
1762 */
1763 if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) {
1764 memset(&init_hca, 0, sizeof(init_hca));
1765 err = mlx4_QUERY_HCA(dev, &init_hca);
1766 if (err) {
1a91de28 1767 mlx4_err(dev, "QUERY_HCA command failed, disable timestamp\n");
ddd8a6c1
EE		 1768 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1769 } else {
1770 dev->caps.hca_core_clock =
1771 init_hca.hca_core_clock;
1772 }
1773
1774 /* In case we got HCA frequency 0 - disable timestamping
1775 * to avoid dividing by zero
1776 */
1777 if (!dev->caps.hca_core_clock) {
1778 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1779 mlx4_err(dev,
1a91de28 1780 "HCA frequency is 0 - timestamping is not supported\n");
ddd8a6c1
EE		 1781 } else if (map_internal_clock(dev)) {
1782 /*
1783 * Map internal clock,
1784 * in case of failure disable timestamping
1785 */
1786 dev->caps.flags2 &= ~MLX4_DEV_CAP_FLAG2_TS;
1a91de28 1787 mlx4_err(dev, "Failed to map internal clock. Timestamping is not supported\n");
ddd8a6c1
EE		 1788 }
1789 }
ab9c17a0
JM		 1790 } else {
1791 err = mlx4_init_slave(dev);
1792 if (err) {
5efe5355
JM		 1793 if (err != -EPROBE_DEFER)
1794 mlx4_err(dev, "Failed to initialize slave\n");
bef772eb 1795 return err;
ab9c17a0 1796 }
225c7b1f 1797
ab9c17a0
JM		 1798 err = mlx4_slave_cap(dev);
1799 if (err) {
1800 mlx4_err(dev, "Failed to obtain slave caps\n");
1801 goto err_close;
1802 }
225c7b1f
RD		 1803 }
1804
ab9c17a0
JM		 1805 if (map_bf_area(dev))
1806 mlx4_dbg(dev, "Failed to map blue flame area\n");
1807
1808 /*Only the master set the ports, all the rest got it from it.*/
1809 if (!mlx4_is_slave(dev))
1810 mlx4_set_port_mask(dev);
1811
225c7b1f
RD		 1812 err = mlx4_QUERY_ADAPTER(dev, &adapter);
1813 if (err) {
1a91de28 1814 mlx4_err(dev, "QUERY_ADAPTER command failed, aborting\n");
bef772eb 1815 goto unmap_bf;
225c7b1f
RD		 1816 }
1817
f8c6455b
SM		 1818 /* Query CONFIG_DEV parameters */
1819 err = mlx4_config_dev_retrieval(dev, &params);
1820 if (err && err != -ENOTSUPP) {
1821 mlx4_err(dev, "Failed to query CONFIG_DEV parameters\n");
1822 } else if (!err) {
1823 dev->caps.rx_checksum_flags_port[1] = params.rx_csum_flags_port_1;
1824 dev->caps.rx_checksum_flags_port[2] = params.rx_csum_flags_port_2;
1825 }
225c7b1f 1826 priv->eq_table.inta_pin = adapter.inta_pin;
cd9281d8 1827 memcpy(dev->board_id, adapter.board_id, sizeof dev->board_id);
225c7b1f
RD		 1828
1829 return 0;
1830
bef772eb 1831unmap_bf:
ddd8a6c1 1832 unmap_internal_clock(dev);
bef772eb
AY		 1833 unmap_bf_area(dev);
1834
b38f2879 1835 if (mlx4_is_slave(dev)) {
99ec41d0 1836 kfree(dev->caps.qp0_qkey);
b38f2879
DB		 1837 kfree(dev->caps.qp0_tunnel);
1838 kfree(dev->caps.qp0_proxy);
1839 kfree(dev->caps.qp1_tunnel);
1840 kfree(dev->caps.qp1_proxy);
1841 }
1842
225c7b1f 1843err_close:
41929ed2
DB		 1844 if (mlx4_is_slave(dev))
1845 mlx4_slave_exit(dev);
1846 else
1847 mlx4_CLOSE_HCA(dev, 0);
225c7b1f
RD		 1848
1849err_free_icm:
ab9c17a0
JM		 1850 if (!mlx4_is_slave(dev))
1851 mlx4_free_icms(dev);
225c7b1f
RD		 1852
1853err_stop_fw:
ab9c17a0
JM		 1854 if (!mlx4_is_slave(dev)) {
1855 mlx4_UNMAP_FA(dev);
1856 mlx4_free_icm(dev, priv->fw.fw_icm, 0);
1857 }
225c7b1f
RD		 1858 return err;
1859}
1860
f2a3f6a3
OG		 1861static int mlx4_init_counters_table(struct mlx4_dev *dev)
1862{
1863 struct mlx4_priv *priv = mlx4_priv(dev);
1864 int nent;
1865
1866 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
1867 return -ENOENT;
1868
1869 nent = dev->caps.max_counters;
1870 return mlx4_bitmap_init(&priv->counters_bitmap, nent, nent - 1, 0, 0);
1871}
1872
1873static void mlx4_cleanup_counters_table(struct mlx4_dev *dev)
1874{
1875 mlx4_bitmap_cleanup(&mlx4_priv(dev)->counters_bitmap);
1876}
1877
ba062d52 1878int __mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
f2a3f6a3
OG		 1879{
1880 struct mlx4_priv *priv = mlx4_priv(dev);
1881
1882 if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_COUNTERS))
1883 return -ENOENT;
1884
1885 *idx = mlx4_bitmap_alloc(&priv->counters_bitmap);
1886 if (*idx == -1)
1887 return -ENOMEM;
1888
1889 return 0;
1890}
ba062d52
JM		 1891
1892int mlx4_counter_alloc(struct mlx4_dev *dev, u32 *idx)
1893{
1894 u64 out_param;
1895 int err;
1896
1897 if (mlx4_is_mfunc(dev)) {
1898 err = mlx4_cmd_imm(dev, 0, &out_param, RES_COUNTER,
1899 RES_OP_RESERVE, MLX4_CMD_ALLOC_RES,
1900 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
1901 if (!err)
1902 *idx = get_param_l(&out_param);
1903
1904 return err;
1905 }
1906 return __mlx4_counter_alloc(dev, idx);
1907}
f2a3f6a3
OG		 1908EXPORT_SYMBOL_GPL(mlx4_counter_alloc);
1909
ba062d52 1910void __mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
f2a3f6a3 1911{
7c6d74d2 1912 mlx4_bitmap_free(&mlx4_priv(dev)->counters_bitmap, idx, MLX4_USE_RR);
f2a3f6a3
OG		 1913 return;
1914}
ba062d52
JM		 1915
1916void mlx4_counter_free(struct mlx4_dev *dev, u32 idx)
1917{
e7dbeba8 1918 u64 in_param = 0;
ba062d52
JM		 1919
1920 if (mlx4_is_mfunc(dev)) {
1921 set_param_l(&in_param, idx);
1922 mlx4_cmd(dev, in_param, RES_COUNTER, RES_OP_RESERVE,
1923 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A,
1924 MLX4_CMD_WRAPPED);
1925 return;
1926 }
1927 __mlx4_counter_free(dev, idx);
1928}
f2a3f6a3
OG		 1929EXPORT_SYMBOL_GPL(mlx4_counter_free);
1930
3d73c288 1931static int mlx4_setup_hca(struct mlx4_dev *dev)
225c7b1f
RD		 1932{
1933 struct mlx4_priv *priv = mlx4_priv(dev);
1934 int err;
7ff93f8b 1935 int port;
9a5aa622 1936 __be32 ib_port_default_caps;
225c7b1f 1937
225c7b1f
RD		 1938 err = mlx4_init_uar_table(dev);
1939 if (err) {
1a91de28
JP		 1940 mlx4_err(dev, "Failed to initialize user access region table, aborting\n");
1941 return err;
225c7b1f
RD		 1942 }
1943
1944 err = mlx4_uar_alloc(dev, &priv->driver_uar);
1945 if (err) {
1a91de28 1946 mlx4_err(dev, "Failed to allocate driver access region, aborting\n");
225c7b1f
RD		 1947 goto err_uar_table_free;
1948 }
1949
4979d18f 1950 priv->kar = ioremap((phys_addr_t) priv->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
225c7b1f 1951 if (!priv->kar) {
1a91de28 1952 mlx4_err(dev, "Couldn't map kernel access region, aborting\n");
225c7b1f
RD		 1953 err = -ENOMEM;
1954 goto err_uar_free;
1955 }
1956
1957 err = mlx4_init_pd_table(dev);
1958 if (err) {
1a91de28 1959 mlx4_err(dev, "Failed to initialize protection domain table, aborting\n");
225c7b1f
RD		 1960 goto err_kar_unmap;
1961 }
1962
012a8ff5
SH		 1963 err = mlx4_init_xrcd_table(dev);
1964 if (err) {
1a91de28 1965 mlx4_err(dev, "Failed to initialize reliable connection domain table, aborting\n");
012a8ff5
SH		 1966 goto err_pd_table_free;
1967 }
1968
225c7b1f
RD		 1969 err = mlx4_init_mr_table(dev);
1970 if (err) {
1a91de28 1971 mlx4_err(dev, "Failed to initialize memory region table, aborting\n");
012a8ff5 1972 goto err_xrcd_table_free;
225c7b1f
RD		 1973 }
1974
fe6f700d
YP		 1975 if (!mlx4_is_slave(dev)) {
1976 err = mlx4_init_mcg_table(dev);
1977 if (err) {
1a91de28 1978 mlx4_err(dev, "Failed to initialize multicast group table, aborting\n");
fe6f700d
YP		 1979 goto err_mr_table_free;
1980 }
114840c3
JM		 1981 err = mlx4_config_mad_demux(dev);
1982 if (err) {
1983 mlx4_err(dev, "Failed in config_mad_demux, aborting\n");
1984 goto err_mcg_table_free;
1985 }
fe6f700d
YP		 1986 }
1987
225c7b1f
RD		 1988 err = mlx4_init_eq_table(dev);
1989 if (err) {
1a91de28 1990 mlx4_err(dev, "Failed to initialize event queue table, aborting\n");
fe6f700d 1991 goto err_mcg_table_free;
225c7b1f
RD		 1992 }
1993
1994 err = mlx4_cmd_use_events(dev);
1995 if (err) {
1a91de28 1996 mlx4_err(dev, "Failed to switch to event-driven firmware commands, aborting\n");
225c7b1f
RD		 1997 goto err_eq_table_free;
1998 }
1999
2000 err = mlx4_NOP(dev);
2001 if (err) {
08fb1055 2002 if (dev->flags & MLX4_FLAG_MSI_X) {
1a91de28 2003 mlx4_warn(dev, "NOP command failed to generate MSI-X interrupt IRQ %d)\n",
b8dd786f 2004 priv->eq_table.eq[dev->caps.num_comp_vectors].irq);
1a91de28 2005 mlx4_warn(dev, "Trying again without MSI-X\n");
08fb1055 2006 } else {
1a91de28 2007 mlx4_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting\n",
b8dd786f 2008 priv->eq_table.eq[dev->caps.num_comp_vectors].irq);
225c7b1f 2009 mlx4_err(dev, "BIOS or ACPI interrupt routing problem?\n");
08fb1055 2010 }
225c7b1f
RD		 2011
2012 goto err_cmd_poll;
2013 }
2014
2015 mlx4_dbg(dev, "NOP command IRQ test passed\n");
2016
2017 err = mlx4_init_cq_table(dev);
2018 if (err) {
1a91de28 2019 mlx4_err(dev, "Failed to initialize completion queue table, aborting\n");
225c7b1f
RD		 2020 goto err_cmd_poll;
2021 }
2022
2023 err = mlx4_init_srq_table(dev);
2024 if (err) {
1a91de28 2025 mlx4_err(dev, "Failed to initialize shared receive queue table, aborting\n");
225c7b1f
RD		 2026 goto err_cq_table_free;
2027 }
2028
2029 err = mlx4_init_qp_table(dev);
2030 if (err) {
1a91de28 2031 mlx4_err(dev, "Failed to initialize queue pair table, aborting\n");
225c7b1f
RD		 2032 goto err_srq_table_free;
2033 }
2034
f2a3f6a3
OG		 2035 err = mlx4_init_counters_table(dev);
2036 if (err && err != -ENOENT) {
1a91de28 2037 mlx4_err(dev, "Failed to initialize counters table, aborting\n");
fe6f700d 2038 goto err_qp_table_free;
f2a3f6a3
OG		 2039 }
2040
ab9c17a0
JM		 2041 if (!mlx4_is_slave(dev)) {
2042 for (port = 1; port <= dev->caps.num_ports; port++) {
ab9c17a0
JM		 2043 ib_port_default_caps = 0;
2044 err = mlx4_get_port_ib_caps(dev, port,
2045 &ib_port_default_caps);
2046 if (err)
1a91de28
JP		 2047 mlx4_warn(dev, "failed to get port %d default ib capabilities (%d). Continuing with caps = 0\n",
2048 port, err);
ab9c17a0
JM		 2049 dev->caps.ib_port_def_cap[port] = ib_port_default_caps;
2050
2aca1172
JM		 2051 /* initialize per-slave default ib port capabilities */
2052 if (mlx4_is_master(dev)) {
2053 int i;
2054 for (i = 0; i < dev->num_slaves; i++) {
2055 if (i == mlx4_master_func_num(dev))
2056 continue;
2057 priv->mfunc.master.slave_state[i].ib_cap_mask[port] =
1a91de28 2058 ib_port_default_caps;
2aca1172
JM		 2059 }
2060 }
2061
096335b3
OG		 2062 if (mlx4_is_mfunc(dev))
2063 dev->caps.port_ib_mtu[port] = IB_MTU_2048;
2064 else
2065 dev->caps.port_ib_mtu[port] = IB_MTU_4096;
97285b78 2066
6634961c
JM		 2067 err = mlx4_SET_PORT(dev, port, mlx4_is_master(dev) ?
2068 dev->caps.pkey_table_len[port] : -1);
ab9c17a0
JM		 2069 if (err) {
2070 mlx4_err(dev, "Failed to set port %d, aborting\n",
1a91de28 2071 port);
ab9c17a0
JM		 2072 goto err_counters_table_free;
2073 }
7ff93f8b
YP		 2074 }
2075 }
2076
225c7b1f
RD		 2077 return 0;
2078
f2a3f6a3
OG		 2079err_counters_table_free:
2080 mlx4_cleanup_counters_table(dev);
2081
225c7b1f
RD		 2082err_qp_table_free:
2083 mlx4_cleanup_qp_table(dev);
2084
2085err_srq_table_free:
2086 mlx4_cleanup_srq_table(dev);
2087
2088err_cq_table_free:
2089 mlx4_cleanup_cq_table(dev);
2090
2091err_cmd_poll:
2092 mlx4_cmd_use_polling(dev);
2093
2094err_eq_table_free:
2095 mlx4_cleanup_eq_table(dev);
2096
fe6f700d
YP		 2097err_mcg_table_free:
2098 if (!mlx4_is_slave(dev))
2099 mlx4_cleanup_mcg_table(dev);
2100
ee49bd93 2101err_mr_table_free:
225c7b1f
RD		 2102 mlx4_cleanup_mr_table(dev);
2103
012a8ff5
SH		 2104err_xrcd_table_free:
2105 mlx4_cleanup_xrcd_table(dev);
2106
225c7b1f
RD		 2107err_pd_table_free:
2108 mlx4_cleanup_pd_table(dev);
2109
2110err_kar_unmap:
2111 iounmap(priv->kar);
2112
2113err_uar_free:
2114 mlx4_uar_free(dev, &priv->driver_uar);
2115
2116err_uar_table_free:
2117 mlx4_cleanup_uar_table(dev);
2118 return err;
2119}
2120
e8f9b2ed 2121static void mlx4_enable_msi_x(struct mlx4_dev *dev)
225c7b1f
RD		 2122{
2123 struct mlx4_priv *priv = mlx4_priv(dev);
b8dd786f 2124 struct msix_entry *entries;
225c7b1f
RD		 2125 int i;
2126
2127 if (msi_x) {
7ae0e400
MB		 2128 int nreq = dev->caps.num_ports * num_online_cpus() + MSIX_LEGACY_SZ;
2129
ca4c7b35
OG		 2130 nreq = min_t(int, dev->caps.num_eqs - dev->caps.reserved_eqs,
2131 nreq);
ab9c17a0 2132
b8dd786f
YP		 2133 entries = kcalloc(nreq, sizeof *entries, GFP_KERNEL);
2134 if (!entries)
2135 goto no_msi;
2136
2137 for (i = 0; i < nreq; ++i)
225c7b1f
RD		 2138 entries[i].entry = i;
2139
66e2f9c1
AG		 2140 nreq = pci_enable_msix_range(dev->pdev, entries, 2, nreq);
2141
2142 if (nreq < 0) {
5bf0da7d 2143 kfree(entries);
225c7b1f 2144 goto no_msi;
66e2f9c1 2145 } else if (nreq < MSIX_LEGACY_SZ +
1a91de28 2146 dev->caps.num_ports * MIN_MSIX_P_PORT) {
0b7ca5a9
YP		 2147 /*Working in legacy mode , all EQ's shared*/
2148 dev->caps.comp_pool = 0;
2149 dev->caps.num_comp_vectors = nreq - 1;
2150 } else {
2151 dev->caps.comp_pool = nreq - MSIX_LEGACY_SZ;
2152 dev->caps.num_comp_vectors = MSIX_LEGACY_SZ - 1;
2153 }
b8dd786f 2154 for (i = 0; i < nreq; ++i)
225c7b1f
RD		 2155 priv->eq_table.eq[i].irq = entries[i].vector;
2156
2157 dev->flags |= MLX4_FLAG_MSI_X;
b8dd786f
YP		 2158
2159 kfree(entries);
225c7b1f
RD		 2160 return;
2161 }
2162
2163no_msi:
b8dd786f 2164 dev->caps.num_comp_vectors = 1;
0b7ca5a9 2165 dev->caps.comp_pool = 0;
b8dd786f
YP		 2166
2167 for (i = 0; i < 2; ++i)
225c7b1f
RD		 2168 priv->eq_table.eq[i].irq = dev->pdev->irq;
2169}
2170
7ff93f8b 2171static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
2a2336f8
YP		 2172{
2173 struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
7ff93f8b 2174 int err = 0;
2a2336f8
YP		 2175
2176 info->dev = dev;
2177 info->port = port;
ab9c17a0 2178 if (!mlx4_is_slave(dev)) {
ab9c17a0
JM		 2179 mlx4_init_mac_table(dev, &info->mac_table);
2180 mlx4_init_vlan_table(dev, &info->vlan_table);
111c6094 2181 mlx4_init_roce_gid_table(dev, &info->gid_table);
16a10ffd 2182 info->base_qpn = mlx4_get_base_qpn(dev, port);
ab9c17a0 2183 }
7ff93f8b
YP		 2184
2185 sprintf(info->dev_name, "mlx4_port%d", port);
2186 info->port_attr.attr.name = info->dev_name;
ab9c17a0
JM		 2187 if (mlx4_is_mfunc(dev))
2188 info->port_attr.attr.mode = S_IRUGO;
2189 else {
2190 info->port_attr.attr.mode = S_IRUGO | S_IWUSR;
2191 info->port_attr.store = set_port_type;
2192 }
7ff93f8b 2193 info->port_attr.show = show_port_type;
3691c964 2194 sysfs_attr_init(&info->port_attr.attr);
7ff93f8b
YP		 2195
2196 err = device_create_file(&dev->pdev->dev, &info->port_attr);
2197 if (err) {
2198 mlx4_err(dev, "Failed to create file for port %d\n", port);
2199 info->port = -1;
2200 }
2201
096335b3
OG		 2202 sprintf(info->dev_mtu_name, "mlx4_port%d_mtu", port);
2203 info->port_mtu_attr.attr.name = info->dev_mtu_name;
2204 if (mlx4_is_mfunc(dev))
2205 info->port_mtu_attr.attr.mode = S_IRUGO;
2206 else {
2207 info->port_mtu_attr.attr.mode = S_IRUGO | S_IWUSR;
2208 info->port_mtu_attr.store = set_port_ib_mtu;
2209 }
2210 info->port_mtu_attr.show = show_port_ib_mtu;
2211 sysfs_attr_init(&info->port_mtu_attr.attr);
2212
2213 err = device_create_file(&dev->pdev->dev, &info->port_mtu_attr);
2214 if (err) {
2215 mlx4_err(dev, "Failed to create mtu file for port %d\n", port);
2216 device_remove_file(&info->dev->pdev->dev, &info->port_attr);
2217 info->port = -1;
2218 }
2219
7ff93f8b
YP		 2220 return err;
2221}
2222
2223static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
2224{
2225 if (info->port < 0)
2226 return;
2227
2228 device_remove_file(&info->dev->pdev->dev, &info->port_attr);
096335b3 2229 device_remove_file(&info->dev->pdev->dev, &info->port_mtu_attr);
2a2336f8
YP		 2230}
2231
b12d93d6
YP		 2232static int mlx4_init_steering(struct mlx4_dev *dev)
2233{
2234 struct mlx4_priv *priv = mlx4_priv(dev);
2235 int num_entries = dev->caps.num_ports;
2236 int i, j;
2237
2238 priv->steer = kzalloc(sizeof(struct mlx4_steer) * num_entries, GFP_KERNEL);
2239 if (!priv->steer)
2240 return -ENOMEM;
2241
45b51365 2242 for (i = 0; i < num_entries; i++)
b12d93d6
YP		 2243 for (j = 0; j < MLX4_NUM_STEERS; j++) {
2244 INIT_LIST_HEAD(&priv->steer[i].promisc_qps[j]);
2245 INIT_LIST_HEAD(&priv->steer[i].steer_entries[j]);
2246 }
b12d93d6
YP		 2247 return 0;
2248}
2249
2250static void mlx4_clear_steering(struct mlx4_dev *dev)
2251{
2252 struct mlx4_priv *priv = mlx4_priv(dev);
2253 struct mlx4_steer_index *entry, *tmp_entry;
2254 struct mlx4_promisc_qp *pqp, *tmp_pqp;
2255 int num_entries = dev->caps.num_ports;
2256 int i, j;
2257
2258 for (i = 0; i < num_entries; i++) {
2259 for (j = 0; j < MLX4_NUM_STEERS; j++) {
2260 list_for_each_entry_safe(pqp, tmp_pqp,
2261 &priv->steer[i].promisc_qps[j],
2262 list) {
2263 list_del(&pqp->list);
2264 kfree(pqp);
2265 }
2266 list_for_each_entry_safe(entry, tmp_entry,
2267 &priv->steer[i].steer_entries[j],
2268 list) {
2269 list_del(&entry->list);
2270 list_for_each_entry_safe(pqp, tmp_pqp,
2271 &entry->duplicates,
2272 list) {
2273 list_del(&pqp->list);
2274 kfree(pqp);
2275 }
2276 kfree(entry);
2277 }
2278 }
2279 }
2280 kfree(priv->steer);
2281}
2282
ab9c17a0
JM		 2283static int extended_func_num(struct pci_dev *pdev)
2284{
2285 return PCI_SLOT(pdev->devfn) * 8 + PCI_FUNC(pdev->devfn);
2286}
2287
2288#define MLX4_OWNER_BASE 0x8069c
2289#define MLX4_OWNER_SIZE 4
2290
2291static int mlx4_get_ownership(struct mlx4_dev *dev)
2292{
2293 void __iomem *owner;
2294 u32 ret;
2295
57dbf29a
KSS		 2296 if (pci_channel_offline(dev->pdev))
2297 return -EIO;
2298
ab9c17a0
JM		 2299 owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE,
2300 MLX4_OWNER_SIZE);
2301 if (!owner) {
2302 mlx4_err(dev, "Failed to obtain ownership bit\n");
2303 return -ENOMEM;
2304 }
2305
2306 ret = readl(owner);
2307 iounmap(owner);
2308 return (int) !!ret;
2309}
2310
2311static void mlx4_free_ownership(struct mlx4_dev *dev)
2312{
2313 void __iomem *owner;
2314
57dbf29a
KSS		 2315 if (pci_channel_offline(dev->pdev))
2316 return;
2317
ab9c17a0
JM		 2318 owner = ioremap(pci_resource_start(dev->pdev, 0) + MLX4_OWNER_BASE,
2319 MLX4_OWNER_SIZE);
2320 if (!owner) {
2321 mlx4_err(dev, "Failed to obtain ownership bit\n");
2322 return;
2323 }
2324 writel(0, owner);
2325 msleep(1000);
2326 iounmap(owner);
2327}
2328
a0eacca9
MB		 2329#define SRIOV_VALID_STATE(flags) (!!((flags) & MLX4_FLAG_SRIOV) ==\
2330 !!((flags) & MLX4_FLAG_MASTER))
2331
2332static u64 mlx4_enable_sriov(struct mlx4_dev *dev, struct pci_dev *pdev,
2333 u8 total_vfs, int existing_vfs)
2334{
2335 u64 dev_flags = dev->flags;
2336
2337 dev->dev_vfs = kzalloc(
2338 total_vfs * sizeof(*dev->dev_vfs),
2339 GFP_KERNEL);
2340 if (NULL == dev->dev_vfs) {
2341 mlx4_err(dev, "Failed to allocate memory for VFs\n");
2342 goto disable_sriov;
2343 } else if (!(dev->flags & MLX4_FLAG_SRIOV)) {
2344 int err = 0;
2345
2346 atomic_inc(&pf_loading);
2347 if (existing_vfs) {
2348 if (existing_vfs != total_vfs)
2349 mlx4_err(dev, "SR-IOV was already enabled, but with num_vfs (%d) different than requested (%d)\n",
2350 existing_vfs, total_vfs);
2351 } else {
2352 mlx4_warn(dev, "Enabling SR-IOV with %d VFs\n", total_vfs);
2353 err = pci_enable_sriov(pdev, total_vfs);
2354 }
2355 if (err) {
2356 mlx4_err(dev, "Failed to enable SR-IOV, continuing without SR-IOV (err = %d)\n",
2357 err);
2358 atomic_dec(&pf_loading);
2359 goto disable_sriov;
2360 } else {
2361 mlx4_warn(dev, "Running in master mode\n");
2362 dev_flags |= MLX4_FLAG_SRIOV |
2363 MLX4_FLAG_MASTER;
2364 dev_flags &= ~MLX4_FLAG_SLAVE;
2365 dev->num_vfs = total_vfs;
2366 }
2367 }
2368 return dev_flags;
2369
2370disable_sriov:
2371 dev->num_vfs = 0;
2372 kfree(dev->dev_vfs);
2373 return dev_flags & ~MLX4_FLAG_MASTER;
2374}
2375
de966c59
MB		 2376enum {
2377 MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64 = -1,
2378};
2379
2380static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap,
2381 int *nvfs)
2382{
2383 int requested_vfs = nvfs[0] + nvfs[1] + nvfs[2];
2384 /* Checking for 64 VFs as a limitation of CX2 */
2385 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_80_VFS) &&
2386 requested_vfs >= 64) {
2387 mlx4_err(dev, "Requested %d VFs, but FW does not support more than 64\n",
2388 requested_vfs);
2389 return MLX4_DEV_CAP_CHECK_NUM_VFS_ABOVE_64;
2390 }
2391 return 0;
2392}
2393
e1c00e10
MD		 2394static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data,
2395 int total_vfs, int *nvfs, struct mlx4_priv *priv)
225c7b1f 2396{
225c7b1f 2397 struct mlx4_dev *dev;
e1c00e10 2398 unsigned sum = 0;
225c7b1f 2399 int err;
2a2336f8 2400 int port;
e1c00e10 2401 int i;
7ae0e400 2402 struct mlx4_dev_cap *dev_cap = NULL;
bbb07af4 2403 int existing_vfs = 0;
225c7b1f 2404
e1c00e10 2405 dev = &priv->dev;
225c7b1f 2406
b581401e
RD		 2407 INIT_LIST_HEAD(&priv->ctx_list);
2408 spin_lock_init(&priv->ctx_lock);
225c7b1f 2409
7ff93f8b
YP		 2410 mutex_init(&priv->port_mutex);
2411
6296883c
YP		 2412 INIT_LIST_HEAD(&priv->pgdir_list);
2413 mutex_init(&priv->pgdir_mutex);
2414
c1b43dca
EC		 2415 INIT_LIST_HEAD(&priv->bf_list);
2416 mutex_init(&priv->bf_mutex);
2417
aca7a3ac 2418 dev->rev_id = pdev->revision;
6e7136ed 2419 dev->numa_node = dev_to_node(&pdev->dev);
e1c00e10 2420
ab9c17a0 2421 /* Detect if this device is a virtual function */
839f1243 2422 if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
ab9c17a0
JM		 2423 mlx4_warn(dev, "Detected virtual function - running in slave mode\n");
2424 dev->flags |= MLX4_FLAG_SLAVE;
2425 } else {
2426 /* We reset the device and enable SRIOV only for physical
2427 * devices. Try to claim ownership on the device;
2428 * if already taken, skip -- do not allow multiple PFs */
2429 err = mlx4_get_ownership(dev);
2430 if (err) {
2431 if (err < 0)
e1c00e10 2432 return err;
ab9c17a0 2433 else {
1a91de28 2434 mlx4_warn(dev, "Multiple PFs not yet supported - Skipping PF\n");
e1c00e10 2435 return -EINVAL;
ab9c17a0
JM		 2436 }
2437 }
aca7a3ac 2438
fe6f700d
YP		 2439 atomic_set(&priv->opreq_count, 0);
2440 INIT_WORK(&priv->opreq_task, mlx4_opreq_action);
2441
ab9c17a0
JM		 2442 /*
2443 * Now reset the HCA before we touch the PCI capabilities or
2444 * attempt a firmware command, since a boot ROM may have left
2445 * the HCA in an undefined state.
2446 */
2447 err = mlx4_reset(dev);
2448 if (err) {
1a91de28 2449 mlx4_err(dev, "Failed to reset HCA, aborting\n");
e1c00e10 2450 goto err_sriov;
ab9c17a0 2451 }
7ae0e400
MB		 2452
2453 if (total_vfs) {
2454 existing_vfs = pci_num_vf(pdev);
2455 dev->flags = MLX4_FLAG_MASTER;
2456 dev->num_vfs = total_vfs;
2457 }
225c7b1f
RD		 2458 }
2459
ab9c17a0 2460slave_start:
521130d1
EE		 2461 err = mlx4_cmd_init(dev);
2462 if (err) {
1a91de28 2463 mlx4_err(dev, "Failed to init command interface, aborting\n");
ab9c17a0
JM		 2464 goto err_sriov;
2465 }
2466
2467 /* In slave functions, the communication channel must be initialized
2468 * before posting commands. Also, init num_slaves before calling
2469 * mlx4_init_hca */
2470 if (mlx4_is_mfunc(dev)) {
7ae0e400 2471 if (mlx4_is_master(dev)) {
ab9c17a0 2472 dev->num_slaves = MLX4_MAX_NUM_SLAVES;
7ae0e400
MB		 2473
2474 } else {
ab9c17a0 2475 dev->num_slaves = 0;
f356fcbe
JM		 2476 err = mlx4_multi_func_init(dev);
2477 if (err) {
1a91de28 2478 mlx4_err(dev, "Failed to init slave mfunc interface, aborting\n");
ab9c17a0
JM		 2479 goto err_cmd;
2480 }
2481 }
225c7b1f
RD		 2482 }
2483
a0eacca9
MB		 2484 err = mlx4_init_fw(dev);
2485 if (err) {
2486 mlx4_err(dev, "Failed to init fw, aborting.\n");
2487 goto err_mfunc;
2488 }
2489
7ae0e400
MB		 2490 if (mlx4_is_master(dev)) {
2491 if (!dev_cap) {
2492 dev_cap = kzalloc(sizeof(*dev_cap), GFP_KERNEL);
2493
2494 if (!dev_cap) {
2495 err = -ENOMEM;
2496 goto err_fw;
2497 }
2498
2499 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
2500 if (err) {
2501 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
2502 goto err_fw;
2503 }
2504
de966c59
MB		 2505 if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
2506 goto err_fw;
2507
7ae0e400
MB		 2508 if (!(dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
2509 u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs,
2510 existing_vfs);
2511
2512 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
2513 dev->flags = dev_flags;
2514 if (!SRIOV_VALID_STATE(dev->flags)) {
2515 mlx4_err(dev, "Invalid SRIOV state\n");
2516 goto err_sriov;
2517 }
2518 err = mlx4_reset(dev);
2519 if (err) {
2520 mlx4_err(dev, "Failed to reset HCA, aborting.\n");
2521 goto err_sriov;
2522 }
2523 goto slave_start;
2524 }
2525 } else {
2526 /* Legacy mode FW requires SRIOV to be enabled before
2527 * doing QUERY_DEV_CAP, since max_eq's value is different if
2528 * SRIOV is enabled.
2529 */
2530 memset(dev_cap, 0, sizeof(*dev_cap));
2531 err = mlx4_QUERY_DEV_CAP(dev, dev_cap);
2532 if (err) {
2533 mlx4_err(dev, "QUERY_DEV_CAP command failed, aborting.\n");
2534 goto err_fw;
2535 }
de966c59
MB		 2536
2537 if (mlx4_check_dev_cap(dev, dev_cap, nvfs))
2538 goto err_fw;
7ae0e400
MB		 2539 }
2540 }
2541
225c7b1f 2542 err = mlx4_init_hca(dev);
ab9c17a0
JM		 2543 if (err) {
2544 if (err == -EACCES) {
2545 /* Not primary Physical function
2546 * Running in slave mode */
ffc39f6d 2547 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
a0eacca9
MB		 2548 /* We're not a PF */
2549 if (dev->flags & MLX4_FLAG_SRIOV) {
2550 if (!existing_vfs)
2551 pci_disable_sriov(pdev);
2552 if (mlx4_is_master(dev))
2553 atomic_dec(&pf_loading);
2554 dev->flags &= ~MLX4_FLAG_SRIOV;
2555 }
2556 if (!mlx4_is_slave(dev))
2557 mlx4_free_ownership(dev);
ab9c17a0
JM		 2558 dev->flags |= MLX4_FLAG_SLAVE;
2559 dev->flags &= ~MLX4_FLAG_MASTER;
2560 goto slave_start;
2561 } else
a0eacca9 2562 goto err_fw;
ab9c17a0
JM		 2563 }
2564
7ae0e400
MB		 2565 if (mlx4_is_master(dev) && (dev_cap->flags2 & MLX4_DEV_CAP_FLAG2_SYS_EQS)) {
2566 u64 dev_flags = mlx4_enable_sriov(dev, pdev, total_vfs, existing_vfs);
2567
2568 if ((dev->flags ^ dev_flags) & (MLX4_FLAG_MASTER | MLX4_FLAG_SLAVE)) {
2569 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_VHCR);
2570 dev->flags = dev_flags;
2571 err = mlx4_cmd_init(dev);
2572 if (err) {
2573 /* Only VHCR is cleaned up, so could still
2574 * send FW commands
2575 */
2576 mlx4_err(dev, "Failed to init VHCR command interface, aborting\n");
2577 goto err_close;
2578 }
2579 } else {
2580 dev->flags = dev_flags;
2581 }
2582
2583 if (!SRIOV_VALID_STATE(dev->flags)) {
2584 mlx4_err(dev, "Invalid SRIOV state\n");
2585 goto err_close;
2586 }
2587 }
2588
b912b2f8
EP		 2589 /* check if the device is functioning at its maximum possible speed.
2590 * No return code for this call, just warn the user in case of PCI
2591 * express device capabilities are under-satisfied by the bus.
2592 */
83d3459a
EP		 2593 if (!mlx4_is_slave(dev))
2594 mlx4_check_pcie_caps(dev);
b912b2f8 2595
ab9c17a0
JM		 2596 /* In master functions, the communication channel must be initialized
2597 * after obtaining its address from fw */
2598 if (mlx4_is_master(dev)) {
e1c00e10
MD		 2599 int ib_ports = 0;
2600
2601 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
2602 ib_ports++;
2603
2604 if (ib_ports &&
2605 (num_vfs_argc > 1 || probe_vfs_argc > 1)) {
2606 mlx4_err(dev,
2607 "Invalid syntax of num_vfs/probe_vfs with IB port - single port VFs syntax is only supported when all ports are configured as ethernet\n");
2608 err = -EINVAL;
2609 goto err_close;
2610 }
2611 if (dev->caps.num_ports < 2 &&
2612 num_vfs_argc > 1) {
2613 err = -EINVAL;
2614 mlx4_err(dev,
2615 "Error: Trying to configure VFs on port 2, but HCA has only %d physical ports\n",
2616 dev->caps.num_ports);
ab9c17a0
JM		 2617 goto err_close;
2618 }
e1c00e10 2619 memcpy(dev->nvfs, nvfs, sizeof(dev->nvfs));
dd41cc3b 2620
e1c00e10
MD		 2621 for (i = 0; i < sizeof(dev->nvfs)/sizeof(dev->nvfs[0]); i++) {
2622 unsigned j;
2623
2624 for (j = 0; j < dev->nvfs[i]; ++sum, ++j) {
2625 dev->dev_vfs[sum].min_port = i < 2 ? i + 1 : 1;
2626 dev->dev_vfs[sum].n_ports = i < 2 ? 1 :
2627 dev->caps.num_ports;
1ab95d37
MB		 2628 }
2629 }
e1c00e10
MD		 2630
2631 /* In master functions, the communication channel
2632 * must be initialized after obtaining its address from fw
2633 */
2634 err = mlx4_multi_func_init(dev);
2635 if (err) {
2636 mlx4_err(dev, "Failed to init master mfunc interface, aborting.\n");
2637 goto err_close;
2638 }
ab9c17a0 2639 }
225c7b1f 2640
b8dd786f
YP		 2641 err = mlx4_alloc_eq_table(dev);
2642 if (err)
ab9c17a0 2643 goto err_master_mfunc;
b8dd786f 2644
0b7ca5a9 2645 priv->msix_ctl.pool_bm = 0;
730c41d5 2646 mutex_init(&priv->msix_ctl.pool_lock);
0b7ca5a9 2647
08fb1055 2648 mlx4_enable_msi_x(dev);
ab9c17a0
JM		 2649 if ((mlx4_is_mfunc(dev)) &&
2650 !(dev->flags & MLX4_FLAG_MSI_X)) {
f356fcbe 2651 err = -ENOSYS;
1a91de28 2652 mlx4_err(dev, "INTx is not supported in multi-function mode, aborting\n");
b12d93d6 2653 goto err_free_eq;
ab9c17a0
JM		 2654 }
2655
2656 if (!mlx4_is_slave(dev)) {
2657 err = mlx4_init_steering(dev);
2658 if (err)
e1c00e10 2659 goto err_disable_msix;
ab9c17a0 2660 }
b12d93d6 2661
225c7b1f 2662 err = mlx4_setup_hca(dev);
ab9c17a0
JM		 2663 if (err == -EBUSY && (dev->flags & MLX4_FLAG_MSI_X) &&
2664 !mlx4_is_mfunc(dev)) {
08fb1055 2665 dev->flags &= ~MLX4_FLAG_MSI_X;
9858d2d1
YP		 2666 dev->caps.num_comp_vectors = 1;
2667 dev->caps.comp_pool = 0;
08fb1055
MT		 2668 pci_disable_msix(pdev);
2669 err = mlx4_setup_hca(dev);
2670 }
2671
225c7b1f 2672 if (err)
b12d93d6 2673 goto err_steer;
225c7b1f 2674
5a0d0a61
JM		 2675 mlx4_init_quotas(dev);
2676
7ff93f8b
YP		 2677 for (port = 1; port <= dev->caps.num_ports; port++) {
2678 err = mlx4_init_port_info(dev, port);
2679 if (err)
2680 goto err_port;
2681 }
2a2336f8 2682
225c7b1f
RD		 2683 err = mlx4_register_device(dev);
2684 if (err)
7ff93f8b 2685 goto err_port;
225c7b1f 2686
b046ffe5
EP		 2687 mlx4_request_modules(dev);
2688
27bf91d6
YP		 2689 mlx4_sense_init(dev);
2690 mlx4_start_sense(dev);
2691
befdf897 2692 priv->removed = 0;
225c7b1f 2693
e1a5ddc5
AV		 2694 if (mlx4_is_master(dev) && dev->num_vfs)
2695 atomic_dec(&pf_loading);
2696
225c7b1f
RD		 2697 return 0;
2698
7ff93f8b 2699err_port:
b4f77264 2700 for (--port; port >= 1; --port)
7ff93f8b
YP		 2701 mlx4_cleanup_port_info(&priv->port[port]);
2702
f2a3f6a3 2703 mlx4_cleanup_counters_table(dev);
225c7b1f
RD		 2704 mlx4_cleanup_qp_table(dev);
2705 mlx4_cleanup_srq_table(dev);
2706 mlx4_cleanup_cq_table(dev);
2707 mlx4_cmd_use_polling(dev);
2708 mlx4_cleanup_eq_table(dev);
fe6f700d 2709 mlx4_cleanup_mcg_table(dev);
225c7b1f 2710 mlx4_cleanup_mr_table(dev);
012a8ff5 2711 mlx4_cleanup_xrcd_table(dev);
225c7b1f
RD		 2712 mlx4_cleanup_pd_table(dev);
2713 mlx4_cleanup_uar_table(dev);
2714
b12d93d6 2715err_steer:
ab9c17a0
JM		 2716 if (!mlx4_is_slave(dev))
2717 mlx4_clear_steering(dev);
b12d93d6 2718
e1c00e10
MD		 2719err_disable_msix:
2720 if (dev->flags & MLX4_FLAG_MSI_X)
2721 pci_disable_msix(pdev);
2722
b8dd786f
YP		 2723err_free_eq:
2724 mlx4_free_eq_table(dev);
2725
ab9c17a0
JM		 2726err_master_mfunc:
2727 if (mlx4_is_master(dev))
2728 mlx4_multi_func_cleanup(dev);
2729
b38f2879 2730 if (mlx4_is_slave(dev)) {
99ec41d0 2731 kfree(dev->caps.qp0_qkey);
b38f2879
DB		 2732 kfree(dev->caps.qp0_tunnel);
2733 kfree(dev->caps.qp0_proxy);
2734 kfree(dev->caps.qp1_tunnel);
2735 kfree(dev->caps.qp1_proxy);
2736 }
2737
225c7b1f
RD		 2738err_close:
2739 mlx4_close_hca(dev);
2740
a0eacca9
MB		 2741err_fw:
2742 mlx4_close_fw(dev);
2743
ab9c17a0
JM		 2744err_mfunc:
2745 if (mlx4_is_slave(dev))
2746 mlx4_multi_func_cleanup(dev);
2747
225c7b1f 2748err_cmd:
ffc39f6d 2749 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
225c7b1f 2750
ab9c17a0 2751err_sriov:
bbb07af4 2752 if (dev->flags & MLX4_FLAG_SRIOV && !existing_vfs)
ab9c17a0
JM		 2753 pci_disable_sriov(pdev);
2754
e1a5ddc5
AV		 2755 if (mlx4_is_master(dev) && dev->num_vfs)
2756 atomic_dec(&pf_loading);
2757
1ab95d37
MB		 2758 kfree(priv->dev.dev_vfs);
2759
e1c00e10
MD		 2760 if (!mlx4_is_slave(dev))
2761 mlx4_free_ownership(dev);
2762
7ae0e400 2763 kfree(dev_cap);
e1c00e10
MD		 2764 return err;
2765}
2766
2767static int __mlx4_init_one(struct pci_dev *pdev, int pci_dev_data,
2768 struct mlx4_priv *priv)
2769{
2770 int err;
2771 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
2772 int prb_vf[MLX4_MAX_PORTS + 1] = {0, 0, 0};
2773 const int param_map[MLX4_MAX_PORTS + 1][MLX4_MAX_PORTS + 1] = {
2774 {2, 0, 0}, {0, 1, 2}, {0, 1, 2} };
2775 unsigned total_vfs = 0;
2776 unsigned int i;
2777
2778 pr_info(DRV_NAME ": Initializing %s\n", pci_name(pdev));
2779
2780 err = pci_enable_device(pdev);
2781 if (err) {
2782 dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
2783 return err;
2784 }
2785
2786 /* Due to requirement that all VFs and the PF are *guaranteed* 2 MACS
2787 * per port, we must limit the number of VFs to 63 (since their are
2788 * 128 MACs)
2789 */
2790 for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) && i < num_vfs_argc;
2791 total_vfs += nvfs[param_map[num_vfs_argc - 1][i]], i++) {
2792 nvfs[param_map[num_vfs_argc - 1][i]] = num_vfs[i];
2793 if (nvfs[i] < 0) {
2794 dev_err(&pdev->dev, "num_vfs module parameter cannot be negative\n");
2795 err = -EINVAL;
2796 goto err_disable_pdev;
2797 }
2798 }
2799 for (i = 0; i < sizeof(prb_vf)/sizeof(prb_vf[0]) && i < probe_vfs_argc;
2800 i++) {
2801 prb_vf[param_map[probe_vfs_argc - 1][i]] = probe_vf[i];
2802 if (prb_vf[i] < 0 || prb_vf[i] > nvfs[i]) {
2803 dev_err(&pdev->dev, "probe_vf module parameter cannot be negative or greater than num_vfs\n");
2804 err = -EINVAL;
2805 goto err_disable_pdev;
2806 }
2807 }
2808 if (total_vfs >= MLX4_MAX_NUM_VF) {
2809 dev_err(&pdev->dev,
2810 "Requested more VF's (%d) than allowed (%d)\n",
2811 total_vfs, MLX4_MAX_NUM_VF - 1);
2812 err = -EINVAL;
2813 goto err_disable_pdev;
2814 }
2815
2816 for (i = 0; i < MLX4_MAX_PORTS; i++) {
2817 if (nvfs[i] + nvfs[2] >= MLX4_MAX_NUM_VF_P_PORT) {
2818 dev_err(&pdev->dev,
2819 "Requested more VF's (%d) for port (%d) than allowed (%d)\n",
2820 nvfs[i] + nvfs[2], i + 1,
2821 MLX4_MAX_NUM_VF_P_PORT - 1);
2822 err = -EINVAL;
2823 goto err_disable_pdev;
2824 }
2825 }
2826
2827 /* Check for BARs. */
2828 if (!(pci_dev_data & MLX4_PCI_DEV_IS_VF) &&
2829 !(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
2830 dev_err(&pdev->dev, "Missing DCS, aborting (driver_data: 0x%x, pci_resource_flags(pdev, 0):0x%lx)\n",
2831 pci_dev_data, pci_resource_flags(pdev, 0));
2832 err = -ENODEV;
2833 goto err_disable_pdev;
2834 }
2835 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
2836 dev_err(&pdev->dev, "Missing UAR, aborting\n");
2837 err = -ENODEV;
2838 goto err_disable_pdev;
2839 }
2840
2841 err = pci_request_regions(pdev, DRV_NAME);
2842 if (err) {
2843 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
2844 goto err_disable_pdev;
2845 }
2846
2847 pci_set_master(pdev);
2848
2849 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2850 if (err) {
2851 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
2852 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2853 if (err) {
2854 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
2855 goto err_release_regions;
2856 }
2857 }
2858 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2859 if (err) {
2860 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit consistent PCI DMA mask\n");
2861 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2862 if (err) {
2863 dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, aborting\n");
2864 goto err_release_regions;
2865 }
2866 }
2867
2868 /* Allow large DMA segments, up to the firmware limit of 1 GB */
2869 dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024);
2870 /* Detect if this device is a virtual function */
2871 if (pci_dev_data & MLX4_PCI_DEV_IS_VF) {
2872 /* When acting as pf, we normally skip vfs unless explicitly
2873 * requested to probe them.
2874 */
2875 if (total_vfs) {
2876 unsigned vfs_offset = 0;
2877
2878 for (i = 0; i < sizeof(nvfs)/sizeof(nvfs[0]) &&
2879 vfs_offset + nvfs[i] < extended_func_num(pdev);
2880 vfs_offset += nvfs[i], i++)
2881 ;
2882 if (i == sizeof(nvfs)/sizeof(nvfs[0])) {
2883 err = -ENODEV;
2884 goto err_release_regions;
2885 }
2886 if ((extended_func_num(pdev) - vfs_offset)
2887 > prb_vf[i]) {
2888 dev_warn(&pdev->dev, "Skipping virtual function:%d\n",
2889 extended_func_num(pdev));
2890 err = -ENODEV;
2891 goto err_release_regions;
2892 }
2893 }
2894 }
2895
2896 err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv);
2897 if (err)
2898 goto err_release_regions;
2899 return 0;
225c7b1f 2900
a01df0fe
RD		 2901err_release_regions:
2902 pci_release_regions(pdev);
225c7b1f
RD		 2903
2904err_disable_pdev:
2905 pci_disable_device(pdev);
2906 pci_set_drvdata(pdev, NULL);
2907 return err;
2908}
2909
1dd06ae8 2910static int mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
3d73c288 2911{
befdf897
WY		 2912 struct mlx4_priv *priv;
2913 struct mlx4_dev *dev;
e1c00e10 2914 int ret;
befdf897 2915
0a645e80 2916 printk_once(KERN_INFO "%s", mlx4_version);
3d73c288 2917
befdf897
WY		 2918 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2919 if (!priv)
2920 return -ENOMEM;
2921
2922 dev = &priv->dev;
e1c00e10 2923 dev->pdev = pdev;
befdf897
WY		 2924 pci_set_drvdata(pdev, dev);
2925 priv->pci_dev_data = id->driver_data;
2926
e1c00e10
MD		 2927 ret = __mlx4_init_one(pdev, id->driver_data, priv);
2928 if (ret)
2929 kfree(priv);
2930
2931 return ret;
3d73c288
RD		 2932}
2933
e1c00e10 2934static void mlx4_unload_one(struct pci_dev *pdev)
225c7b1f
RD		 2935{
2936 struct mlx4_dev *dev = pci_get_drvdata(pdev);
2937 struct mlx4_priv *priv = mlx4_priv(dev);
befdf897 2938 int pci_dev_data;
225c7b1f 2939 int p;
bbb07af4 2940 int active_vfs = 0;
225c7b1f 2941
befdf897
WY		 2942 if (priv->removed)
2943 return;
225c7b1f 2944
befdf897 2945 pci_dev_data = priv->pci_dev_data;
225c7b1f 2946
bbb07af4
JM		 2947 /* Disabling SR-IOV is not allowed while there are active vf's */
2948 if (mlx4_is_master(dev)) {
2949 active_vfs = mlx4_how_many_lives_vf(dev);
2950 if (active_vfs) {
2951 pr_warn("Removing PF when there are active VF's !!\n");
2952 pr_warn("Will not disable SR-IOV.\n");
2953 }
2954 }
befdf897
WY		 2955 mlx4_stop_sense(dev);
2956 mlx4_unregister_device(dev);
225c7b1f 2957
befdf897
WY		 2958 for (p = 1; p <= dev->caps.num_ports; p++) {
2959 mlx4_cleanup_port_info(&priv->port[p]);
2960 mlx4_CLOSE_PORT(dev, p);
2961 }
2962
2963 if (mlx4_is_master(dev))
2964 mlx4_free_resource_tracker(dev,
2965 RES_TR_FREE_SLAVES_ONLY);
2966
2967 mlx4_cleanup_counters_table(dev);
2968 mlx4_cleanup_qp_table(dev);
2969 mlx4_cleanup_srq_table(dev);
2970 mlx4_cleanup_cq_table(dev);
2971 mlx4_cmd_use_polling(dev);
2972 mlx4_cleanup_eq_table(dev);
2973 mlx4_cleanup_mcg_table(dev);
2974 mlx4_cleanup_mr_table(dev);
2975 mlx4_cleanup_xrcd_table(dev);
2976 mlx4_cleanup_pd_table(dev);
225c7b1f 2977
befdf897
WY		 2978 if (mlx4_is_master(dev))
2979 mlx4_free_resource_tracker(dev,
2980 RES_TR_FREE_STRUCTS_ONLY);
47605df9 2981
befdf897
WY		 2982 iounmap(priv->kar);
2983 mlx4_uar_free(dev, &priv->driver_uar);
2984 mlx4_cleanup_uar_table(dev);
2985 if (!mlx4_is_slave(dev))
2986 mlx4_clear_steering(dev);
2987 mlx4_free_eq_table(dev);
2988 if (mlx4_is_master(dev))
2989 mlx4_multi_func_cleanup(dev);
2990 mlx4_close_hca(dev);
a0eacca9 2991 mlx4_close_fw(dev);
befdf897
WY		 2992 if (mlx4_is_slave(dev))
2993 mlx4_multi_func_cleanup(dev);
ffc39f6d 2994 mlx4_cmd_cleanup(dev, MLX4_CMD_CLEANUP_ALL);
47605df9 2995
befdf897
WY		 2996 if (dev->flags & MLX4_FLAG_MSI_X)
2997 pci_disable_msix(pdev);
bbb07af4 2998 if (dev->flags & MLX4_FLAG_SRIOV && !active_vfs) {
befdf897
WY		 2999 mlx4_warn(dev, "Disabling SR-IOV\n");
3000 pci_disable_sriov(pdev);
a0eacca9 3001 dev->flags &= ~MLX4_FLAG_SRIOV;
e1a5ddc5 3002 dev->num_vfs = 0;
225c7b1f 3003 }
befdf897
WY		 3004
3005 if (!mlx4_is_slave(dev))
3006 mlx4_free_ownership(dev);
3007
99ec41d0 3008 kfree(dev->caps.qp0_qkey);
befdf897
WY		 3009 kfree(dev->caps.qp0_tunnel);
3010 kfree(dev->caps.qp0_proxy);
3011 kfree(dev->caps.qp1_tunnel);
3012 kfree(dev->caps.qp1_proxy);
3013 kfree(dev->dev_vfs);
3014
befdf897
WY		 3015 memset(priv, 0, sizeof(*priv));
3016 priv->pci_dev_data = pci_dev_data;
3017 priv->removed = 1;
3018}
3019
3020static void mlx4_remove_one(struct pci_dev *pdev)
3021{
3022 struct mlx4_dev *dev = pci_get_drvdata(pdev);
3023 struct mlx4_priv *priv = mlx4_priv(dev);
3024
e1c00e10
MD		 3025 mlx4_unload_one(pdev);
3026 pci_release_regions(pdev);
3027 pci_disable_device(pdev);
befdf897
WY		 3028 kfree(priv);
3029 pci_set_drvdata(pdev, NULL);
225c7b1f
RD		 3030}
3031
ee49bd93
JM		 3032int mlx4_restart_one(struct pci_dev *pdev)
3033{
839f1243
RD		 3034 struct mlx4_dev *dev = pci_get_drvdata(pdev);
3035 struct mlx4_priv *priv = mlx4_priv(dev);
e1c00e10
MD		 3036 int nvfs[MLX4_MAX_PORTS + 1] = {0, 0, 0};
3037 int pci_dev_data, err, total_vfs;
839f1243
RD		 3038
3039 pci_dev_data = priv->pci_dev_data;
e1c00e10
MD		 3040 total_vfs = dev->num_vfs;
3041 memcpy(nvfs, dev->nvfs, sizeof(dev->nvfs));
3042
3043 mlx4_unload_one(pdev);
3044 err = mlx4_load_one(pdev, pci_dev_data, total_vfs, nvfs, priv);
3045 if (err) {
3046 mlx4_err(dev, "%s: ERROR: mlx4_load_one failed, pci_name=%s, err=%d\n",
3047 __func__, pci_name(pdev), err);
3048 return err;
3049 }
3050
3051 return err;
ee49bd93
JM		 3052}
3053
9baa3c34 3054static const struct pci_device_id mlx4_pci_table[] = {
ab9c17a0 3055 /* MT25408 "Hermon" SDR */
ca3e57a5 3056 { PCI_VDEVICE(MELLANOX, 0x6340), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3057 /* MT25408 "Hermon" DDR */
ca3e57a5 3058 { PCI_VDEVICE(MELLANOX, 0x634a), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3059 /* MT25408 "Hermon" QDR */
ca3e57a5 3060 { PCI_VDEVICE(MELLANOX, 0x6354), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3061 /* MT25408 "Hermon" DDR PCIe gen2 */
ca3e57a5 3062 { PCI_VDEVICE(MELLANOX, 0x6732), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3063 /* MT25408 "Hermon" QDR PCIe gen2 */
ca3e57a5 3064 { PCI_VDEVICE(MELLANOX, 0x673c), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3065 /* MT25408 "Hermon" EN 10GigE */
ca3e57a5 3066 { PCI_VDEVICE(MELLANOX, 0x6368), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3067 /* MT25408 "Hermon" EN 10GigE PCIe gen2 */
ca3e57a5 3068 { PCI_VDEVICE(MELLANOX, 0x6750), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3069 /* MT25458 ConnectX EN 10GBASE-T 10GigE */
ca3e57a5 3070 { PCI_VDEVICE(MELLANOX, 0x6372), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3071 /* MT25458 ConnectX EN 10GBASE-T+Gen2 10GigE */
ca3e57a5 3072 { PCI_VDEVICE(MELLANOX, 0x675a), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3073 /* MT26468 ConnectX EN 10GigE PCIe gen2*/
ca3e57a5 3074 { PCI_VDEVICE(MELLANOX, 0x6764), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3075 /* MT26438 ConnectX EN 40GigE PCIe gen2 5GT/s */
ca3e57a5 3076 { PCI_VDEVICE(MELLANOX, 0x6746), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3077 /* MT26478 ConnectX2 40GigE PCIe gen2 */
ca3e57a5 3078 { PCI_VDEVICE(MELLANOX, 0x676e), MLX4_PCI_DEV_FORCE_SENSE_PORT },
ab9c17a0 3079 /* MT25400 Family [ConnectX-2 Virtual Function] */
839f1243 3080 { PCI_VDEVICE(MELLANOX, 0x1002), MLX4_PCI_DEV_IS_VF },
ab9c17a0
JM		 3081 /* MT27500 Family [ConnectX-3] */
3082 { PCI_VDEVICE(MELLANOX, 0x1003), 0 },
3083 /* MT27500 Family [ConnectX-3 Virtual Function] */
839f1243 3084 { PCI_VDEVICE(MELLANOX, 0x1004), MLX4_PCI_DEV_IS_VF },
ab9c17a0
JM		 3085 { PCI_VDEVICE(MELLANOX, 0x1005), 0 }, /* MT27510 Family */
3086 { PCI_VDEVICE(MELLANOX, 0x1006), 0 }, /* MT27511 Family */
3087 { PCI_VDEVICE(MELLANOX, 0x1007), 0 }, /* MT27520 Family */
3088 { PCI_VDEVICE(MELLANOX, 0x1008), 0 }, /* MT27521 Family */
3089 { PCI_VDEVICE(MELLANOX, 0x1009), 0 }, /* MT27530 Family */
3090 { PCI_VDEVICE(MELLANOX, 0x100a), 0 }, /* MT27531 Family */
3091 { PCI_VDEVICE(MELLANOX, 0x100b), 0 }, /* MT27540 Family */
3092 { PCI_VDEVICE(MELLANOX, 0x100c), 0 }, /* MT27541 Family */
3093 { PCI_VDEVICE(MELLANOX, 0x100d), 0 }, /* MT27550 Family */
3094 { PCI_VDEVICE(MELLANOX, 0x100e), 0 }, /* MT27551 Family */
3095 { PCI_VDEVICE(MELLANOX, 0x100f), 0 }, /* MT27560 Family */
3096 { PCI_VDEVICE(MELLANOX, 0x1010), 0 }, /* MT27561 Family */
225c7b1f
RD		 3097 { 0, }
3098};
3099
3100MODULE_DEVICE_TABLE(pci, mlx4_pci_table);
3101
57dbf29a
KSS		 3102static pci_ers_result_t mlx4_pci_err_detected(struct pci_dev *pdev,
3103 pci_channel_state_t state)
3104{
e1c00e10 3105 mlx4_unload_one(pdev);
57dbf29a
KSS		 3106
3107 return state == pci_channel_io_perm_failure ?
3108 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3109}
3110
3111static pci_ers_result_t mlx4_pci_slot_reset(struct pci_dev *pdev)
3112{
befdf897
WY		 3113 struct mlx4_dev *dev = pci_get_drvdata(pdev);
3114 struct mlx4_priv *priv = mlx4_priv(dev);
3115 int ret;
97a5221f 3116
e1c00e10 3117 ret = __mlx4_init_one(pdev, priv->pci_dev_data, priv);
57dbf29a
KSS		 3118
3119 return ret ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
3120}
3121
3646f0e5 3122static const struct pci_error_handlers mlx4_err_handler = {
57dbf29a
KSS		 3123 .error_detected = mlx4_pci_err_detected,
3124 .slot_reset = mlx4_pci_slot_reset,
3125};
3126
225c7b1f
RD		 3127static struct pci_driver mlx4_driver = {
3128 .name = DRV_NAME,
3129 .id_table = mlx4_pci_table,
3130 .probe = mlx4_init_one,
e1c00e10 3131 .shutdown = mlx4_unload_one,
f57e6848 3132 .remove = mlx4_remove_one,
57dbf29a 3133 .err_handler = &mlx4_err_handler,
225c7b1f
RD		 3134};
3135
7ff93f8b
YP		 3136static int __init mlx4_verify_params(void)
3137{
3138 if ((log_num_mac < 0) || (log_num_mac > 7)) {
c20862c8 3139 pr_warn("mlx4_core: bad num_mac: %d\n", log_num_mac);
7ff93f8b
YP		 3140 return -1;
3141 }
3142
cb29688a 3143 if (log_num_vlan != 0)
c20862c8
AV		 3144 pr_warn("mlx4_core: log_num_vlan - obsolete module param, using %d\n",
3145 MLX4_LOG_NUM_VLANS);
7ff93f8b 3146
ecc8fb11
AV		 3147 if (use_prio != 0)
3148 pr_warn("mlx4_core: use_prio - obsolete module param, ignored\n");
7ff93f8b 3149
0498628f 3150 if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 7)) {
c20862c8
AV		 3151 pr_warn("mlx4_core: bad log_mtts_per_seg: %d\n",
3152 log_mtts_per_seg);
ab6bf42e
EC		 3153 return -1;
3154 }
3155
ab9c17a0
JM		 3156 /* Check if module param for ports type has legal combination */
3157 if (port_type_array[0] == false && port_type_array[1] == true) {
c20862c8 3158 pr_warn("Module parameter configuration ETH/IB is not supported. Switching to default configuration IB/IB\n");
ab9c17a0
JM		 3159 port_type_array[0] = true;
3160 }
3161
3c439b55
JM		 3162 if (mlx4_log_num_mgm_entry_size != -1 &&
3163 (mlx4_log_num_mgm_entry_size < MLX4_MIN_MGM_LOG_ENTRY_SIZE ||
3164 mlx4_log_num_mgm_entry_size > MLX4_MAX_MGM_LOG_ENTRY_SIZE)) {
1a91de28
JP		 3165 pr_warn("mlx4_core: mlx4_log_num_mgm_entry_size (%d) not in legal range (-1 or %d..%d)\n",
3166 mlx4_log_num_mgm_entry_size,
3167 MLX4_MIN_MGM_LOG_ENTRY_SIZE,
3168 MLX4_MAX_MGM_LOG_ENTRY_SIZE);
3c439b55
JM		 3169 return -1;
3170 }
3171
7ff93f8b
YP		 3172 return 0;
3173}
3174
225c7b1f
RD		 3175static int __init mlx4_init(void)
3176{
3177 int ret;
3178
7ff93f8b
YP		 3179 if (mlx4_verify_params())
3180 return -EINVAL;
3181
27bf91d6
YP		 3182 mlx4_catas_init();
3183
3184 mlx4_wq = create_singlethread_workqueue("mlx4");
3185 if (!mlx4_wq)
3186 return -ENOMEM;
ee49bd93 3187
225c7b1f 3188 ret = pci_register_driver(&mlx4_driver);
1b85ee09
WY		 3189 if (ret < 0)
3190 destroy_workqueue(mlx4_wq);
225c7b1f
RD		 3191 return ret < 0 ? ret : 0;
3192}
3193
3194static void __exit mlx4_cleanup(void)
3195{
3196 pci_unregister_driver(&mlx4_driver);
27bf91d6 3197 destroy_workqueue(mlx4_wq);
225c7b1f
RD		 3198}
3199
3200module_init(mlx4_init);
3201module_exit(mlx4_cleanup);
Linux kernel - Deliverables
This page took 0.93815 seconds and 5 git commands to generate.