Commit | Line | Data |
---|---|---|
fce8a7bb JM |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * BSD LICENSE | |
14 | * | |
15 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
16 | * | |
17 | * Redistribution and use in source and binary forms, with or without | |
18 | * modification, are permitted provided that the following conditions | |
19 | * are met: | |
20 | * | |
21 | * * Redistributions of source code must retain the above copyright | |
22 | * notice, this list of conditions and the following disclaimer. | |
23 | * * Redistributions in binary form must reproduce the above copy | |
24 | * notice, this list of conditions and the following disclaimer in | |
25 | * the documentation and/or other materials provided with the | |
26 | * distribution. | |
27 | * * Neither the name of Intel Corporation nor the names of its | |
28 | * contributors may be used to endorse or promote products derived | |
29 | * from this software without specific prior written permission. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
32 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
33 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
34 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
35 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
36 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
37 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
38 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
39 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
40 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
41 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
42 | * | |
43 | * Intel PCIe NTB Linux driver | |
44 | * | |
45 | * Contact Information: | |
46 | * Jon Mason <jon.mason@intel.com> | |
47 | */ | |
48 | #include <linux/debugfs.h> | |
113bf1c9 | 49 | #include <linux/delay.h> |
fce8a7bb JM |
50 | #include <linux/init.h> |
51 | #include <linux/interrupt.h> | |
52 | #include <linux/module.h> | |
53 | #include <linux/pci.h> | |
113bf1c9 | 54 | #include <linux/random.h> |
fce8a7bb JM |
55 | #include <linux/slab.h> |
56 | #include "ntb_hw.h" | |
57 | #include "ntb_regs.h" | |
58 | ||
59 | #define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver" | |
db3bb3f0 | 60 | #define NTB_VER "1.0" |
fce8a7bb JM |
61 | |
62 | MODULE_DESCRIPTION(NTB_NAME); | |
63 | MODULE_VERSION(NTB_VER); | |
64 | MODULE_LICENSE("Dual BSD/GPL"); | |
65 | MODULE_AUTHOR("Intel Corporation"); | |
66 | ||
948d3a65 JM |
67 | static bool xeon_errata_workaround = true; |
68 | module_param(xeon_errata_workaround, bool, 0644); | |
69 | MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata"); | |
70 | ||
fce8a7bb | 71 | enum { |
ed6c24ed | 72 | NTB_CONN_TRANSPARENT = 0, |
fce8a7bb JM |
73 | NTB_CONN_B2B, |
74 | NTB_CONN_RP, | |
75 | }; | |
76 | ||
77 | enum { | |
78 | NTB_DEV_USD = 0, | |
79 | NTB_DEV_DSD, | |
80 | }; | |
81 | ||
82 | enum { | |
83 | SNB_HW = 0, | |
84 | BWD_HW, | |
85 | }; | |
86 | ||
1517a3f2 JM |
87 | static struct dentry *debugfs_dir; |
88 | ||
113bf1c9 JM |
89 | #define BWD_LINK_RECOVERY_TIME 500 |
90 | ||
fce8a7bb | 91 | /* Translate memory window 0,1 to BAR 2,4 */ |
948d3a65 | 92 | #define MW_TO_BAR(mw) (mw * NTB_MAX_NUM_MW + 2) |
fce8a7bb | 93 | |
53ca4fea | 94 | static const struct pci_device_id ntb_pci_tbl[] = { |
fce8a7bb JM |
95 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)}, |
96 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, | |
fce8a7bb | 97 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, |
be4dac0f JM |
98 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, |
99 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, | |
100 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, | |
101 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, | |
102 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, | |
103 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, | |
104 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, | |
105 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, | |
106 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, | |
107 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, | |
fce8a7bb JM |
108 | {0} |
109 | }; | |
110 | MODULE_DEVICE_TABLE(pci, ntb_pci_tbl); | |
111 | ||
112 | /** | |
113 | * ntb_register_event_callback() - register event callback | |
114 | * @ndev: pointer to ntb_device instance | |
115 | * @func: callback function to register | |
116 | * | |
117 | * This function registers a callback for any HW driver events such as link | |
118 | * up/down, power management notices and etc. | |
119 | * | |
120 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
121 | */ | |
122 | int ntb_register_event_callback(struct ntb_device *ndev, | |
53ca4fea JM |
123 | void (*func)(void *handle, |
124 | enum ntb_hw_event event)) | |
fce8a7bb JM |
125 | { |
126 | if (ndev->event_cb) | |
127 | return -EINVAL; | |
128 | ||
129 | ndev->event_cb = func; | |
130 | ||
131 | return 0; | |
132 | } | |
133 | ||
134 | /** | |
135 | * ntb_unregister_event_callback() - unregisters the event callback | |
136 | * @ndev: pointer to ntb_device instance | |
137 | * | |
138 | * This function unregisters the existing callback from transport | |
139 | */ | |
140 | void ntb_unregister_event_callback(struct ntb_device *ndev) | |
141 | { | |
142 | ndev->event_cb = NULL; | |
143 | } | |
144 | ||
e8aeb60c JM |
145 | static void ntb_irq_work(unsigned long data) |
146 | { | |
147 | struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data; | |
148 | int rc; | |
149 | ||
150 | rc = db_cb->callback(db_cb->data, db_cb->db_num); | |
151 | if (rc) | |
152 | tasklet_schedule(&db_cb->irq_work); | |
153 | else { | |
154 | struct ntb_device *ndev = db_cb->ndev; | |
155 | unsigned long mask; | |
156 | ||
157 | mask = readw(ndev->reg_ofs.ldb_mask); | |
158 | clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
159 | writew(mask, ndev->reg_ofs.ldb_mask); | |
160 | } | |
161 | } | |
162 | ||
fce8a7bb JM |
163 | /** |
164 | * ntb_register_db_callback() - register a callback for doorbell interrupt | |
165 | * @ndev: pointer to ntb_device instance | |
166 | * @idx: doorbell index to register callback, zero based | |
f9a2cf89 | 167 | * @data: pointer to be returned to caller with every callback |
fce8a7bb JM |
168 | * @func: callback function to register |
169 | * | |
170 | * This function registers a callback function for the doorbell interrupt | |
171 | * on the primary side. The function will unmask the doorbell as well to | |
172 | * allow interrupt. | |
173 | * | |
174 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
175 | */ | |
176 | int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, | |
e8aeb60c | 177 | void *data, int (*func)(void *data, int db_num)) |
fce8a7bb JM |
178 | { |
179 | unsigned long mask; | |
180 | ||
181 | if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) { | |
182 | dev_warn(&ndev->pdev->dev, "Invalid Index.\n"); | |
183 | return -EINVAL; | |
184 | } | |
185 | ||
186 | ndev->db_cb[idx].callback = func; | |
187 | ndev->db_cb[idx].data = data; | |
e8aeb60c JM |
188 | ndev->db_cb[idx].ndev = ndev; |
189 | ||
190 | tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, | |
191 | (unsigned long) &ndev->db_cb[idx]); | |
fce8a7bb JM |
192 | |
193 | /* unmask interrupt */ | |
49793889 | 194 | mask = readw(ndev->reg_ofs.ldb_mask); |
fce8a7bb | 195 | clear_bit(idx * ndev->bits_per_vector, &mask); |
49793889 | 196 | writew(mask, ndev->reg_ofs.ldb_mask); |
fce8a7bb JM |
197 | |
198 | return 0; | |
199 | } | |
200 | ||
201 | /** | |
202 | * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt | |
203 | * @ndev: pointer to ntb_device instance | |
204 | * @idx: doorbell index to register callback, zero based | |
205 | * | |
206 | * This function unregisters a callback function for the doorbell interrupt | |
207 | * on the primary side. The function will also mask the said doorbell. | |
208 | */ | |
209 | void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) | |
210 | { | |
211 | unsigned long mask; | |
212 | ||
213 | if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) | |
214 | return; | |
215 | ||
49793889 | 216 | mask = readw(ndev->reg_ofs.ldb_mask); |
fce8a7bb | 217 | set_bit(idx * ndev->bits_per_vector, &mask); |
49793889 | 218 | writew(mask, ndev->reg_ofs.ldb_mask); |
fce8a7bb | 219 | |
e8aeb60c JM |
220 | tasklet_disable(&ndev->db_cb[idx].irq_work); |
221 | ||
fce8a7bb JM |
222 | ndev->db_cb[idx].callback = NULL; |
223 | } | |
224 | ||
225 | /** | |
226 | * ntb_find_transport() - find the transport pointer | |
227 | * @transport: pointer to pci device | |
228 | * | |
229 | * Given the pci device pointer, return the transport pointer passed in when | |
230 | * the transport attached when it was inited. | |
231 | * | |
232 | * RETURNS: pointer to transport. | |
233 | */ | |
234 | void *ntb_find_transport(struct pci_dev *pdev) | |
235 | { | |
236 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
237 | return ndev->ntb_transport; | |
238 | } | |
239 | ||
240 | /** | |
241 | * ntb_register_transport() - Register NTB transport with NTB HW driver | |
242 | * @transport: transport identifier | |
243 | * | |
244 | * This function allows a transport to reserve the hardware driver for | |
245 | * NTB usage. | |
246 | * | |
247 | * RETURNS: pointer to ntb_device, NULL on error. | |
248 | */ | |
249 | struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport) | |
250 | { | |
251 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
252 | ||
253 | if (ndev->ntb_transport) | |
254 | return NULL; | |
255 | ||
256 | ndev->ntb_transport = transport; | |
257 | return ndev; | |
258 | } | |
259 | ||
260 | /** | |
261 | * ntb_unregister_transport() - Unregister the transport with the NTB HW driver | |
262 | * @ndev - ntb_device of the transport to be freed | |
263 | * | |
264 | * This function unregisters the transport from the HW driver and performs any | |
265 | * necessary cleanups. | |
266 | */ | |
267 | void ntb_unregister_transport(struct ntb_device *ndev) | |
268 | { | |
269 | int i; | |
270 | ||
271 | if (!ndev->ntb_transport) | |
272 | return; | |
273 | ||
274 | for (i = 0; i < ndev->max_cbs; i++) | |
275 | ntb_unregister_db_callback(ndev, i); | |
276 | ||
277 | ntb_unregister_event_callback(ndev); | |
278 | ndev->ntb_transport = NULL; | |
279 | } | |
280 | ||
fce8a7bb JM |
281 | /** |
282 | * ntb_write_local_spad() - write to the secondary scratchpad register | |
283 | * @ndev: pointer to ntb_device instance | |
284 | * @idx: index to the scratchpad register, 0 based | |
285 | * @val: the data value to put into the register | |
286 | * | |
287 | * This function allows writing of a 32bit value to the indexed scratchpad | |
288 | * register. This writes over the data mirrored to the local scratchpad register | |
289 | * by the remote system. | |
290 | * | |
291 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
292 | */ | |
293 | int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
294 | { | |
295 | if (idx >= ndev->limits.max_spads) | |
296 | return -EINVAL; | |
297 | ||
298 | dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", | |
299 | val, idx); | |
300 | writel(val, ndev->reg_ofs.spad_read + idx * 4); | |
301 | ||
302 | return 0; | |
303 | } | |
304 | ||
305 | /** | |
306 | * ntb_read_local_spad() - read from the primary scratchpad register | |
307 | * @ndev: pointer to ntb_device instance | |
308 | * @idx: index to scratchpad register, 0 based | |
309 | * @val: pointer to 32bit integer for storing the register value | |
310 | * | |
311 | * This function allows reading of the 32bit scratchpad register on | |
312 | * the primary (internal) side. This allows the local system to read data | |
313 | * written and mirrored to the scratchpad register by the remote system. | |
314 | * | |
315 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
316 | */ | |
317 | int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
318 | { | |
319 | if (idx >= ndev->limits.max_spads) | |
320 | return -EINVAL; | |
321 | ||
322 | *val = readl(ndev->reg_ofs.spad_write + idx * 4); | |
323 | dev_dbg(&ndev->pdev->dev, | |
324 | "Reading %x from local scratch pad index %d\n", *val, idx); | |
325 | ||
326 | return 0; | |
327 | } | |
328 | ||
329 | /** | |
330 | * ntb_write_remote_spad() - write to the secondary scratchpad register | |
331 | * @ndev: pointer to ntb_device instance | |
332 | * @idx: index to the scratchpad register, 0 based | |
333 | * @val: the data value to put into the register | |
334 | * | |
335 | * This function allows writing of a 32bit value to the indexed scratchpad | |
336 | * register. The register resides on the secondary (external) side. This allows | |
337 | * the local system to write data to be mirrored to the remote systems | |
338 | * scratchpad register. | |
339 | * | |
340 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
341 | */ | |
342 | int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
343 | { | |
344 | if (idx >= ndev->limits.max_spads) | |
345 | return -EINVAL; | |
346 | ||
347 | dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", | |
348 | val, idx); | |
349 | writel(val, ndev->reg_ofs.spad_write + idx * 4); | |
350 | ||
351 | return 0; | |
352 | } | |
353 | ||
354 | /** | |
355 | * ntb_read_remote_spad() - read from the primary scratchpad register | |
356 | * @ndev: pointer to ntb_device instance | |
357 | * @idx: index to scratchpad register, 0 based | |
358 | * @val: pointer to 32bit integer for storing the register value | |
359 | * | |
360 | * This function allows reading of the 32bit scratchpad register on | |
361 | * the primary (internal) side. This alloows the local system to read the data | |
362 | * it wrote to be mirrored on the remote system. | |
363 | * | |
364 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
365 | */ | |
366 | int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
367 | { | |
368 | if (idx >= ndev->limits.max_spads) | |
369 | return -EINVAL; | |
370 | ||
371 | *val = readl(ndev->reg_ofs.spad_read + idx * 4); | |
372 | dev_dbg(&ndev->pdev->dev, | |
373 | "Reading %x from remote scratch pad index %d\n", *val, idx); | |
374 | ||
375 | return 0; | |
376 | } | |
377 | ||
282a2fee JM |
378 | /** |
379 | * ntb_get_mw_base() - get addr for the NTB memory window | |
380 | * @ndev: pointer to ntb_device instance | |
381 | * @mw: memory window number | |
382 | * | |
383 | * This function provides the base address of the memory window specified. | |
384 | * | |
385 | * RETURNS: address, or NULL on error. | |
386 | */ | |
387 | resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw) | |
388 | { | |
389 | if (mw >= ntb_max_mw(ndev)) | |
390 | return 0; | |
391 | ||
392 | return pci_resource_start(ndev->pdev, MW_TO_BAR(mw)); | |
393 | } | |
394 | ||
fce8a7bb JM |
395 | /** |
396 | * ntb_get_mw_vbase() - get virtual addr for the NTB memory window | |
397 | * @ndev: pointer to ntb_device instance | |
398 | * @mw: memory window number | |
399 | * | |
400 | * This function provides the base virtual address of the memory window | |
401 | * specified. | |
402 | * | |
403 | * RETURNS: pointer to virtual address, or NULL on error. | |
404 | */ | |
74465645 | 405 | void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw) |
fce8a7bb | 406 | { |
948d3a65 | 407 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
408 | return NULL; |
409 | ||
410 | return ndev->mw[mw].vbase; | |
411 | } | |
412 | ||
413 | /** | |
414 | * ntb_get_mw_size() - return size of NTB memory window | |
415 | * @ndev: pointer to ntb_device instance | |
416 | * @mw: memory window number | |
417 | * | |
418 | * This function provides the physical size of the memory window specified | |
419 | * | |
420 | * RETURNS: the size of the memory window or zero on error | |
421 | */ | |
ac477afb | 422 | u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw) |
fce8a7bb | 423 | { |
948d3a65 | 424 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
425 | return 0; |
426 | ||
427 | return ndev->mw[mw].bar_sz; | |
428 | } | |
429 | ||
430 | /** | |
431 | * ntb_set_mw_addr - set the memory window address | |
432 | * @ndev: pointer to ntb_device instance | |
433 | * @mw: memory window number | |
434 | * @addr: base address for data | |
435 | * | |
436 | * This function sets the base physical address of the memory window. This | |
437 | * memory address is where data from the remote system will be transfered into | |
438 | * or out of depending on how the transport is configured. | |
439 | */ | |
440 | void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr) | |
441 | { | |
948d3a65 | 442 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
443 | return; |
444 | ||
445 | dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, | |
446 | MW_TO_BAR(mw)); | |
447 | ||
448 | ndev->mw[mw].phys_addr = addr; | |
449 | ||
450 | switch (MW_TO_BAR(mw)) { | |
451 | case NTB_BAR_23: | |
49793889 | 452 | writeq(addr, ndev->reg_ofs.bar2_xlat); |
fce8a7bb JM |
453 | break; |
454 | case NTB_BAR_45: | |
49793889 | 455 | writeq(addr, ndev->reg_ofs.bar4_xlat); |
fce8a7bb JM |
456 | break; |
457 | } | |
458 | } | |
459 | ||
460 | /** | |
49793889 | 461 | * ntb_ring_doorbell() - Set the doorbell on the secondary/external side |
fce8a7bb JM |
462 | * @ndev: pointer to ntb_device instance |
463 | * @db: doorbell to ring | |
464 | * | |
465 | * This function allows triggering of a doorbell on the secondary/external | |
466 | * side that will initiate an interrupt on the remote host | |
467 | * | |
468 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
469 | */ | |
49793889 | 470 | void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db) |
fce8a7bb JM |
471 | { |
472 | dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db); | |
473 | ||
474 | if (ndev->hw_type == BWD_HW) | |
49793889 | 475 | writeq((u64) 1 << db, ndev->reg_ofs.rdb); |
fce8a7bb JM |
476 | else |
477 | writew(((1 << ndev->bits_per_vector) - 1) << | |
49793889 | 478 | (db * ndev->bits_per_vector), ndev->reg_ofs.rdb); |
fce8a7bb JM |
479 | } |
480 | ||
113bf1c9 JM |
481 | static void bwd_recover_link(struct ntb_device *ndev) |
482 | { | |
483 | u32 status; | |
484 | ||
485 | /* Driver resets the NTB ModPhy lanes - magic! */ | |
486 | writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6); | |
487 | writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4); | |
488 | writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4); | |
489 | writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6); | |
490 | ||
491 | /* Driver waits 100ms to allow the NTB ModPhy to settle */ | |
492 | msleep(100); | |
493 | ||
494 | /* Clear AER Errors, write to clear */ | |
495 | status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET); | |
496 | dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status); | |
497 | status &= PCI_ERR_COR_REP_ROLL; | |
498 | writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET); | |
499 | ||
500 | /* Clear unexpected electrical idle event in LTSSM, write to clear */ | |
501 | status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); | |
502 | dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status); | |
503 | status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI; | |
504 | writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); | |
505 | ||
506 | /* Clear DeSkew Buffer error, write to clear */ | |
507 | status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET); | |
508 | dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status); | |
509 | status |= BWD_DESKEWSTS_DBERR; | |
510 | writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET); | |
511 | ||
512 | status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
513 | dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status); | |
514 | status &= BWD_IBIST_ERR_OFLOW; | |
515 | writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
516 | ||
517 | /* Releases the NTB state machine to allow the link to retrain */ | |
518 | status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
519 | dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status); | |
520 | status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT; | |
521 | writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
522 | } | |
523 | ||
fce8a7bb JM |
524 | static void ntb_link_event(struct ntb_device *ndev, int link_state) |
525 | { | |
526 | unsigned int event; | |
527 | ||
528 | if (ndev->link_status == link_state) | |
529 | return; | |
530 | ||
531 | if (link_state == NTB_LINK_UP) { | |
532 | u16 status; | |
533 | ||
534 | dev_info(&ndev->pdev->dev, "Link Up\n"); | |
535 | ndev->link_status = NTB_LINK_UP; | |
536 | event = NTB_EVENT_HW_LINK_UP; | |
537 | ||
ed6c24ed JM |
538 | if (ndev->hw_type == BWD_HW || |
539 | ndev->conn_type == NTB_CONN_TRANSPARENT) | |
fce8a7bb JM |
540 | status = readw(ndev->reg_ofs.lnk_stat); |
541 | else { | |
542 | int rc = pci_read_config_word(ndev->pdev, | |
543 | SNB_LINK_STATUS_OFFSET, | |
544 | &status); | |
545 | if (rc) | |
546 | return; | |
547 | } | |
113bf1c9 JM |
548 | |
549 | ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; | |
550 | ndev->link_speed = (status & NTB_LINK_SPEED_MASK); | |
fce8a7bb | 551 | dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", |
113bf1c9 | 552 | ndev->link_width, ndev->link_speed); |
fce8a7bb JM |
553 | } else { |
554 | dev_info(&ndev->pdev->dev, "Link Down\n"); | |
555 | ndev->link_status = NTB_LINK_DOWN; | |
556 | event = NTB_EVENT_HW_LINK_DOWN; | |
113bf1c9 | 557 | /* Don't modify link width/speed, we need it in link recovery */ |
fce8a7bb JM |
558 | } |
559 | ||
560 | /* notify the upper layer if we have an event change */ | |
561 | if (ndev->event_cb) | |
562 | ndev->event_cb(ndev->ntb_transport, event); | |
563 | } | |
564 | ||
565 | static int ntb_link_status(struct ntb_device *ndev) | |
566 | { | |
567 | int link_state; | |
568 | ||
569 | if (ndev->hw_type == BWD_HW) { | |
570 | u32 ntb_cntl; | |
571 | ||
572 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
573 | if (ntb_cntl & BWD_CNTL_LINK_DOWN) | |
574 | link_state = NTB_LINK_DOWN; | |
575 | else | |
576 | link_state = NTB_LINK_UP; | |
577 | } else { | |
578 | u16 status; | |
579 | int rc; | |
580 | ||
581 | rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, | |
582 | &status); | |
583 | if (rc) | |
584 | return rc; | |
585 | ||
586 | if (status & NTB_LINK_STATUS_ACTIVE) | |
587 | link_state = NTB_LINK_UP; | |
588 | else | |
589 | link_state = NTB_LINK_DOWN; | |
590 | } | |
591 | ||
592 | ntb_link_event(ndev, link_state); | |
593 | ||
594 | return 0; | |
595 | } | |
596 | ||
113bf1c9 JM |
597 | static void bwd_link_recovery(struct work_struct *work) |
598 | { | |
599 | struct ntb_device *ndev = container_of(work, struct ntb_device, | |
600 | lr_timer.work); | |
601 | u32 status32; | |
602 | ||
603 | bwd_recover_link(ndev); | |
604 | /* There is a potential race between the 2 NTB devices recovering at the | |
605 | * same time. If the times are the same, the link will not recover and | |
606 | * the driver will be stuck in this loop forever. Add a random interval | |
607 | * to the recovery time to prevent this race. | |
608 | */ | |
609 | msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME); | |
610 | ||
611 | status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
612 | if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) | |
613 | goto retry; | |
614 | ||
615 | status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
616 | if (status32 & BWD_IBIST_ERR_OFLOW) | |
617 | goto retry; | |
618 | ||
619 | status32 = readl(ndev->reg_ofs.lnk_cntl); | |
620 | if (!(status32 & BWD_CNTL_LINK_DOWN)) { | |
621 | unsigned char speed, width; | |
622 | u16 status16; | |
623 | ||
624 | status16 = readw(ndev->reg_ofs.lnk_stat); | |
625 | width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; | |
626 | speed = (status16 & NTB_LINK_SPEED_MASK); | |
627 | if (ndev->link_width != width || ndev->link_speed != speed) | |
628 | goto retry; | |
629 | } | |
630 | ||
631 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
632 | return; | |
633 | ||
634 | retry: | |
635 | schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT); | |
636 | } | |
637 | ||
fce8a7bb JM |
638 | /* BWD doesn't have link status interrupt, poll on that platform */ |
639 | static void bwd_link_poll(struct work_struct *work) | |
640 | { | |
641 | struct ntb_device *ndev = container_of(work, struct ntb_device, | |
642 | hb_timer.work); | |
643 | unsigned long ts = jiffies; | |
644 | ||
645 | /* If we haven't gotten an interrupt in a while, check the BWD link | |
646 | * status bit | |
647 | */ | |
648 | if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { | |
649 | int rc = ntb_link_status(ndev); | |
650 | if (rc) | |
651 | dev_err(&ndev->pdev->dev, | |
652 | "Error determining link status\n"); | |
113bf1c9 JM |
653 | |
654 | /* Check to see if a link error is the cause of the link down */ | |
655 | if (ndev->link_status == NTB_LINK_DOWN) { | |
656 | u32 status32 = readl(ndev->reg_base + | |
657 | BWD_LTSSMSTATEJMP_OFFSET); | |
658 | if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) { | |
659 | schedule_delayed_work(&ndev->lr_timer, 0); | |
660 | return; | |
661 | } | |
662 | } | |
fce8a7bb JM |
663 | } |
664 | ||
665 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
666 | } | |
667 | ||
668 | static int ntb_xeon_setup(struct ntb_device *ndev) | |
669 | { | |
670 | int rc; | |
671 | u8 val; | |
672 | ||
673 | ndev->hw_type = SNB_HW; | |
674 | ||
675 | rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val); | |
676 | if (rc) | |
677 | return rc; | |
678 | ||
fce8a7bb | 679 | if (val & SNB_PPD_DEV_TYPE) |
fce8a7bb | 680 | ndev->dev_type = NTB_DEV_USD; |
b6750cfe JM |
681 | else |
682 | ndev->dev_type = NTB_DEV_DSD; | |
fce8a7bb | 683 | |
ed6c24ed JM |
684 | switch (val & SNB_PPD_CONN_TYPE) { |
685 | case NTB_CONN_B2B: | |
686 | dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); | |
687 | ndev->conn_type = NTB_CONN_B2B; | |
688 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; | |
689 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; | |
690 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; | |
691 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; | |
692 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; | |
693 | ndev->limits.max_spads = SNB_MAX_B2B_SPADS; | |
694 | ||
695 | /* There is a Xeon hardware errata related to writes to | |
696 | * SDOORBELL or B2BDOORBELL in conjunction with inbound access | |
697 | * to NTB MMIO Space, which may hang the system. To workaround | |
698 | * this use the second memory window to access the interrupt and | |
699 | * scratch pad registers on the remote system. | |
948d3a65 | 700 | */ |
ed6c24ed JM |
701 | if (xeon_errata_workaround) { |
702 | if (!ndev->mw[1].bar_sz) | |
703 | return -EINVAL; | |
704 | ||
705 | ndev->limits.max_mw = SNB_ERRATA_MAX_MW; | |
c529aa30 | 706 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ed6c24ed JM |
707 | ndev->reg_ofs.spad_write = ndev->mw[1].vbase + |
708 | SNB_SPAD_OFFSET; | |
709 | ndev->reg_ofs.rdb = ndev->mw[1].vbase + | |
710 | SNB_PDOORBELL_OFFSET; | |
711 | ||
712 | /* Set the Limit register to 4k, the minimum size, to | |
713 | * prevent an illegal access | |
714 | */ | |
715 | writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base + | |
716 | SNB_PBAR4LMT_OFFSET); | |
58b88920 JM |
717 | /* HW errata on the Limit registers. They can only be |
718 | * written when the base register is 4GB aligned and | |
53ca4fea JM |
719 | * < 32bit. This should already be the case based on |
720 | * the driver defaults, but write the Limit registers | |
721 | * first just in case. | |
58b88920 | 722 | */ |
ed6c24ed JM |
723 | } else { |
724 | ndev->limits.max_mw = SNB_MAX_MW; | |
c529aa30 JM |
725 | |
726 | /* HW Errata on bit 14 of b2bdoorbell register. Writes | |
727 | * will not be mirrored to the remote system. Shrink | |
728 | * the number of bits by one, since bit 14 is the last | |
729 | * bit. | |
730 | */ | |
731 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1; | |
ed6c24ed JM |
732 | ndev->reg_ofs.spad_write = ndev->reg_base + |
733 | SNB_B2B_SPAD_OFFSET; | |
734 | ndev->reg_ofs.rdb = ndev->reg_base + | |
735 | SNB_B2B_DOORBELL_OFFSET; | |
736 | ||
737 | /* Disable the Limit register, just incase it is set to | |
738 | * something silly | |
739 | */ | |
740 | writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET); | |
58b88920 JM |
741 | /* HW errata on the Limit registers. They can only be |
742 | * written when the base register is 4GB aligned and | |
53ca4fea JM |
743 | * < 32bit. This should already be the case based on |
744 | * the driver defaults, but write the Limit registers | |
745 | * first just in case. | |
58b88920 | 746 | */ |
ed6c24ed | 747 | } |
948d3a65 | 748 | |
ed6c24ed JM |
749 | /* The Xeon errata workaround requires setting SBAR Base |
750 | * addresses to known values, so that the PBAR XLAT can be | |
751 | * pointed at SBAR0 of the remote system. | |
948d3a65 | 752 | */ |
ed6c24ed JM |
753 | if (ndev->dev_type == NTB_DEV_USD) { |
754 | writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + | |
755 | SNB_PBAR2XLAT_OFFSET); | |
756 | if (xeon_errata_workaround) | |
757 | writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + | |
758 | SNB_PBAR4XLAT_OFFSET); | |
759 | else { | |
760 | writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base + | |
761 | SNB_PBAR4XLAT_OFFSET); | |
762 | /* B2B_XLAT_OFFSET is a 64bit register, but can | |
763 | * only take 32bit writes | |
764 | */ | |
765 | writel(SNB_MBAR01_DSD_ADDR & 0xffffffff, | |
766 | ndev->reg_base + SNB_B2B_XLAT_OFFSETL); | |
767 | writel(SNB_MBAR01_DSD_ADDR >> 32, | |
768 | ndev->reg_base + SNB_B2B_XLAT_OFFSETU); | |
769 | } | |
948d3a65 | 770 | |
ed6c24ed JM |
771 | writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + |
772 | SNB_SBAR0BASE_OFFSET); | |
773 | writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + | |
774 | SNB_SBAR2BASE_OFFSET); | |
775 | writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base + | |
776 | SNB_SBAR4BASE_OFFSET); | |
777 | } else { | |
778 | writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + | |
779 | SNB_PBAR2XLAT_OFFSET); | |
780 | if (xeon_errata_workaround) | |
781 | writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + | |
782 | SNB_PBAR4XLAT_OFFSET); | |
783 | else { | |
784 | writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base + | |
785 | SNB_PBAR4XLAT_OFFSET); | |
786 | /* B2B_XLAT_OFFSET is a 64bit register, but can | |
787 | * only take 32bit writes | |
788 | */ | |
c8eee379 | 789 | writel(SNB_MBAR01_USD_ADDR & 0xffffffff, |
ed6c24ed JM |
790 | ndev->reg_base + SNB_B2B_XLAT_OFFSETL); |
791 | writel(SNB_MBAR01_USD_ADDR >> 32, | |
792 | ndev->reg_base + SNB_B2B_XLAT_OFFSETU); | |
793 | } | |
948d3a65 | 794 | writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + |
ed6c24ed JM |
795 | SNB_SBAR0BASE_OFFSET); |
796 | writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + | |
797 | SNB_SBAR2BASE_OFFSET); | |
948d3a65 | 798 | writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base + |
ed6c24ed | 799 | SNB_SBAR4BASE_OFFSET); |
948d3a65 | 800 | } |
ed6c24ed JM |
801 | break; |
802 | case NTB_CONN_RP: | |
803 | dev_info(&ndev->pdev->dev, "Conn Type = RP\n"); | |
804 | ndev->conn_type = NTB_CONN_RP; | |
948d3a65 | 805 | |
ed6c24ed | 806 | if (xeon_errata_workaround) { |
53ca4fea | 807 | dev_err(&ndev->pdev->dev, |
ed6c24ed JM |
808 | "NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n"); |
809 | return -EINVAL; | |
948d3a65 | 810 | } |
ed6c24ed JM |
811 | |
812 | /* Scratch pads need to have exclusive access from the primary | |
813 | * or secondary side. Halve the num spads so that each side can | |
814 | * have an equal amount. | |
815 | */ | |
816 | ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; | |
c529aa30 | 817 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ed6c24ed JM |
818 | /* Note: The SDOORBELL is the cause of the errata. You REALLY |
819 | * don't want to touch it. | |
820 | */ | |
821 | ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET; | |
822 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; | |
823 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; | |
824 | /* Offset the start of the spads to correspond to whether it is | |
825 | * primary or secondary | |
826 | */ | |
827 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET + | |
828 | ndev->limits.max_spads * 4; | |
829 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; | |
830 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; | |
831 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; | |
832 | ndev->limits.max_mw = SNB_MAX_MW; | |
833 | break; | |
834 | case NTB_CONN_TRANSPARENT: | |
835 | dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n"); | |
836 | ndev->conn_type = NTB_CONN_TRANSPARENT; | |
837 | /* Scratch pads need to have exclusive access from the primary | |
838 | * or secondary side. Halve the num spads so that each side can | |
839 | * have an equal amount. | |
840 | */ | |
841 | ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; | |
c529aa30 | 842 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ed6c24ed JM |
843 | ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; |
844 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET; | |
845 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET; | |
846 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET; | |
847 | /* Offset the start of the spads to correspond to whether it is | |
848 | * primary or secondary | |
849 | */ | |
850 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET + | |
851 | ndev->limits.max_spads * 4; | |
852 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET; | |
853 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET; | |
854 | ||
855 | ndev->limits.max_mw = SNB_MAX_MW; | |
856 | break; | |
857 | default: | |
858 | /* Most likely caused by the remote NTB-RP device not being | |
859 | * configured | |
860 | */ | |
861 | dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val); | |
862 | return -EINVAL; | |
fce8a7bb JM |
863 | } |
864 | ||
ed6c24ed JM |
865 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; |
866 | ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; | |
867 | ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; | |
868 | ||
fce8a7bb JM |
869 | ndev->limits.msix_cnt = SNB_MSIX_CNT; |
870 | ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; | |
871 | ||
872 | return 0; | |
873 | } | |
874 | ||
875 | static int ntb_bwd_setup(struct ntb_device *ndev) | |
876 | { | |
877 | int rc; | |
878 | u32 val; | |
879 | ||
880 | ndev->hw_type = BWD_HW; | |
881 | ||
882 | rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); | |
883 | if (rc) | |
884 | return rc; | |
885 | ||
886 | switch ((val & BWD_PPD_CONN_TYPE) >> 8) { | |
887 | case NTB_CONN_B2B: | |
888 | ndev->conn_type = NTB_CONN_B2B; | |
889 | break; | |
890 | case NTB_CONN_RP: | |
891 | default: | |
b1ef0043 | 892 | dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); |
fce8a7bb JM |
893 | return -EINVAL; |
894 | } | |
895 | ||
896 | if (val & BWD_PPD_DEV_TYPE) | |
897 | ndev->dev_type = NTB_DEV_DSD; | |
898 | else | |
899 | ndev->dev_type = NTB_DEV_USD; | |
900 | ||
901 | /* Initiate PCI-E link training */ | |
902 | rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET, | |
903 | val | BWD_PPD_INIT_LINK); | |
904 | if (rc) | |
905 | return rc; | |
906 | ||
49793889 JM |
907 | ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET; |
908 | ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; | |
b1ef0043 | 909 | ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; |
49793889 JM |
910 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; |
911 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; | |
fce8a7bb JM |
912 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; |
913 | ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; | |
914 | ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; | |
b1ef0043 | 915 | ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; |
fce8a7bb | 916 | ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; |
948d3a65 | 917 | ndev->limits.max_mw = BWD_MAX_MW; |
b1ef0043 | 918 | ndev->limits.max_spads = BWD_MAX_SPADS; |
fce8a7bb JM |
919 | ndev->limits.max_db_bits = BWD_MAX_DB_BITS; |
920 | ndev->limits.msix_cnt = BWD_MSIX_CNT; | |
921 | ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; | |
922 | ||
923 | /* Since bwd doesn't have a link interrupt, setup a poll timer */ | |
924 | INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); | |
113bf1c9 | 925 | INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery); |
fce8a7bb JM |
926 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); |
927 | ||
928 | return 0; | |
929 | } | |
930 | ||
78a61ab7 | 931 | static int ntb_device_setup(struct ntb_device *ndev) |
fce8a7bb JM |
932 | { |
933 | int rc; | |
934 | ||
935 | switch (ndev->pdev->device) { | |
be4dac0f JM |
936 | case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: |
937 | case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: | |
938 | case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: | |
939 | case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: | |
940 | case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: | |
941 | case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: | |
942 | case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: | |
943 | case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: | |
fce8a7bb JM |
944 | case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: |
945 | case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: | |
be4dac0f JM |
946 | case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: |
947 | case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: | |
fce8a7bb JM |
948 | rc = ntb_xeon_setup(ndev); |
949 | break; | |
950 | case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: | |
951 | rc = ntb_bwd_setup(ndev); | |
952 | break; | |
953 | default: | |
954 | rc = -ENODEV; | |
955 | } | |
956 | ||
3b12a0d1 JM |
957 | if (rc) |
958 | return rc; | |
959 | ||
b6750cfe JM |
960 | dev_info(&ndev->pdev->dev, "Device Type = %s\n", |
961 | ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); | |
962 | ||
ed6c24ed JM |
963 | if (ndev->conn_type == NTB_CONN_B2B) |
964 | /* Enable Bus Master and Memory Space on the secondary side */ | |
965 | writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, | |
966 | ndev->reg_ofs.spci_cmd); | |
fce8a7bb | 967 | |
3b12a0d1 | 968 | return 0; |
fce8a7bb JM |
969 | } |
970 | ||
971 | static void ntb_device_free(struct ntb_device *ndev) | |
972 | { | |
113bf1c9 | 973 | if (ndev->hw_type == BWD_HW) { |
fce8a7bb | 974 | cancel_delayed_work_sync(&ndev->hb_timer); |
113bf1c9 JM |
975 | cancel_delayed_work_sync(&ndev->lr_timer); |
976 | } | |
fce8a7bb JM |
977 | } |
978 | ||
979 | static irqreturn_t bwd_callback_msix_irq(int irq, void *data) | |
980 | { | |
981 | struct ntb_db_cb *db_cb = data; | |
982 | struct ntb_device *ndev = db_cb->ndev; | |
e8aeb60c | 983 | unsigned long mask; |
fce8a7bb JM |
984 | |
985 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
986 | db_cb->db_num); | |
987 | ||
e8aeb60c JM |
988 | mask = readw(ndev->reg_ofs.ldb_mask); |
989 | set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
990 | writew(mask, ndev->reg_ofs.ldb_mask); | |
991 | ||
992 | tasklet_schedule(&db_cb->irq_work); | |
fce8a7bb JM |
993 | |
994 | /* No need to check for the specific HB irq, any interrupt means | |
995 | * we're connected. | |
996 | */ | |
997 | ndev->last_ts = jiffies; | |
998 | ||
49793889 | 999 | writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb); |
fce8a7bb JM |
1000 | |
1001 | return IRQ_HANDLED; | |
1002 | } | |
1003 | ||
1004 | static irqreturn_t xeon_callback_msix_irq(int irq, void *data) | |
1005 | { | |
1006 | struct ntb_db_cb *db_cb = data; | |
1007 | struct ntb_device *ndev = db_cb->ndev; | |
e8aeb60c | 1008 | unsigned long mask; |
fce8a7bb JM |
1009 | |
1010 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
1011 | db_cb->db_num); | |
1012 | ||
e8aeb60c JM |
1013 | mask = readw(ndev->reg_ofs.ldb_mask); |
1014 | set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
1015 | writew(mask, ndev->reg_ofs.ldb_mask); | |
1016 | ||
1017 | tasklet_schedule(&db_cb->irq_work); | |
fce8a7bb JM |
1018 | |
1019 | /* On Sandybridge, there are 16 bits in the interrupt register | |
1020 | * but only 4 vectors. So, 5 bits are assigned to the first 3 | |
1021 | * vectors, with the 4th having a single bit for link | |
1022 | * interrupts. | |
1023 | */ | |
1024 | writew(((1 << ndev->bits_per_vector) - 1) << | |
49793889 | 1025 | (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb); |
fce8a7bb JM |
1026 | |
1027 | return IRQ_HANDLED; | |
1028 | } | |
1029 | ||
1030 | /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */ | |
1031 | static irqreturn_t xeon_event_msix_irq(int irq, void *dev) | |
1032 | { | |
1033 | struct ntb_device *ndev = dev; | |
1034 | int rc; | |
1035 | ||
1036 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq); | |
1037 | ||
1038 | rc = ntb_link_status(ndev); | |
1039 | if (rc) | |
1040 | dev_err(&ndev->pdev->dev, "Error determining link status\n"); | |
1041 | ||
1042 | /* bit 15 is always the link bit */ | |
c529aa30 | 1043 | writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); |
fce8a7bb JM |
1044 | |
1045 | return IRQ_HANDLED; | |
1046 | } | |
1047 | ||
1048 | static irqreturn_t ntb_interrupt(int irq, void *dev) | |
1049 | { | |
1050 | struct ntb_device *ndev = dev; | |
1051 | unsigned int i = 0; | |
1052 | ||
1053 | if (ndev->hw_type == BWD_HW) { | |
49793889 | 1054 | u64 ldb = readq(ndev->reg_ofs.ldb); |
fce8a7bb | 1055 | |
49793889 | 1056 | dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb); |
fce8a7bb | 1057 | |
49793889 JM |
1058 | while (ldb) { |
1059 | i = __ffs(ldb); | |
1060 | ldb &= ldb - 1; | |
fce8a7bb JM |
1061 | bwd_callback_msix_irq(irq, &ndev->db_cb[i]); |
1062 | } | |
1063 | } else { | |
49793889 | 1064 | u16 ldb = readw(ndev->reg_ofs.ldb); |
fce8a7bb | 1065 | |
49793889 | 1066 | dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb); |
fce8a7bb | 1067 | |
49793889 | 1068 | if (ldb & SNB_DB_HW_LINK) { |
fce8a7bb | 1069 | xeon_event_msix_irq(irq, dev); |
49793889 | 1070 | ldb &= ~SNB_DB_HW_LINK; |
fce8a7bb JM |
1071 | } |
1072 | ||
49793889 JM |
1073 | while (ldb) { |
1074 | i = __ffs(ldb); | |
1075 | ldb &= ldb - 1; | |
fce8a7bb JM |
1076 | xeon_callback_msix_irq(irq, &ndev->db_cb[i]); |
1077 | } | |
1078 | } | |
1079 | ||
1080 | return IRQ_HANDLED; | |
1081 | } | |
1082 | ||
53a788a7 AG |
1083 | static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries) |
1084 | { | |
1085 | struct pci_dev *pdev = ndev->pdev; | |
1086 | struct msix_entry *msix; | |
1087 | int rc, i; | |
1088 | ||
1089 | if (msix_entries < ndev->limits.msix_cnt) | |
1090 | return -ENOSPC; | |
1091 | ||
f220baad | 1092 | rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries); |
53a788a7 AG |
1093 | if (rc < 0) |
1094 | return rc; | |
53a788a7 AG |
1095 | |
1096 | for (i = 0; i < msix_entries; i++) { | |
1097 | msix = &ndev->msix_entries[i]; | |
1098 | WARN_ON(!msix->vector); | |
1099 | ||
1100 | if (i == msix_entries - 1) { | |
1101 | rc = request_irq(msix->vector, | |
1102 | xeon_event_msix_irq, 0, | |
1103 | "ntb-event-msix", ndev); | |
1104 | if (rc) | |
1105 | goto err; | |
1106 | } else { | |
1107 | rc = request_irq(msix->vector, | |
1108 | xeon_callback_msix_irq, 0, | |
1109 | "ntb-callback-msix", | |
1110 | &ndev->db_cb[i]); | |
1111 | if (rc) | |
1112 | goto err; | |
1113 | } | |
1114 | } | |
1115 | ||
1116 | ndev->num_msix = msix_entries; | |
1117 | ndev->max_cbs = msix_entries - 1; | |
1118 | ||
1119 | return 0; | |
1120 | ||
1121 | err: | |
1122 | while (--i >= 0) { | |
1123 | /* Code never reaches here for entry nr 'ndev->num_msix - 1' */ | |
1124 | msix = &ndev->msix_entries[i]; | |
1125 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1126 | } | |
1127 | ||
1128 | pci_disable_msix(pdev); | |
1129 | ndev->num_msix = 0; | |
1130 | ||
1131 | return rc; | |
1132 | } | |
1133 | ||
1134 | static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries) | |
fce8a7bb JM |
1135 | { |
1136 | struct pci_dev *pdev = ndev->pdev; | |
1137 | struct msix_entry *msix; | |
53a788a7 AG |
1138 | int rc, i; |
1139 | ||
f220baad AG |
1140 | msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries, |
1141 | 1, msix_entries); | |
1142 | if (msix_entries < 0) | |
1143 | return msix_entries; | |
53a788a7 AG |
1144 | |
1145 | for (i = 0; i < msix_entries; i++) { | |
1146 | msix = &ndev->msix_entries[i]; | |
1147 | WARN_ON(!msix->vector); | |
1148 | ||
1149 | rc = request_irq(msix->vector, bwd_callback_msix_irq, 0, | |
1150 | "ntb-callback-msix", &ndev->db_cb[i]); | |
1151 | if (rc) | |
1152 | goto err; | |
1153 | } | |
1154 | ||
1155 | ndev->num_msix = msix_entries; | |
1156 | ndev->max_cbs = msix_entries; | |
1157 | ||
1158 | return 0; | |
1159 | ||
1160 | err: | |
1161 | while (--i >= 0) | |
1162 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1163 | ||
1164 | pci_disable_msix(pdev); | |
1165 | ndev->num_msix = 0; | |
1166 | ||
1167 | return rc; | |
1168 | } | |
1169 | ||
1170 | static int ntb_setup_msix(struct ntb_device *ndev) | |
1171 | { | |
1172 | struct pci_dev *pdev = ndev->pdev; | |
fce8a7bb | 1173 | int msix_entries; |
73f47cad | 1174 | int rc, i; |
fce8a7bb | 1175 | |
77733513 AG |
1176 | msix_entries = pci_msix_vec_count(pdev); |
1177 | if (msix_entries < 0) { | |
1178 | rc = msix_entries; | |
fce8a7bb | 1179 | goto err; |
77733513 | 1180 | } else if (msix_entries > ndev->limits.msix_cnt) { |
fce8a7bb JM |
1181 | rc = -EINVAL; |
1182 | goto err; | |
1183 | } | |
1184 | ||
1185 | ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, | |
1186 | GFP_KERNEL); | |
1187 | if (!ndev->msix_entries) { | |
1188 | rc = -ENOMEM; | |
1189 | goto err; | |
1190 | } | |
1191 | ||
1192 | for (i = 0; i < msix_entries; i++) | |
1193 | ndev->msix_entries[i].entry = i; | |
1194 | ||
fce8a7bb | 1195 | if (ndev->hw_type == BWD_HW) |
53a788a7 | 1196 | rc = ntb_setup_bwd_msix(ndev, msix_entries); |
fce8a7bb | 1197 | else |
53a788a7 AG |
1198 | rc = ntb_setup_snb_msix(ndev, msix_entries); |
1199 | if (rc) | |
1200 | goto err1; | |
fce8a7bb JM |
1201 | |
1202 | return 0; | |
1203 | ||
fce8a7bb JM |
1204 | err1: |
1205 | kfree(ndev->msix_entries); | |
fce8a7bb | 1206 | err: |
53a788a7 | 1207 | dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n"); |
fce8a7bb JM |
1208 | return rc; |
1209 | } | |
1210 | ||
1211 | static int ntb_setup_msi(struct ntb_device *ndev) | |
1212 | { | |
1213 | struct pci_dev *pdev = ndev->pdev; | |
1214 | int rc; | |
1215 | ||
1216 | rc = pci_enable_msi(pdev); | |
1217 | if (rc) | |
1218 | return rc; | |
1219 | ||
1220 | rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); | |
1221 | if (rc) { | |
1222 | pci_disable_msi(pdev); | |
1223 | dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); | |
1224 | return rc; | |
1225 | } | |
1226 | ||
1227 | return 0; | |
1228 | } | |
1229 | ||
1230 | static int ntb_setup_intx(struct ntb_device *ndev) | |
1231 | { | |
1232 | struct pci_dev *pdev = ndev->pdev; | |
1233 | int rc; | |
1234 | ||
1235 | pci_msi_off(pdev); | |
1236 | ||
1237 | /* Verify intx is enabled */ | |
1238 | pci_intx(pdev, 1); | |
1239 | ||
1240 | rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx", | |
1241 | ndev); | |
1242 | if (rc) | |
1243 | return rc; | |
1244 | ||
1245 | return 0; | |
1246 | } | |
1247 | ||
78a61ab7 | 1248 | static int ntb_setup_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
1249 | { |
1250 | int rc; | |
1251 | ||
1252 | /* On BWD, disable all interrupts. On SNB, disable all but Link | |
1253 | * Interrupt. The rest will be unmasked as callbacks are registered. | |
1254 | */ | |
1255 | if (ndev->hw_type == BWD_HW) | |
49793889 | 1256 | writeq(~0, ndev->reg_ofs.ldb_mask); |
c529aa30 JM |
1257 | else { |
1258 | u16 var = 1 << SNB_LINK_DB; | |
1259 | writew(~var, ndev->reg_ofs.ldb_mask); | |
1260 | } | |
fce8a7bb JM |
1261 | |
1262 | rc = ntb_setup_msix(ndev); | |
1263 | if (!rc) | |
1264 | goto done; | |
1265 | ||
1266 | ndev->bits_per_vector = 1; | |
1267 | ndev->max_cbs = ndev->limits.max_db_bits; | |
1268 | ||
1269 | rc = ntb_setup_msi(ndev); | |
1270 | if (!rc) | |
1271 | goto done; | |
1272 | ||
1273 | rc = ntb_setup_intx(ndev); | |
1274 | if (rc) { | |
1275 | dev_err(&ndev->pdev->dev, "no usable interrupts\n"); | |
1276 | return rc; | |
1277 | } | |
1278 | ||
1279 | done: | |
1280 | return 0; | |
1281 | } | |
1282 | ||
78a61ab7 | 1283 | static void ntb_free_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
1284 | { |
1285 | struct pci_dev *pdev = ndev->pdev; | |
1286 | ||
1287 | /* mask interrupts */ | |
1288 | if (ndev->hw_type == BWD_HW) | |
49793889 | 1289 | writeq(~0, ndev->reg_ofs.ldb_mask); |
fce8a7bb | 1290 | else |
49793889 | 1291 | writew(~0, ndev->reg_ofs.ldb_mask); |
fce8a7bb JM |
1292 | |
1293 | if (ndev->num_msix) { | |
1294 | struct msix_entry *msix; | |
1295 | u32 i; | |
1296 | ||
1297 | for (i = 0; i < ndev->num_msix; i++) { | |
1298 | msix = &ndev->msix_entries[i]; | |
1299 | if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1) | |
1300 | free_irq(msix->vector, ndev); | |
1301 | else | |
1302 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1303 | } | |
1304 | pci_disable_msix(pdev); | |
717e8e8b | 1305 | kfree(ndev->msix_entries); |
fce8a7bb JM |
1306 | } else { |
1307 | free_irq(pdev->irq, ndev); | |
1308 | ||
1309 | if (pci_dev_msi_enabled(pdev)) | |
1310 | pci_disable_msi(pdev); | |
1311 | } | |
1312 | } | |
1313 | ||
78a61ab7 | 1314 | static int ntb_create_callbacks(struct ntb_device *ndev) |
fce8a7bb JM |
1315 | { |
1316 | int i; | |
1317 | ||
f9a2cf89 | 1318 | /* Chicken-egg issue. We won't know how many callbacks are necessary |
fce8a7bb | 1319 | * until we see how many MSI-X vectors we get, but these pointers need |
f9a2cf89 | 1320 | * to be passed into the MSI-X register function. So, we allocate the |
fce8a7bb JM |
1321 | * max, knowing that they might not all be used, to work around this. |
1322 | */ | |
1323 | ndev->db_cb = kcalloc(ndev->limits.max_db_bits, | |
1324 | sizeof(struct ntb_db_cb), | |
1325 | GFP_KERNEL); | |
1326 | if (!ndev->db_cb) | |
1327 | return -ENOMEM; | |
1328 | ||
1329 | for (i = 0; i < ndev->limits.max_db_bits; i++) { | |
1330 | ndev->db_cb[i].db_num = i; | |
1331 | ndev->db_cb[i].ndev = ndev; | |
1332 | } | |
1333 | ||
1334 | return 0; | |
1335 | } | |
1336 | ||
1337 | static void ntb_free_callbacks(struct ntb_device *ndev) | |
1338 | { | |
1339 | int i; | |
1340 | ||
1341 | for (i = 0; i < ndev->limits.max_db_bits; i++) | |
1342 | ntb_unregister_db_callback(ndev, i); | |
1343 | ||
1344 | kfree(ndev->db_cb); | |
1345 | } | |
1346 | ||
1517a3f2 JM |
1347 | static void ntb_setup_debugfs(struct ntb_device *ndev) |
1348 | { | |
1349 | if (!debugfs_initialized()) | |
1350 | return; | |
1351 | ||
1352 | if (!debugfs_dir) | |
1353 | debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); | |
1354 | ||
1355 | ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev), | |
1356 | debugfs_dir); | |
1357 | } | |
1358 | ||
1359 | static void ntb_free_debugfs(struct ntb_device *ndev) | |
1360 | { | |
1361 | debugfs_remove_recursive(ndev->debugfs_dir); | |
1362 | ||
1363 | if (debugfs_dir && simple_empty(debugfs_dir)) { | |
1364 | debugfs_remove_recursive(debugfs_dir); | |
1365 | debugfs_dir = NULL; | |
1366 | } | |
1367 | } | |
1368 | ||
9fec60c4 JM |
1369 | static void ntb_hw_link_up(struct ntb_device *ndev) |
1370 | { | |
1371 | if (ndev->conn_type == NTB_CONN_TRANSPARENT) | |
1372 | ntb_link_event(ndev, NTB_LINK_UP); | |
78958433 JM |
1373 | else { |
1374 | u32 ntb_cntl; | |
1375 | ||
9fec60c4 | 1376 | /* Let's bring the NTB link up */ |
78958433 JM |
1377 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); |
1378 | ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); | |
1379 | ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; | |
1380 | ntb_cntl |= NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP; | |
1381 | writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); | |
1382 | } | |
9fec60c4 JM |
1383 | } |
1384 | ||
1385 | static void ntb_hw_link_down(struct ntb_device *ndev) | |
1386 | { | |
1387 | u32 ntb_cntl; | |
1388 | ||
1389 | if (ndev->conn_type == NTB_CONN_TRANSPARENT) { | |
1390 | ntb_link_event(ndev, NTB_LINK_DOWN); | |
1391 | return; | |
1392 | } | |
1393 | ||
1394 | /* Bring NTB link down */ | |
1395 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
78958433 JM |
1396 | ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); |
1397 | ntb_cntl &= ~(NTB_CNTL_P2S_BAR45_SNOOP | NTB_CNTL_S2P_BAR45_SNOOP); | |
1398 | ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; | |
9fec60c4 JM |
1399 | writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); |
1400 | } | |
1401 | ||
78a61ab7 | 1402 | static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
fce8a7bb JM |
1403 | { |
1404 | struct ntb_device *ndev; | |
1405 | int rc, i; | |
1406 | ||
1407 | ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); | |
1408 | if (!ndev) | |
1409 | return -ENOMEM; | |
1410 | ||
1411 | ndev->pdev = pdev; | |
1412 | ndev->link_status = NTB_LINK_DOWN; | |
1413 | pci_set_drvdata(pdev, ndev); | |
1517a3f2 | 1414 | ntb_setup_debugfs(ndev); |
fce8a7bb JM |
1415 | |
1416 | rc = pci_enable_device(pdev); | |
1417 | if (rc) | |
1418 | goto err; | |
1419 | ||
1420 | pci_set_master(ndev->pdev); | |
1421 | ||
1422 | rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME); | |
1423 | if (rc) | |
1424 | goto err1; | |
1425 | ||
1426 | ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); | |
1427 | if (!ndev->reg_base) { | |
1428 | dev_warn(&pdev->dev, "Cannot remap BAR 0\n"); | |
1429 | rc = -EIO; | |
1430 | goto err2; | |
1431 | } | |
1432 | ||
948d3a65 | 1433 | for (i = 0; i < NTB_MAX_NUM_MW; i++) { |
fce8a7bb JM |
1434 | ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i)); |
1435 | ndev->mw[i].vbase = | |
1436 | ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)), | |
1437 | ndev->mw[i].bar_sz); | |
113fc505 | 1438 | dev_info(&pdev->dev, "MW %d size %llu\n", i, |
ac477afb | 1439 | (unsigned long long) ndev->mw[i].bar_sz); |
fce8a7bb JM |
1440 | if (!ndev->mw[i].vbase) { |
1441 | dev_warn(&pdev->dev, "Cannot remap BAR %d\n", | |
1442 | MW_TO_BAR(i)); | |
1443 | rc = -EIO; | |
1444 | goto err3; | |
1445 | } | |
1446 | } | |
1447 | ||
1448 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1449 | if (rc) { | |
1450 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1451 | if (rc) | |
1452 | goto err3; | |
1453 | ||
1454 | dev_warn(&pdev->dev, "Cannot DMA highmem\n"); | |
1455 | } | |
1456 | ||
1457 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1458 | if (rc) { | |
1459 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1460 | if (rc) | |
1461 | goto err3; | |
1462 | ||
1463 | dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); | |
1464 | } | |
1465 | ||
1466 | rc = ntb_device_setup(ndev); | |
1467 | if (rc) | |
1468 | goto err3; | |
1469 | ||
1470 | rc = ntb_create_callbacks(ndev); | |
1471 | if (rc) | |
1472 | goto err4; | |
1473 | ||
1474 | rc = ntb_setup_interrupts(ndev); | |
1475 | if (rc) | |
1476 | goto err5; | |
1477 | ||
1478 | /* The scratchpad registers keep the values between rmmod/insmod, | |
1479 | * blast them now | |
1480 | */ | |
1481 | for (i = 0; i < ndev->limits.max_spads; i++) { | |
1482 | ntb_write_local_spad(ndev, i, 0); | |
1483 | ntb_write_remote_spad(ndev, i, 0); | |
1484 | } | |
1485 | ||
1486 | rc = ntb_transport_init(pdev); | |
1487 | if (rc) | |
1488 | goto err6; | |
1489 | ||
9fec60c4 | 1490 | ntb_hw_link_up(ndev); |
fce8a7bb JM |
1491 | |
1492 | return 0; | |
1493 | ||
1494 | err6: | |
1495 | ntb_free_interrupts(ndev); | |
1496 | err5: | |
1497 | ntb_free_callbacks(ndev); | |
1498 | err4: | |
1499 | ntb_device_free(ndev); | |
1500 | err3: | |
1501 | for (i--; i >= 0; i--) | |
1502 | iounmap(ndev->mw[i].vbase); | |
1503 | iounmap(ndev->reg_base); | |
1504 | err2: | |
1505 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
1506 | err1: | |
1507 | pci_disable_device(pdev); | |
1508 | err: | |
1517a3f2 | 1509 | ntb_free_debugfs(ndev); |
fce8a7bb JM |
1510 | kfree(ndev); |
1511 | ||
1512 | dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME); | |
1513 | return rc; | |
1514 | } | |
1515 | ||
78a61ab7 | 1516 | static void ntb_pci_remove(struct pci_dev *pdev) |
fce8a7bb JM |
1517 | { |
1518 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
1519 | int i; | |
fce8a7bb | 1520 | |
9fec60c4 | 1521 | ntb_hw_link_down(ndev); |
fce8a7bb JM |
1522 | |
1523 | ntb_transport_free(ndev->ntb_transport); | |
1524 | ||
1525 | ntb_free_interrupts(ndev); | |
1526 | ntb_free_callbacks(ndev); | |
1527 | ntb_device_free(ndev); | |
1528 | ||
948d3a65 | 1529 | for (i = 0; i < NTB_MAX_NUM_MW; i++) |
fce8a7bb JM |
1530 | iounmap(ndev->mw[i].vbase); |
1531 | ||
1532 | iounmap(ndev->reg_base); | |
1533 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
1534 | pci_disable_device(pdev); | |
1517a3f2 | 1535 | ntb_free_debugfs(ndev); |
fce8a7bb JM |
1536 | kfree(ndev); |
1537 | } | |
1538 | ||
1539 | static struct pci_driver ntb_pci_driver = { | |
1540 | .name = KBUILD_MODNAME, | |
1541 | .id_table = ntb_pci_tbl, | |
1542 | .probe = ntb_pci_probe, | |
78a61ab7 | 1543 | .remove = ntb_pci_remove, |
fce8a7bb JM |
1544 | }; |
1545 | module_pci_driver(ntb_pci_driver); |