Commit | Line | Data |
---|---|---|
fce8a7bb JM |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * BSD LICENSE | |
14 | * | |
15 | * Copyright(c) 2012 Intel Corporation. All rights reserved. | |
16 | * | |
17 | * Redistribution and use in source and binary forms, with or without | |
18 | * modification, are permitted provided that the following conditions | |
19 | * are met: | |
20 | * | |
21 | * * Redistributions of source code must retain the above copyright | |
22 | * notice, this list of conditions and the following disclaimer. | |
23 | * * Redistributions in binary form must reproduce the above copy | |
24 | * notice, this list of conditions and the following disclaimer in | |
25 | * the documentation and/or other materials provided with the | |
26 | * distribution. | |
27 | * * Neither the name of Intel Corporation nor the names of its | |
28 | * contributors may be used to endorse or promote products derived | |
29 | * from this software without specific prior written permission. | |
30 | * | |
31 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
32 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
33 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
34 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
35 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
36 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
37 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
38 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
39 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
40 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
41 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
42 | * | |
43 | * Intel PCIe NTB Linux driver | |
44 | * | |
45 | * Contact Information: | |
46 | * Jon Mason <jon.mason@intel.com> | |
47 | */ | |
48 | #include <linux/debugfs.h> | |
113bf1c9 | 49 | #include <linux/delay.h> |
fce8a7bb JM |
50 | #include <linux/init.h> |
51 | #include <linux/interrupt.h> | |
52 | #include <linux/module.h> | |
53 | #include <linux/pci.h> | |
113bf1c9 | 54 | #include <linux/random.h> |
fce8a7bb JM |
55 | #include <linux/slab.h> |
56 | #include "ntb_hw.h" | |
57 | #include "ntb_regs.h" | |
58 | ||
59 | #define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver" | |
db3bb3f0 | 60 | #define NTB_VER "1.0" |
fce8a7bb JM |
61 | |
62 | MODULE_DESCRIPTION(NTB_NAME); | |
63 | MODULE_VERSION(NTB_VER); | |
64 | MODULE_LICENSE("Dual BSD/GPL"); | |
65 | MODULE_AUTHOR("Intel Corporation"); | |
66 | ||
67 | enum { | |
ed6c24ed | 68 | NTB_CONN_TRANSPARENT = 0, |
fce8a7bb JM |
69 | NTB_CONN_B2B, |
70 | NTB_CONN_RP, | |
71 | }; | |
72 | ||
73 | enum { | |
74 | NTB_DEV_USD = 0, | |
75 | NTB_DEV_DSD, | |
76 | }; | |
77 | ||
78 | enum { | |
79 | SNB_HW = 0, | |
80 | BWD_HW, | |
81 | }; | |
82 | ||
1517a3f2 JM |
83 | static struct dentry *debugfs_dir; |
84 | ||
113bf1c9 JM |
85 | #define BWD_LINK_RECOVERY_TIME 500 |
86 | ||
ab760a0c DJ |
87 | /* Translate memory window 0,1,2 to BAR 2,4,5 */ |
88 | #define MW_TO_BAR(mw) (mw == 0 ? 2 : (mw == 1 ? 4 : 5)) | |
fce8a7bb | 89 | |
53ca4fea | 90 | static const struct pci_device_id ntb_pci_tbl[] = { |
fce8a7bb JM |
91 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)}, |
92 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, | |
fce8a7bb | 93 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, |
be4dac0f JM |
94 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, |
95 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, | |
96 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, | |
97 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, | |
98 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, | |
99 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, | |
100 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, | |
101 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, | |
102 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, | |
103 | {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, | |
fce8a7bb JM |
104 | {0} |
105 | }; | |
106 | MODULE_DEVICE_TABLE(pci, ntb_pci_tbl); | |
107 | ||
b775e85b DJ |
108 | static int is_ntb_xeon(struct ntb_device *ndev) |
109 | { | |
110 | switch (ndev->pdev->device) { | |
111 | case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: | |
112 | case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: | |
113 | case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: | |
114 | case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: | |
115 | case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: | |
116 | case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: | |
117 | case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: | |
118 | case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: | |
119 | case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: | |
120 | case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: | |
121 | case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: | |
122 | case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: | |
123 | return 1; | |
124 | default: | |
125 | return 0; | |
126 | } | |
127 | ||
128 | return 0; | |
129 | } | |
130 | ||
131 | static int is_ntb_atom(struct ntb_device *ndev) | |
132 | { | |
133 | switch (ndev->pdev->device) { | |
134 | case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: | |
135 | return 1; | |
136 | default: | |
137 | return 0; | |
138 | } | |
139 | ||
140 | return 0; | |
141 | } | |
142 | ||
069684e8 DJ |
143 | static void ntb_set_errata_flags(struct ntb_device *ndev) |
144 | { | |
145 | switch (ndev->pdev->device) { | |
146 | /* | |
147 | * this workaround applies to all platform up to IvyBridge | |
148 | * Haswell has splitbar support and use a different workaround | |
149 | */ | |
150 | case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: | |
151 | case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: | |
152 | case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: | |
153 | case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: | |
154 | case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: | |
155 | case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: | |
156 | case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: | |
157 | case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: | |
158 | case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: | |
159 | case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: | |
160 | case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: | |
161 | case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: | |
162 | ndev->wa_flags |= WA_SNB_ERR; | |
163 | break; | |
164 | } | |
165 | } | |
166 | ||
fce8a7bb JM |
167 | /** |
168 | * ntb_register_event_callback() - register event callback | |
169 | * @ndev: pointer to ntb_device instance | |
170 | * @func: callback function to register | |
171 | * | |
172 | * This function registers a callback for any HW driver events such as link | |
173 | * up/down, power management notices and etc. | |
174 | * | |
175 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
176 | */ | |
177 | int ntb_register_event_callback(struct ntb_device *ndev, | |
53ca4fea JM |
178 | void (*func)(void *handle, |
179 | enum ntb_hw_event event)) | |
fce8a7bb JM |
180 | { |
181 | if (ndev->event_cb) | |
182 | return -EINVAL; | |
183 | ||
184 | ndev->event_cb = func; | |
185 | ||
186 | return 0; | |
187 | } | |
188 | ||
189 | /** | |
190 | * ntb_unregister_event_callback() - unregisters the event callback | |
191 | * @ndev: pointer to ntb_device instance | |
192 | * | |
193 | * This function unregisters the existing callback from transport | |
194 | */ | |
195 | void ntb_unregister_event_callback(struct ntb_device *ndev) | |
196 | { | |
197 | ndev->event_cb = NULL; | |
198 | } | |
199 | ||
e8aeb60c JM |
200 | static void ntb_irq_work(unsigned long data) |
201 | { | |
202 | struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data; | |
203 | int rc; | |
204 | ||
205 | rc = db_cb->callback(db_cb->data, db_cb->db_num); | |
206 | if (rc) | |
207 | tasklet_schedule(&db_cb->irq_work); | |
208 | else { | |
209 | struct ntb_device *ndev = db_cb->ndev; | |
210 | unsigned long mask; | |
211 | ||
212 | mask = readw(ndev->reg_ofs.ldb_mask); | |
213 | clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
214 | writew(mask, ndev->reg_ofs.ldb_mask); | |
215 | } | |
216 | } | |
217 | ||
fce8a7bb JM |
218 | /** |
219 | * ntb_register_db_callback() - register a callback for doorbell interrupt | |
220 | * @ndev: pointer to ntb_device instance | |
221 | * @idx: doorbell index to register callback, zero based | |
f9a2cf89 | 222 | * @data: pointer to be returned to caller with every callback |
fce8a7bb JM |
223 | * @func: callback function to register |
224 | * | |
225 | * This function registers a callback function for the doorbell interrupt | |
226 | * on the primary side. The function will unmask the doorbell as well to | |
227 | * allow interrupt. | |
228 | * | |
229 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
230 | */ | |
231 | int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx, | |
e8aeb60c | 232 | void *data, int (*func)(void *data, int db_num)) |
fce8a7bb JM |
233 | { |
234 | unsigned long mask; | |
235 | ||
236 | if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) { | |
237 | dev_warn(&ndev->pdev->dev, "Invalid Index.\n"); | |
238 | return -EINVAL; | |
239 | } | |
240 | ||
241 | ndev->db_cb[idx].callback = func; | |
242 | ndev->db_cb[idx].data = data; | |
e8aeb60c JM |
243 | ndev->db_cb[idx].ndev = ndev; |
244 | ||
245 | tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, | |
246 | (unsigned long) &ndev->db_cb[idx]); | |
fce8a7bb JM |
247 | |
248 | /* unmask interrupt */ | |
49793889 | 249 | mask = readw(ndev->reg_ofs.ldb_mask); |
fce8a7bb | 250 | clear_bit(idx * ndev->bits_per_vector, &mask); |
49793889 | 251 | writew(mask, ndev->reg_ofs.ldb_mask); |
fce8a7bb JM |
252 | |
253 | return 0; | |
254 | } | |
255 | ||
256 | /** | |
257 | * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt | |
258 | * @ndev: pointer to ntb_device instance | |
259 | * @idx: doorbell index to register callback, zero based | |
260 | * | |
261 | * This function unregisters a callback function for the doorbell interrupt | |
262 | * on the primary side. The function will also mask the said doorbell. | |
263 | */ | |
264 | void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx) | |
265 | { | |
266 | unsigned long mask; | |
267 | ||
268 | if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) | |
269 | return; | |
270 | ||
49793889 | 271 | mask = readw(ndev->reg_ofs.ldb_mask); |
fce8a7bb | 272 | set_bit(idx * ndev->bits_per_vector, &mask); |
49793889 | 273 | writew(mask, ndev->reg_ofs.ldb_mask); |
fce8a7bb | 274 | |
e8aeb60c JM |
275 | tasklet_disable(&ndev->db_cb[idx].irq_work); |
276 | ||
fce8a7bb JM |
277 | ndev->db_cb[idx].callback = NULL; |
278 | } | |
279 | ||
280 | /** | |
281 | * ntb_find_transport() - find the transport pointer | |
282 | * @transport: pointer to pci device | |
283 | * | |
284 | * Given the pci device pointer, return the transport pointer passed in when | |
285 | * the transport attached when it was inited. | |
286 | * | |
287 | * RETURNS: pointer to transport. | |
288 | */ | |
289 | void *ntb_find_transport(struct pci_dev *pdev) | |
290 | { | |
291 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
292 | return ndev->ntb_transport; | |
293 | } | |
294 | ||
295 | /** | |
296 | * ntb_register_transport() - Register NTB transport with NTB HW driver | |
297 | * @transport: transport identifier | |
298 | * | |
299 | * This function allows a transport to reserve the hardware driver for | |
300 | * NTB usage. | |
301 | * | |
302 | * RETURNS: pointer to ntb_device, NULL on error. | |
303 | */ | |
304 | struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport) | |
305 | { | |
306 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
307 | ||
308 | if (ndev->ntb_transport) | |
309 | return NULL; | |
310 | ||
311 | ndev->ntb_transport = transport; | |
312 | return ndev; | |
313 | } | |
314 | ||
315 | /** | |
316 | * ntb_unregister_transport() - Unregister the transport with the NTB HW driver | |
317 | * @ndev - ntb_device of the transport to be freed | |
318 | * | |
319 | * This function unregisters the transport from the HW driver and performs any | |
320 | * necessary cleanups. | |
321 | */ | |
322 | void ntb_unregister_transport(struct ntb_device *ndev) | |
323 | { | |
324 | int i; | |
325 | ||
326 | if (!ndev->ntb_transport) | |
327 | return; | |
328 | ||
329 | for (i = 0; i < ndev->max_cbs; i++) | |
330 | ntb_unregister_db_callback(ndev, i); | |
331 | ||
332 | ntb_unregister_event_callback(ndev); | |
333 | ndev->ntb_transport = NULL; | |
334 | } | |
335 | ||
fce8a7bb JM |
336 | /** |
337 | * ntb_write_local_spad() - write to the secondary scratchpad register | |
338 | * @ndev: pointer to ntb_device instance | |
339 | * @idx: index to the scratchpad register, 0 based | |
340 | * @val: the data value to put into the register | |
341 | * | |
342 | * This function allows writing of a 32bit value to the indexed scratchpad | |
343 | * register. This writes over the data mirrored to the local scratchpad register | |
344 | * by the remote system. | |
345 | * | |
346 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
347 | */ | |
348 | int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
349 | { | |
350 | if (idx >= ndev->limits.max_spads) | |
351 | return -EINVAL; | |
352 | ||
353 | dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", | |
354 | val, idx); | |
355 | writel(val, ndev->reg_ofs.spad_read + idx * 4); | |
356 | ||
357 | return 0; | |
358 | } | |
359 | ||
360 | /** | |
361 | * ntb_read_local_spad() - read from the primary scratchpad register | |
362 | * @ndev: pointer to ntb_device instance | |
363 | * @idx: index to scratchpad register, 0 based | |
364 | * @val: pointer to 32bit integer for storing the register value | |
365 | * | |
366 | * This function allows reading of the 32bit scratchpad register on | |
367 | * the primary (internal) side. This allows the local system to read data | |
368 | * written and mirrored to the scratchpad register by the remote system. | |
369 | * | |
370 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
371 | */ | |
372 | int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
373 | { | |
374 | if (idx >= ndev->limits.max_spads) | |
375 | return -EINVAL; | |
376 | ||
377 | *val = readl(ndev->reg_ofs.spad_write + idx * 4); | |
378 | dev_dbg(&ndev->pdev->dev, | |
379 | "Reading %x from local scratch pad index %d\n", *val, idx); | |
380 | ||
381 | return 0; | |
382 | } | |
383 | ||
384 | /** | |
385 | * ntb_write_remote_spad() - write to the secondary scratchpad register | |
386 | * @ndev: pointer to ntb_device instance | |
387 | * @idx: index to the scratchpad register, 0 based | |
388 | * @val: the data value to put into the register | |
389 | * | |
390 | * This function allows writing of a 32bit value to the indexed scratchpad | |
391 | * register. The register resides on the secondary (external) side. This allows | |
392 | * the local system to write data to be mirrored to the remote systems | |
393 | * scratchpad register. | |
394 | * | |
395 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
396 | */ | |
397 | int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val) | |
398 | { | |
399 | if (idx >= ndev->limits.max_spads) | |
400 | return -EINVAL; | |
401 | ||
402 | dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", | |
403 | val, idx); | |
404 | writel(val, ndev->reg_ofs.spad_write + idx * 4); | |
405 | ||
406 | return 0; | |
407 | } | |
408 | ||
409 | /** | |
410 | * ntb_read_remote_spad() - read from the primary scratchpad register | |
411 | * @ndev: pointer to ntb_device instance | |
412 | * @idx: index to scratchpad register, 0 based | |
413 | * @val: pointer to 32bit integer for storing the register value | |
414 | * | |
415 | * This function allows reading of the 32bit scratchpad register on | |
416 | * the primary (internal) side. This alloows the local system to read the data | |
417 | * it wrote to be mirrored on the remote system. | |
418 | * | |
419 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
420 | */ | |
421 | int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val) | |
422 | { | |
423 | if (idx >= ndev->limits.max_spads) | |
424 | return -EINVAL; | |
425 | ||
426 | *val = readl(ndev->reg_ofs.spad_read + idx * 4); | |
427 | dev_dbg(&ndev->pdev->dev, | |
428 | "Reading %x from remote scratch pad index %d\n", *val, idx); | |
429 | ||
430 | return 0; | |
431 | } | |
432 | ||
282a2fee JM |
433 | /** |
434 | * ntb_get_mw_base() - get addr for the NTB memory window | |
435 | * @ndev: pointer to ntb_device instance | |
436 | * @mw: memory window number | |
437 | * | |
438 | * This function provides the base address of the memory window specified. | |
439 | * | |
440 | * RETURNS: address, or NULL on error. | |
441 | */ | |
442 | resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw) | |
443 | { | |
444 | if (mw >= ntb_max_mw(ndev)) | |
445 | return 0; | |
446 | ||
447 | return pci_resource_start(ndev->pdev, MW_TO_BAR(mw)); | |
448 | } | |
449 | ||
fce8a7bb JM |
450 | /** |
451 | * ntb_get_mw_vbase() - get virtual addr for the NTB memory window | |
452 | * @ndev: pointer to ntb_device instance | |
453 | * @mw: memory window number | |
454 | * | |
455 | * This function provides the base virtual address of the memory window | |
456 | * specified. | |
457 | * | |
458 | * RETURNS: pointer to virtual address, or NULL on error. | |
459 | */ | |
74465645 | 460 | void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw) |
fce8a7bb | 461 | { |
948d3a65 | 462 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
463 | return NULL; |
464 | ||
465 | return ndev->mw[mw].vbase; | |
466 | } | |
467 | ||
468 | /** | |
469 | * ntb_get_mw_size() - return size of NTB memory window | |
470 | * @ndev: pointer to ntb_device instance | |
471 | * @mw: memory window number | |
472 | * | |
473 | * This function provides the physical size of the memory window specified | |
474 | * | |
475 | * RETURNS: the size of the memory window or zero on error | |
476 | */ | |
ac477afb | 477 | u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw) |
fce8a7bb | 478 | { |
948d3a65 | 479 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
480 | return 0; |
481 | ||
482 | return ndev->mw[mw].bar_sz; | |
483 | } | |
484 | ||
485 | /** | |
486 | * ntb_set_mw_addr - set the memory window address | |
487 | * @ndev: pointer to ntb_device instance | |
488 | * @mw: memory window number | |
489 | * @addr: base address for data | |
490 | * | |
491 | * This function sets the base physical address of the memory window. This | |
492 | * memory address is where data from the remote system will be transfered into | |
493 | * or out of depending on how the transport is configured. | |
494 | */ | |
495 | void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr) | |
496 | { | |
948d3a65 | 497 | if (mw >= ntb_max_mw(ndev)) |
fce8a7bb JM |
498 | return; |
499 | ||
500 | dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, | |
501 | MW_TO_BAR(mw)); | |
502 | ||
503 | ndev->mw[mw].phys_addr = addr; | |
504 | ||
505 | switch (MW_TO_BAR(mw)) { | |
506 | case NTB_BAR_23: | |
49793889 | 507 | writeq(addr, ndev->reg_ofs.bar2_xlat); |
fce8a7bb | 508 | break; |
ab760a0c DJ |
509 | case NTB_BAR_4: |
510 | if (ndev->split_bar) | |
511 | writel(addr, ndev->reg_ofs.bar4_xlat); | |
512 | else | |
513 | writeq(addr, ndev->reg_ofs.bar4_xlat); | |
514 | break; | |
515 | case NTB_BAR_5: | |
516 | writel(addr, ndev->reg_ofs.bar5_xlat); | |
fce8a7bb JM |
517 | break; |
518 | } | |
519 | } | |
520 | ||
521 | /** | |
49793889 | 522 | * ntb_ring_doorbell() - Set the doorbell on the secondary/external side |
fce8a7bb JM |
523 | * @ndev: pointer to ntb_device instance |
524 | * @db: doorbell to ring | |
525 | * | |
526 | * This function allows triggering of a doorbell on the secondary/external | |
527 | * side that will initiate an interrupt on the remote host | |
528 | * | |
529 | * RETURNS: An appropriate -ERRNO error value on error, or zero for success. | |
530 | */ | |
49793889 | 531 | void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db) |
fce8a7bb JM |
532 | { |
533 | dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db); | |
534 | ||
535 | if (ndev->hw_type == BWD_HW) | |
49793889 | 536 | writeq((u64) 1 << db, ndev->reg_ofs.rdb); |
fce8a7bb JM |
537 | else |
538 | writew(((1 << ndev->bits_per_vector) - 1) << | |
49793889 | 539 | (db * ndev->bits_per_vector), ndev->reg_ofs.rdb); |
fce8a7bb JM |
540 | } |
541 | ||
113bf1c9 JM |
542 | static void bwd_recover_link(struct ntb_device *ndev) |
543 | { | |
544 | u32 status; | |
545 | ||
546 | /* Driver resets the NTB ModPhy lanes - magic! */ | |
547 | writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6); | |
548 | writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4); | |
549 | writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4); | |
550 | writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6); | |
551 | ||
552 | /* Driver waits 100ms to allow the NTB ModPhy to settle */ | |
553 | msleep(100); | |
554 | ||
555 | /* Clear AER Errors, write to clear */ | |
556 | status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET); | |
557 | dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status); | |
558 | status &= PCI_ERR_COR_REP_ROLL; | |
559 | writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET); | |
560 | ||
561 | /* Clear unexpected electrical idle event in LTSSM, write to clear */ | |
562 | status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); | |
563 | dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status); | |
564 | status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI; | |
565 | writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); | |
566 | ||
567 | /* Clear DeSkew Buffer error, write to clear */ | |
568 | status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET); | |
569 | dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status); | |
570 | status |= BWD_DESKEWSTS_DBERR; | |
571 | writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET); | |
572 | ||
573 | status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
574 | dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status); | |
575 | status &= BWD_IBIST_ERR_OFLOW; | |
576 | writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
577 | ||
578 | /* Releases the NTB state machine to allow the link to retrain */ | |
579 | status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
580 | dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status); | |
581 | status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT; | |
582 | writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
583 | } | |
584 | ||
fce8a7bb JM |
585 | static void ntb_link_event(struct ntb_device *ndev, int link_state) |
586 | { | |
587 | unsigned int event; | |
588 | ||
589 | if (ndev->link_status == link_state) | |
590 | return; | |
591 | ||
592 | if (link_state == NTB_LINK_UP) { | |
593 | u16 status; | |
594 | ||
595 | dev_info(&ndev->pdev->dev, "Link Up\n"); | |
596 | ndev->link_status = NTB_LINK_UP; | |
597 | event = NTB_EVENT_HW_LINK_UP; | |
598 | ||
b775e85b | 599 | if (is_ntb_atom(ndev) || |
ed6c24ed | 600 | ndev->conn_type == NTB_CONN_TRANSPARENT) |
fce8a7bb JM |
601 | status = readw(ndev->reg_ofs.lnk_stat); |
602 | else { | |
603 | int rc = pci_read_config_word(ndev->pdev, | |
604 | SNB_LINK_STATUS_OFFSET, | |
605 | &status); | |
606 | if (rc) | |
607 | return; | |
608 | } | |
113bf1c9 JM |
609 | |
610 | ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; | |
611 | ndev->link_speed = (status & NTB_LINK_SPEED_MASK); | |
fce8a7bb | 612 | dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", |
113bf1c9 | 613 | ndev->link_width, ndev->link_speed); |
fce8a7bb JM |
614 | } else { |
615 | dev_info(&ndev->pdev->dev, "Link Down\n"); | |
616 | ndev->link_status = NTB_LINK_DOWN; | |
617 | event = NTB_EVENT_HW_LINK_DOWN; | |
113bf1c9 | 618 | /* Don't modify link width/speed, we need it in link recovery */ |
fce8a7bb JM |
619 | } |
620 | ||
621 | /* notify the upper layer if we have an event change */ | |
622 | if (ndev->event_cb) | |
623 | ndev->event_cb(ndev->ntb_transport, event); | |
624 | } | |
625 | ||
626 | static int ntb_link_status(struct ntb_device *ndev) | |
627 | { | |
628 | int link_state; | |
629 | ||
b775e85b | 630 | if (is_ntb_atom(ndev)) { |
fce8a7bb JM |
631 | u32 ntb_cntl; |
632 | ||
633 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
634 | if (ntb_cntl & BWD_CNTL_LINK_DOWN) | |
635 | link_state = NTB_LINK_DOWN; | |
636 | else | |
637 | link_state = NTB_LINK_UP; | |
638 | } else { | |
639 | u16 status; | |
640 | int rc; | |
641 | ||
642 | rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, | |
643 | &status); | |
644 | if (rc) | |
645 | return rc; | |
646 | ||
647 | if (status & NTB_LINK_STATUS_ACTIVE) | |
648 | link_state = NTB_LINK_UP; | |
649 | else | |
650 | link_state = NTB_LINK_DOWN; | |
651 | } | |
652 | ||
653 | ntb_link_event(ndev, link_state); | |
654 | ||
655 | return 0; | |
656 | } | |
657 | ||
113bf1c9 JM |
658 | static void bwd_link_recovery(struct work_struct *work) |
659 | { | |
660 | struct ntb_device *ndev = container_of(work, struct ntb_device, | |
661 | lr_timer.work); | |
662 | u32 status32; | |
663 | ||
664 | bwd_recover_link(ndev); | |
665 | /* There is a potential race between the 2 NTB devices recovering at the | |
666 | * same time. If the times are the same, the link will not recover and | |
667 | * the driver will be stuck in this loop forever. Add a random interval | |
668 | * to the recovery time to prevent this race. | |
669 | */ | |
670 | msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME); | |
671 | ||
672 | status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); | |
673 | if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) | |
674 | goto retry; | |
675 | ||
676 | status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); | |
677 | if (status32 & BWD_IBIST_ERR_OFLOW) | |
678 | goto retry; | |
679 | ||
680 | status32 = readl(ndev->reg_ofs.lnk_cntl); | |
681 | if (!(status32 & BWD_CNTL_LINK_DOWN)) { | |
682 | unsigned char speed, width; | |
683 | u16 status16; | |
684 | ||
685 | status16 = readw(ndev->reg_ofs.lnk_stat); | |
686 | width = (status16 & NTB_LINK_WIDTH_MASK) >> 4; | |
687 | speed = (status16 & NTB_LINK_SPEED_MASK); | |
688 | if (ndev->link_width != width || ndev->link_speed != speed) | |
689 | goto retry; | |
690 | } | |
691 | ||
692 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
693 | return; | |
694 | ||
695 | retry: | |
696 | schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT); | |
697 | } | |
698 | ||
fce8a7bb JM |
699 | /* BWD doesn't have link status interrupt, poll on that platform */ |
700 | static void bwd_link_poll(struct work_struct *work) | |
701 | { | |
702 | struct ntb_device *ndev = container_of(work, struct ntb_device, | |
703 | hb_timer.work); | |
704 | unsigned long ts = jiffies; | |
705 | ||
706 | /* If we haven't gotten an interrupt in a while, check the BWD link | |
707 | * status bit | |
708 | */ | |
709 | if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { | |
710 | int rc = ntb_link_status(ndev); | |
711 | if (rc) | |
712 | dev_err(&ndev->pdev->dev, | |
713 | "Error determining link status\n"); | |
113bf1c9 JM |
714 | |
715 | /* Check to see if a link error is the cause of the link down */ | |
716 | if (ndev->link_status == NTB_LINK_DOWN) { | |
717 | u32 status32 = readl(ndev->reg_base + | |
718 | BWD_LTSSMSTATEJMP_OFFSET); | |
719 | if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) { | |
720 | schedule_delayed_work(&ndev->lr_timer, 0); | |
721 | return; | |
722 | } | |
723 | } | |
fce8a7bb JM |
724 | } |
725 | ||
726 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); | |
727 | } | |
728 | ||
729 | static int ntb_xeon_setup(struct ntb_device *ndev) | |
730 | { | |
1db97f25 | 731 | switch (ndev->conn_type) { |
ed6c24ed | 732 | case NTB_CONN_B2B: |
ed6c24ed JM |
733 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; |
734 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; | |
735 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; | |
736 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; | |
737 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; | |
ab760a0c DJ |
738 | if (ndev->split_bar) |
739 | ndev->reg_ofs.bar5_xlat = | |
740 | ndev->reg_base + SNB_SBAR5XLAT_OFFSET; | |
ed6c24ed JM |
741 | ndev->limits.max_spads = SNB_MAX_B2B_SPADS; |
742 | ||
743 | /* There is a Xeon hardware errata related to writes to | |
744 | * SDOORBELL or B2BDOORBELL in conjunction with inbound access | |
745 | * to NTB MMIO Space, which may hang the system. To workaround | |
746 | * this use the second memory window to access the interrupt and | |
747 | * scratch pad registers on the remote system. | |
948d3a65 | 748 | */ |
069684e8 | 749 | if (ndev->wa_flags & WA_SNB_ERR) { |
ab760a0c | 750 | if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz) |
ed6c24ed JM |
751 | return -EINVAL; |
752 | ||
c529aa30 | 753 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ab760a0c DJ |
754 | ndev->reg_ofs.spad_write = |
755 | ndev->mw[ndev->limits.max_mw - 1].vbase + | |
756 | SNB_SPAD_OFFSET; | |
757 | ndev->reg_ofs.rdb = | |
758 | ndev->mw[ndev->limits.max_mw - 1].vbase + | |
759 | SNB_PDOORBELL_OFFSET; | |
ed6c24ed JM |
760 | |
761 | /* Set the Limit register to 4k, the minimum size, to | |
762 | * prevent an illegal access | |
763 | */ | |
764 | writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base + | |
765 | SNB_PBAR4LMT_OFFSET); | |
58b88920 JM |
766 | /* HW errata on the Limit registers. They can only be |
767 | * written when the base register is 4GB aligned and | |
53ca4fea JM |
768 | * < 32bit. This should already be the case based on |
769 | * the driver defaults, but write the Limit registers | |
770 | * first just in case. | |
58b88920 | 771 | */ |
c529aa30 | 772 | |
ab760a0c DJ |
773 | ndev->limits.max_mw = SNB_ERRATA_MAX_MW; |
774 | } else { | |
c529aa30 JM |
775 | /* HW Errata on bit 14 of b2bdoorbell register. Writes |
776 | * will not be mirrored to the remote system. Shrink | |
777 | * the number of bits by one, since bit 14 is the last | |
778 | * bit. | |
779 | */ | |
780 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1; | |
ed6c24ed JM |
781 | ndev->reg_ofs.spad_write = ndev->reg_base + |
782 | SNB_B2B_SPAD_OFFSET; | |
783 | ndev->reg_ofs.rdb = ndev->reg_base + | |
784 | SNB_B2B_DOORBELL_OFFSET; | |
785 | ||
786 | /* Disable the Limit register, just incase it is set to | |
ab760a0c DJ |
787 | * something silly. A 64bit write should handle it |
788 | * regardless of whether it has a split BAR or not. | |
ed6c24ed JM |
789 | */ |
790 | writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET); | |
58b88920 JM |
791 | /* HW errata on the Limit registers. They can only be |
792 | * written when the base register is 4GB aligned and | |
53ca4fea JM |
793 | * < 32bit. This should already be the case based on |
794 | * the driver defaults, but write the Limit registers | |
795 | * first just in case. | |
58b88920 | 796 | */ |
ab760a0c DJ |
797 | if (ndev->split_bar) |
798 | ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; | |
799 | else | |
800 | ndev->limits.max_mw = SNB_MAX_MW; | |
ed6c24ed | 801 | } |
948d3a65 | 802 | |
ed6c24ed JM |
803 | /* The Xeon errata workaround requires setting SBAR Base |
804 | * addresses to known values, so that the PBAR XLAT can be | |
805 | * pointed at SBAR0 of the remote system. | |
948d3a65 | 806 | */ |
ed6c24ed JM |
807 | if (ndev->dev_type == NTB_DEV_USD) { |
808 | writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + | |
809 | SNB_PBAR2XLAT_OFFSET); | |
069684e8 | 810 | if (ndev->wa_flags & WA_SNB_ERR) |
ed6c24ed JM |
811 | writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + |
812 | SNB_PBAR4XLAT_OFFSET); | |
813 | else { | |
ab760a0c DJ |
814 | if (ndev->split_bar) { |
815 | writel(SNB_MBAR4_DSD_ADDR, | |
816 | ndev->reg_base + | |
817 | SNB_PBAR4XLAT_OFFSET); | |
818 | writel(SNB_MBAR5_DSD_ADDR, | |
819 | ndev->reg_base + | |
820 | SNB_PBAR5XLAT_OFFSET); | |
821 | } else | |
822 | writeq(SNB_MBAR4_DSD_ADDR, | |
823 | ndev->reg_base + | |
824 | SNB_PBAR4XLAT_OFFSET); | |
825 | ||
ed6c24ed JM |
826 | /* B2B_XLAT_OFFSET is a 64bit register, but can |
827 | * only take 32bit writes | |
828 | */ | |
829 | writel(SNB_MBAR01_DSD_ADDR & 0xffffffff, | |
830 | ndev->reg_base + SNB_B2B_XLAT_OFFSETL); | |
831 | writel(SNB_MBAR01_DSD_ADDR >> 32, | |
832 | ndev->reg_base + SNB_B2B_XLAT_OFFSETU); | |
833 | } | |
948d3a65 | 834 | |
ed6c24ed JM |
835 | writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + |
836 | SNB_SBAR0BASE_OFFSET); | |
837 | writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + | |
838 | SNB_SBAR2BASE_OFFSET); | |
ab760a0c DJ |
839 | if (ndev->split_bar) { |
840 | writel(SNB_MBAR4_USD_ADDR, ndev->reg_base + | |
841 | SNB_SBAR4BASE_OFFSET); | |
842 | writel(SNB_MBAR5_USD_ADDR, ndev->reg_base + | |
843 | SNB_SBAR5BASE_OFFSET); | |
844 | } else | |
845 | writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base + | |
846 | SNB_SBAR4BASE_OFFSET); | |
ed6c24ed JM |
847 | } else { |
848 | writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base + | |
849 | SNB_PBAR2XLAT_OFFSET); | |
069684e8 | 850 | if (ndev->wa_flags & WA_SNB_ERR) |
ed6c24ed JM |
851 | writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base + |
852 | SNB_PBAR4XLAT_OFFSET); | |
853 | else { | |
ab760a0c DJ |
854 | if (ndev->split_bar) { |
855 | writel(SNB_MBAR4_USD_ADDR, | |
856 | ndev->reg_base + | |
857 | SNB_PBAR4XLAT_OFFSET); | |
858 | writel(SNB_MBAR5_USD_ADDR, | |
859 | ndev->reg_base + | |
860 | SNB_PBAR5XLAT_OFFSET); | |
861 | } else | |
862 | writeq(SNB_MBAR4_USD_ADDR, | |
863 | ndev->reg_base + | |
864 | SNB_PBAR4XLAT_OFFSET); | |
865 | ||
866 | /* | |
867 | * B2B_XLAT_OFFSET is a 64bit register, but can | |
ed6c24ed JM |
868 | * only take 32bit writes |
869 | */ | |
c8eee379 | 870 | writel(SNB_MBAR01_USD_ADDR & 0xffffffff, |
ed6c24ed JM |
871 | ndev->reg_base + SNB_B2B_XLAT_OFFSETL); |
872 | writel(SNB_MBAR01_USD_ADDR >> 32, | |
873 | ndev->reg_base + SNB_B2B_XLAT_OFFSETU); | |
874 | } | |
948d3a65 | 875 | writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base + |
ed6c24ed JM |
876 | SNB_SBAR0BASE_OFFSET); |
877 | writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base + | |
878 | SNB_SBAR2BASE_OFFSET); | |
ab760a0c DJ |
879 | if (ndev->split_bar) { |
880 | writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base + | |
881 | SNB_SBAR4BASE_OFFSET); | |
882 | writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base + | |
883 | SNB_SBAR5BASE_OFFSET); | |
884 | } else | |
885 | writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base + | |
886 | SNB_SBAR4BASE_OFFSET); | |
887 | ||
948d3a65 | 888 | } |
ed6c24ed JM |
889 | break; |
890 | case NTB_CONN_RP: | |
069684e8 | 891 | if (ndev->wa_flags & WA_SNB_ERR) { |
53ca4fea | 892 | dev_err(&ndev->pdev->dev, |
069684e8 | 893 | "NTB-RP disabled due to hardware errata.\n"); |
ed6c24ed | 894 | return -EINVAL; |
948d3a65 | 895 | } |
ed6c24ed JM |
896 | |
897 | /* Scratch pads need to have exclusive access from the primary | |
898 | * or secondary side. Halve the num spads so that each side can | |
899 | * have an equal amount. | |
900 | */ | |
901 | ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; | |
c529aa30 | 902 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ed6c24ed JM |
903 | /* Note: The SDOORBELL is the cause of the errata. You REALLY |
904 | * don't want to touch it. | |
905 | */ | |
906 | ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET; | |
907 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; | |
908 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; | |
909 | /* Offset the start of the spads to correspond to whether it is | |
910 | * primary or secondary | |
911 | */ | |
912 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET + | |
913 | ndev->limits.max_spads * 4; | |
914 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; | |
915 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; | |
916 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET; | |
ab760a0c DJ |
917 | if (ndev->split_bar) { |
918 | ndev->reg_ofs.bar5_xlat = | |
919 | ndev->reg_base + SNB_SBAR5XLAT_OFFSET; | |
920 | ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; | |
921 | } else | |
922 | ndev->limits.max_mw = SNB_MAX_MW; | |
ed6c24ed JM |
923 | break; |
924 | case NTB_CONN_TRANSPARENT: | |
069684e8 DJ |
925 | if (ndev->wa_flags & WA_SNB_ERR) { |
926 | dev_err(&ndev->pdev->dev, | |
927 | "NTB-TRANSPARENT disabled due to hardware errata.\n"); | |
928 | return -EINVAL; | |
929 | } | |
930 | ||
ed6c24ed JM |
931 | /* Scratch pads need to have exclusive access from the primary |
932 | * or secondary side. Halve the num spads so that each side can | |
933 | * have an equal amount. | |
934 | */ | |
935 | ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; | |
c529aa30 | 936 | ndev->limits.max_db_bits = SNB_MAX_DB_BITS; |
ed6c24ed JM |
937 | ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET; |
938 | ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET; | |
939 | ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET; | |
940 | ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET; | |
941 | /* Offset the start of the spads to correspond to whether it is | |
942 | * primary or secondary | |
943 | */ | |
944 | ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET + | |
945 | ndev->limits.max_spads * 4; | |
946 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET; | |
947 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET; | |
948 | ||
ab760a0c DJ |
949 | if (ndev->split_bar) { |
950 | ndev->reg_ofs.bar5_xlat = | |
951 | ndev->reg_base + SNB_PBAR5XLAT_OFFSET; | |
952 | ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; | |
953 | } else | |
954 | ndev->limits.max_mw = SNB_MAX_MW; | |
ed6c24ed JM |
955 | break; |
956 | default: | |
1db97f25 DJ |
957 | /* |
958 | * we should never hit this. the detect function should've | |
959 | * take cared of everything. | |
ed6c24ed | 960 | */ |
ed6c24ed | 961 | return -EINVAL; |
fce8a7bb JM |
962 | } |
963 | ||
ed6c24ed JM |
964 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; |
965 | ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; | |
966 | ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; | |
967 | ||
fce8a7bb JM |
968 | ndev->limits.msix_cnt = SNB_MSIX_CNT; |
969 | ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; | |
970 | ||
971 | return 0; | |
972 | } | |
973 | ||
974 | static int ntb_bwd_setup(struct ntb_device *ndev) | |
975 | { | |
976 | int rc; | |
977 | u32 val; | |
978 | ||
979 | ndev->hw_type = BWD_HW; | |
980 | ||
981 | rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); | |
982 | if (rc) | |
983 | return rc; | |
984 | ||
985 | switch ((val & BWD_PPD_CONN_TYPE) >> 8) { | |
986 | case NTB_CONN_B2B: | |
987 | ndev->conn_type = NTB_CONN_B2B; | |
988 | break; | |
989 | case NTB_CONN_RP: | |
990 | default: | |
b1ef0043 | 991 | dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); |
fce8a7bb JM |
992 | return -EINVAL; |
993 | } | |
994 | ||
995 | if (val & BWD_PPD_DEV_TYPE) | |
996 | ndev->dev_type = NTB_DEV_DSD; | |
997 | else | |
998 | ndev->dev_type = NTB_DEV_USD; | |
999 | ||
1000 | /* Initiate PCI-E link training */ | |
1001 | rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET, | |
1002 | val | BWD_PPD_INIT_LINK); | |
1003 | if (rc) | |
1004 | return rc; | |
1005 | ||
49793889 JM |
1006 | ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET; |
1007 | ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; | |
b1ef0043 | 1008 | ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; |
49793889 JM |
1009 | ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; |
1010 | ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; | |
fce8a7bb JM |
1011 | ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; |
1012 | ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; | |
1013 | ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; | |
b1ef0043 | 1014 | ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; |
fce8a7bb | 1015 | ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; |
948d3a65 | 1016 | ndev->limits.max_mw = BWD_MAX_MW; |
b1ef0043 | 1017 | ndev->limits.max_spads = BWD_MAX_SPADS; |
fce8a7bb JM |
1018 | ndev->limits.max_db_bits = BWD_MAX_DB_BITS; |
1019 | ndev->limits.msix_cnt = BWD_MSIX_CNT; | |
1020 | ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; | |
1021 | ||
1022 | /* Since bwd doesn't have a link interrupt, setup a poll timer */ | |
1023 | INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); | |
113bf1c9 | 1024 | INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery); |
fce8a7bb JM |
1025 | schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); |
1026 | ||
1027 | return 0; | |
1028 | } | |
1029 | ||
78a61ab7 | 1030 | static int ntb_device_setup(struct ntb_device *ndev) |
fce8a7bb JM |
1031 | { |
1032 | int rc; | |
1033 | ||
b775e85b | 1034 | if (is_ntb_xeon(ndev)) |
fce8a7bb | 1035 | rc = ntb_xeon_setup(ndev); |
b775e85b | 1036 | else if (is_ntb_atom(ndev)) |
fce8a7bb | 1037 | rc = ntb_bwd_setup(ndev); |
b775e85b | 1038 | else |
fce8a7bb | 1039 | rc = -ENODEV; |
fce8a7bb | 1040 | |
3b12a0d1 JM |
1041 | if (rc) |
1042 | return rc; | |
1043 | ||
ed6c24ed JM |
1044 | if (ndev->conn_type == NTB_CONN_B2B) |
1045 | /* Enable Bus Master and Memory Space on the secondary side */ | |
1046 | writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, | |
1047 | ndev->reg_ofs.spci_cmd); | |
fce8a7bb | 1048 | |
3b12a0d1 | 1049 | return 0; |
fce8a7bb JM |
1050 | } |
1051 | ||
1052 | static void ntb_device_free(struct ntb_device *ndev) | |
1053 | { | |
b775e85b | 1054 | if (is_ntb_atom(ndev)) { |
fce8a7bb | 1055 | cancel_delayed_work_sync(&ndev->hb_timer); |
113bf1c9 JM |
1056 | cancel_delayed_work_sync(&ndev->lr_timer); |
1057 | } | |
fce8a7bb JM |
1058 | } |
1059 | ||
1060 | static irqreturn_t bwd_callback_msix_irq(int irq, void *data) | |
1061 | { | |
1062 | struct ntb_db_cb *db_cb = data; | |
1063 | struct ntb_device *ndev = db_cb->ndev; | |
e8aeb60c | 1064 | unsigned long mask; |
fce8a7bb JM |
1065 | |
1066 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
1067 | db_cb->db_num); | |
1068 | ||
e8aeb60c JM |
1069 | mask = readw(ndev->reg_ofs.ldb_mask); |
1070 | set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
1071 | writew(mask, ndev->reg_ofs.ldb_mask); | |
1072 | ||
1073 | tasklet_schedule(&db_cb->irq_work); | |
fce8a7bb JM |
1074 | |
1075 | /* No need to check for the specific HB irq, any interrupt means | |
1076 | * we're connected. | |
1077 | */ | |
1078 | ndev->last_ts = jiffies; | |
1079 | ||
49793889 | 1080 | writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb); |
fce8a7bb JM |
1081 | |
1082 | return IRQ_HANDLED; | |
1083 | } | |
1084 | ||
1085 | static irqreturn_t xeon_callback_msix_irq(int irq, void *data) | |
1086 | { | |
1087 | struct ntb_db_cb *db_cb = data; | |
1088 | struct ntb_device *ndev = db_cb->ndev; | |
e8aeb60c | 1089 | unsigned long mask; |
fce8a7bb JM |
1090 | |
1091 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, | |
1092 | db_cb->db_num); | |
1093 | ||
e8aeb60c JM |
1094 | mask = readw(ndev->reg_ofs.ldb_mask); |
1095 | set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); | |
1096 | writew(mask, ndev->reg_ofs.ldb_mask); | |
1097 | ||
1098 | tasklet_schedule(&db_cb->irq_work); | |
fce8a7bb JM |
1099 | |
1100 | /* On Sandybridge, there are 16 bits in the interrupt register | |
1101 | * but only 4 vectors. So, 5 bits are assigned to the first 3 | |
1102 | * vectors, with the 4th having a single bit for link | |
1103 | * interrupts. | |
1104 | */ | |
1105 | writew(((1 << ndev->bits_per_vector) - 1) << | |
49793889 | 1106 | (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb); |
fce8a7bb JM |
1107 | |
1108 | return IRQ_HANDLED; | |
1109 | } | |
1110 | ||
1111 | /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */ | |
1112 | static irqreturn_t xeon_event_msix_irq(int irq, void *dev) | |
1113 | { | |
1114 | struct ntb_device *ndev = dev; | |
1115 | int rc; | |
1116 | ||
1117 | dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq); | |
1118 | ||
1119 | rc = ntb_link_status(ndev); | |
1120 | if (rc) | |
1121 | dev_err(&ndev->pdev->dev, "Error determining link status\n"); | |
1122 | ||
1123 | /* bit 15 is always the link bit */ | |
c529aa30 | 1124 | writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); |
fce8a7bb JM |
1125 | |
1126 | return IRQ_HANDLED; | |
1127 | } | |
1128 | ||
1129 | static irqreturn_t ntb_interrupt(int irq, void *dev) | |
1130 | { | |
1131 | struct ntb_device *ndev = dev; | |
1132 | unsigned int i = 0; | |
1133 | ||
b775e85b | 1134 | if (is_ntb_atom(ndev)) { |
49793889 | 1135 | u64 ldb = readq(ndev->reg_ofs.ldb); |
fce8a7bb | 1136 | |
49793889 | 1137 | dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb); |
fce8a7bb | 1138 | |
49793889 JM |
1139 | while (ldb) { |
1140 | i = __ffs(ldb); | |
1141 | ldb &= ldb - 1; | |
fce8a7bb JM |
1142 | bwd_callback_msix_irq(irq, &ndev->db_cb[i]); |
1143 | } | |
1144 | } else { | |
49793889 | 1145 | u16 ldb = readw(ndev->reg_ofs.ldb); |
fce8a7bb | 1146 | |
49793889 | 1147 | dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb); |
fce8a7bb | 1148 | |
49793889 | 1149 | if (ldb & SNB_DB_HW_LINK) { |
fce8a7bb | 1150 | xeon_event_msix_irq(irq, dev); |
49793889 | 1151 | ldb &= ~SNB_DB_HW_LINK; |
fce8a7bb JM |
1152 | } |
1153 | ||
49793889 JM |
1154 | while (ldb) { |
1155 | i = __ffs(ldb); | |
1156 | ldb &= ldb - 1; | |
fce8a7bb JM |
1157 | xeon_callback_msix_irq(irq, &ndev->db_cb[i]); |
1158 | } | |
1159 | } | |
1160 | ||
1161 | return IRQ_HANDLED; | |
1162 | } | |
1163 | ||
53a788a7 AG |
1164 | static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries) |
1165 | { | |
1166 | struct pci_dev *pdev = ndev->pdev; | |
1167 | struct msix_entry *msix; | |
1168 | int rc, i; | |
1169 | ||
1170 | if (msix_entries < ndev->limits.msix_cnt) | |
1171 | return -ENOSPC; | |
1172 | ||
f220baad | 1173 | rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries); |
53a788a7 AG |
1174 | if (rc < 0) |
1175 | return rc; | |
53a788a7 AG |
1176 | |
1177 | for (i = 0; i < msix_entries; i++) { | |
1178 | msix = &ndev->msix_entries[i]; | |
1179 | WARN_ON(!msix->vector); | |
1180 | ||
1181 | if (i == msix_entries - 1) { | |
1182 | rc = request_irq(msix->vector, | |
1183 | xeon_event_msix_irq, 0, | |
1184 | "ntb-event-msix", ndev); | |
1185 | if (rc) | |
1186 | goto err; | |
1187 | } else { | |
1188 | rc = request_irq(msix->vector, | |
1189 | xeon_callback_msix_irq, 0, | |
1190 | "ntb-callback-msix", | |
1191 | &ndev->db_cb[i]); | |
1192 | if (rc) | |
1193 | goto err; | |
1194 | } | |
1195 | } | |
1196 | ||
1197 | ndev->num_msix = msix_entries; | |
1198 | ndev->max_cbs = msix_entries - 1; | |
1199 | ||
1200 | return 0; | |
1201 | ||
1202 | err: | |
1203 | while (--i >= 0) { | |
1204 | /* Code never reaches here for entry nr 'ndev->num_msix - 1' */ | |
1205 | msix = &ndev->msix_entries[i]; | |
1206 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1207 | } | |
1208 | ||
1209 | pci_disable_msix(pdev); | |
1210 | ndev->num_msix = 0; | |
1211 | ||
1212 | return rc; | |
1213 | } | |
1214 | ||
1215 | static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries) | |
fce8a7bb JM |
1216 | { |
1217 | struct pci_dev *pdev = ndev->pdev; | |
1218 | struct msix_entry *msix; | |
53a788a7 AG |
1219 | int rc, i; |
1220 | ||
f220baad AG |
1221 | msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries, |
1222 | 1, msix_entries); | |
1223 | if (msix_entries < 0) | |
1224 | return msix_entries; | |
53a788a7 AG |
1225 | |
1226 | for (i = 0; i < msix_entries; i++) { | |
1227 | msix = &ndev->msix_entries[i]; | |
1228 | WARN_ON(!msix->vector); | |
1229 | ||
1230 | rc = request_irq(msix->vector, bwd_callback_msix_irq, 0, | |
1231 | "ntb-callback-msix", &ndev->db_cb[i]); | |
1232 | if (rc) | |
1233 | goto err; | |
1234 | } | |
1235 | ||
1236 | ndev->num_msix = msix_entries; | |
1237 | ndev->max_cbs = msix_entries; | |
1238 | ||
1239 | return 0; | |
1240 | ||
1241 | err: | |
1242 | while (--i >= 0) | |
1243 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1244 | ||
1245 | pci_disable_msix(pdev); | |
1246 | ndev->num_msix = 0; | |
1247 | ||
1248 | return rc; | |
1249 | } | |
1250 | ||
1251 | static int ntb_setup_msix(struct ntb_device *ndev) | |
1252 | { | |
1253 | struct pci_dev *pdev = ndev->pdev; | |
fce8a7bb | 1254 | int msix_entries; |
73f47cad | 1255 | int rc, i; |
fce8a7bb | 1256 | |
77733513 AG |
1257 | msix_entries = pci_msix_vec_count(pdev); |
1258 | if (msix_entries < 0) { | |
1259 | rc = msix_entries; | |
fce8a7bb | 1260 | goto err; |
77733513 | 1261 | } else if (msix_entries > ndev->limits.msix_cnt) { |
fce8a7bb JM |
1262 | rc = -EINVAL; |
1263 | goto err; | |
1264 | } | |
1265 | ||
1266 | ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, | |
1267 | GFP_KERNEL); | |
1268 | if (!ndev->msix_entries) { | |
1269 | rc = -ENOMEM; | |
1270 | goto err; | |
1271 | } | |
1272 | ||
1273 | for (i = 0; i < msix_entries; i++) | |
1274 | ndev->msix_entries[i].entry = i; | |
1275 | ||
b775e85b | 1276 | if (is_ntb_atom(ndev)) |
53a788a7 | 1277 | rc = ntb_setup_bwd_msix(ndev, msix_entries); |
fce8a7bb | 1278 | else |
53a788a7 AG |
1279 | rc = ntb_setup_snb_msix(ndev, msix_entries); |
1280 | if (rc) | |
1281 | goto err1; | |
fce8a7bb JM |
1282 | |
1283 | return 0; | |
1284 | ||
fce8a7bb JM |
1285 | err1: |
1286 | kfree(ndev->msix_entries); | |
fce8a7bb | 1287 | err: |
53a788a7 | 1288 | dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n"); |
fce8a7bb JM |
1289 | return rc; |
1290 | } | |
1291 | ||
1292 | static int ntb_setup_msi(struct ntb_device *ndev) | |
1293 | { | |
1294 | struct pci_dev *pdev = ndev->pdev; | |
1295 | int rc; | |
1296 | ||
1297 | rc = pci_enable_msi(pdev); | |
1298 | if (rc) | |
1299 | return rc; | |
1300 | ||
1301 | rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); | |
1302 | if (rc) { | |
1303 | pci_disable_msi(pdev); | |
1304 | dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); | |
1305 | return rc; | |
1306 | } | |
1307 | ||
1308 | return 0; | |
1309 | } | |
1310 | ||
1311 | static int ntb_setup_intx(struct ntb_device *ndev) | |
1312 | { | |
1313 | struct pci_dev *pdev = ndev->pdev; | |
1314 | int rc; | |
1315 | ||
fce8a7bb JM |
1316 | /* Verify intx is enabled */ |
1317 | pci_intx(pdev, 1); | |
1318 | ||
1319 | rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx", | |
1320 | ndev); | |
1321 | if (rc) | |
1322 | return rc; | |
1323 | ||
1324 | return 0; | |
1325 | } | |
1326 | ||
78a61ab7 | 1327 | static int ntb_setup_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
1328 | { |
1329 | int rc; | |
1330 | ||
1331 | /* On BWD, disable all interrupts. On SNB, disable all but Link | |
1332 | * Interrupt. The rest will be unmasked as callbacks are registered. | |
1333 | */ | |
b775e85b | 1334 | if (is_ntb_atom(ndev)) |
49793889 | 1335 | writeq(~0, ndev->reg_ofs.ldb_mask); |
c529aa30 JM |
1336 | else { |
1337 | u16 var = 1 << SNB_LINK_DB; | |
1338 | writew(~var, ndev->reg_ofs.ldb_mask); | |
1339 | } | |
fce8a7bb JM |
1340 | |
1341 | rc = ntb_setup_msix(ndev); | |
1342 | if (!rc) | |
1343 | goto done; | |
1344 | ||
1345 | ndev->bits_per_vector = 1; | |
1346 | ndev->max_cbs = ndev->limits.max_db_bits; | |
1347 | ||
1348 | rc = ntb_setup_msi(ndev); | |
1349 | if (!rc) | |
1350 | goto done; | |
1351 | ||
1352 | rc = ntb_setup_intx(ndev); | |
1353 | if (rc) { | |
1354 | dev_err(&ndev->pdev->dev, "no usable interrupts\n"); | |
1355 | return rc; | |
1356 | } | |
1357 | ||
1358 | done: | |
1359 | return 0; | |
1360 | } | |
1361 | ||
78a61ab7 | 1362 | static void ntb_free_interrupts(struct ntb_device *ndev) |
fce8a7bb JM |
1363 | { |
1364 | struct pci_dev *pdev = ndev->pdev; | |
1365 | ||
1366 | /* mask interrupts */ | |
b775e85b | 1367 | if (is_ntb_atom(ndev)) |
49793889 | 1368 | writeq(~0, ndev->reg_ofs.ldb_mask); |
fce8a7bb | 1369 | else |
49793889 | 1370 | writew(~0, ndev->reg_ofs.ldb_mask); |
fce8a7bb JM |
1371 | |
1372 | if (ndev->num_msix) { | |
1373 | struct msix_entry *msix; | |
1374 | u32 i; | |
1375 | ||
1376 | for (i = 0; i < ndev->num_msix; i++) { | |
1377 | msix = &ndev->msix_entries[i]; | |
b775e85b | 1378 | if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1) |
fce8a7bb JM |
1379 | free_irq(msix->vector, ndev); |
1380 | else | |
1381 | free_irq(msix->vector, &ndev->db_cb[i]); | |
1382 | } | |
1383 | pci_disable_msix(pdev); | |
717e8e8b | 1384 | kfree(ndev->msix_entries); |
fce8a7bb JM |
1385 | } else { |
1386 | free_irq(pdev->irq, ndev); | |
1387 | ||
1388 | if (pci_dev_msi_enabled(pdev)) | |
1389 | pci_disable_msi(pdev); | |
1390 | } | |
1391 | } | |
1392 | ||
78a61ab7 | 1393 | static int ntb_create_callbacks(struct ntb_device *ndev) |
fce8a7bb JM |
1394 | { |
1395 | int i; | |
1396 | ||
f9a2cf89 | 1397 | /* Chicken-egg issue. We won't know how many callbacks are necessary |
fce8a7bb | 1398 | * until we see how many MSI-X vectors we get, but these pointers need |
f9a2cf89 | 1399 | * to be passed into the MSI-X register function. So, we allocate the |
fce8a7bb JM |
1400 | * max, knowing that they might not all be used, to work around this. |
1401 | */ | |
1402 | ndev->db_cb = kcalloc(ndev->limits.max_db_bits, | |
1403 | sizeof(struct ntb_db_cb), | |
1404 | GFP_KERNEL); | |
1405 | if (!ndev->db_cb) | |
1406 | return -ENOMEM; | |
1407 | ||
1408 | for (i = 0; i < ndev->limits.max_db_bits; i++) { | |
1409 | ndev->db_cb[i].db_num = i; | |
1410 | ndev->db_cb[i].ndev = ndev; | |
1411 | } | |
1412 | ||
1413 | return 0; | |
1414 | } | |
1415 | ||
1416 | static void ntb_free_callbacks(struct ntb_device *ndev) | |
1417 | { | |
1418 | int i; | |
1419 | ||
1420 | for (i = 0; i < ndev->limits.max_db_bits; i++) | |
1421 | ntb_unregister_db_callback(ndev, i); | |
1422 | ||
1423 | kfree(ndev->db_cb); | |
1424 | } | |
1425 | ||
6465d02e JM |
1426 | static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf, |
1427 | size_t count, loff_t *offp) | |
1428 | { | |
1429 | struct ntb_device *ndev; | |
1430 | char *buf; | |
1431 | ssize_t ret, offset, out_count; | |
1432 | ||
1433 | out_count = 500; | |
1434 | ||
1435 | buf = kmalloc(out_count, GFP_KERNEL); | |
1436 | if (!buf) | |
1437 | return -ENOMEM; | |
1438 | ||
1439 | ndev = filp->private_data; | |
1440 | offset = 0; | |
1441 | offset += snprintf(buf + offset, out_count - offset, | |
1442 | "NTB Device Information:\n"); | |
1443 | offset += snprintf(buf + offset, out_count - offset, | |
1444 | "Connection Type - \t\t%s\n", | |
1445 | ndev->conn_type == NTB_CONN_TRANSPARENT ? | |
1446 | "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ? | |
1447 | "Back to back" : "Root Port"); | |
1448 | offset += snprintf(buf + offset, out_count - offset, | |
1449 | "Device Type - \t\t\t%s\n", | |
1450 | ndev->dev_type == NTB_DEV_USD ? | |
1451 | "DSD/USP" : "USD/DSP"); | |
1452 | offset += snprintf(buf + offset, out_count - offset, | |
1453 | "Max Number of Callbacks - \t%u\n", | |
1454 | ntb_max_cbs(ndev)); | |
1455 | offset += snprintf(buf + offset, out_count - offset, | |
1456 | "Link Status - \t\t\t%s\n", | |
1457 | ntb_hw_link_status(ndev) ? "Up" : "Down"); | |
1458 | if (ntb_hw_link_status(ndev)) { | |
1459 | offset += snprintf(buf + offset, out_count - offset, | |
1460 | "Link Speed - \t\t\tPCI-E Gen %u\n", | |
1461 | ndev->link_speed); | |
1462 | offset += snprintf(buf + offset, out_count - offset, | |
1463 | "Link Width - \t\t\tx%u\n", | |
1464 | ndev->link_width); | |
1465 | } | |
1466 | ||
b775e85b | 1467 | if (is_ntb_xeon(ndev)) { |
6465d02e JM |
1468 | u32 status32; |
1469 | u16 status16; | |
1470 | int rc; | |
1471 | ||
1472 | offset += snprintf(buf + offset, out_count - offset, | |
1473 | "\nNTB Device Statistics:\n"); | |
1474 | offset += snprintf(buf + offset, out_count - offset, | |
1475 | "Upstream Memory Miss - \t%u\n", | |
1476 | readw(ndev->reg_base + | |
1477 | SNB_USMEMMISS_OFFSET)); | |
1478 | ||
1479 | offset += snprintf(buf + offset, out_count - offset, | |
1480 | "\nNTB Hardware Errors:\n"); | |
1481 | ||
1482 | rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET, | |
1483 | &status16); | |
1484 | if (!rc) | |
1485 | offset += snprintf(buf + offset, out_count - offset, | |
1486 | "DEVSTS - \t%#06x\n", status16); | |
1487 | ||
1488 | rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, | |
1489 | &status16); | |
1490 | if (!rc) | |
1491 | offset += snprintf(buf + offset, out_count - offset, | |
1492 | "LNKSTS - \t%#06x\n", status16); | |
1493 | ||
1494 | rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET, | |
1495 | &status32); | |
1496 | if (!rc) | |
1497 | offset += snprintf(buf + offset, out_count - offset, | |
1498 | "UNCERRSTS - \t%#010x\n", status32); | |
1499 | ||
1500 | rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET, | |
1501 | &status32); | |
1502 | if (!rc) | |
1503 | offset += snprintf(buf + offset, out_count - offset, | |
1504 | "CORERRSTS - \t%#010x\n", status32); | |
1505 | } | |
1506 | ||
1507 | if (offset > out_count) | |
1508 | offset = out_count; | |
1509 | ||
1510 | ret = simple_read_from_buffer(ubuf, count, offp, buf, offset); | |
1511 | kfree(buf); | |
1512 | return ret; | |
1513 | } | |
1514 | ||
1515 | static const struct file_operations ntb_debugfs_info = { | |
1516 | .owner = THIS_MODULE, | |
1517 | .open = simple_open, | |
1518 | .read = ntb_debugfs_read, | |
1519 | }; | |
1520 | ||
1517a3f2 JM |
1521 | static void ntb_setup_debugfs(struct ntb_device *ndev) |
1522 | { | |
1523 | if (!debugfs_initialized()) | |
1524 | return; | |
1525 | ||
1526 | if (!debugfs_dir) | |
1527 | debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); | |
1528 | ||
1529 | ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev), | |
1530 | debugfs_dir); | |
6465d02e JM |
1531 | if (ndev->debugfs_dir) |
1532 | ndev->debugfs_info = debugfs_create_file("info", S_IRUSR, | |
1533 | ndev->debugfs_dir, | |
1534 | ndev, | |
1535 | &ntb_debugfs_info); | |
1517a3f2 JM |
1536 | } |
1537 | ||
1538 | static void ntb_free_debugfs(struct ntb_device *ndev) | |
1539 | { | |
1540 | debugfs_remove_recursive(ndev->debugfs_dir); | |
1541 | ||
1542 | if (debugfs_dir && simple_empty(debugfs_dir)) { | |
1543 | debugfs_remove_recursive(debugfs_dir); | |
1544 | debugfs_dir = NULL; | |
1545 | } | |
1546 | } | |
1547 | ||
9fec60c4 JM |
1548 | static void ntb_hw_link_up(struct ntb_device *ndev) |
1549 | { | |
1550 | if (ndev->conn_type == NTB_CONN_TRANSPARENT) | |
1551 | ntb_link_event(ndev, NTB_LINK_UP); | |
78958433 JM |
1552 | else { |
1553 | u32 ntb_cntl; | |
1554 | ||
9fec60c4 | 1555 | /* Let's bring the NTB link up */ |
78958433 JM |
1556 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); |
1557 | ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); | |
1558 | ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; | |
ab760a0c DJ |
1559 | ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; |
1560 | if (ndev->split_bar) | |
1561 | ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP | | |
1562 | NTB_CNTL_S2P_BAR5_SNOOP; | |
1563 | ||
78958433 JM |
1564 | writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); |
1565 | } | |
9fec60c4 JM |
1566 | } |
1567 | ||
1568 | static void ntb_hw_link_down(struct ntb_device *ndev) | |
1569 | { | |
1570 | u32 ntb_cntl; | |
1571 | ||
1572 | if (ndev->conn_type == NTB_CONN_TRANSPARENT) { | |
1573 | ntb_link_event(ndev, NTB_LINK_DOWN); | |
1574 | return; | |
1575 | } | |
1576 | ||
1577 | /* Bring NTB link down */ | |
1578 | ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); | |
78958433 | 1579 | ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); |
ab760a0c DJ |
1580 | ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); |
1581 | if (ndev->split_bar) | |
1582 | ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | | |
1583 | NTB_CNTL_S2P_BAR5_SNOOP); | |
78958433 | 1584 | ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; |
9fec60c4 JM |
1585 | writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); |
1586 | } | |
1587 | ||
ab760a0c DJ |
1588 | static void ntb_max_mw_detect(struct ntb_device *ndev) |
1589 | { | |
1590 | if (ndev->split_bar) | |
1591 | ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; | |
1592 | else | |
1593 | ndev->limits.max_mw = SNB_MAX_MW; | |
1594 | } | |
1595 | ||
1db97f25 DJ |
1596 | static int ntb_xeon_detect(struct ntb_device *ndev) |
1597 | { | |
ab760a0c DJ |
1598 | int rc, bars_mask; |
1599 | u32 bars; | |
1db97f25 DJ |
1600 | u8 ppd; |
1601 | ||
1602 | ndev->hw_type = SNB_HW; | |
1603 | ||
1604 | rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd); | |
1605 | if (rc) | |
1606 | return -EIO; | |
1607 | ||
1608 | if (ppd & SNB_PPD_DEV_TYPE) | |
1609 | ndev->dev_type = NTB_DEV_USD; | |
1610 | else | |
1611 | ndev->dev_type = NTB_DEV_DSD; | |
1612 | ||
ab760a0c DJ |
1613 | ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0; |
1614 | ||
1db97f25 DJ |
1615 | switch (ppd & SNB_PPD_CONN_TYPE) { |
1616 | case NTB_CONN_B2B: | |
1617 | dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); | |
1618 | ndev->conn_type = NTB_CONN_B2B; | |
1619 | break; | |
1620 | case NTB_CONN_RP: | |
1621 | dev_info(&ndev->pdev->dev, "Conn Type = RP\n"); | |
1622 | ndev->conn_type = NTB_CONN_RP; | |
1623 | break; | |
1624 | case NTB_CONN_TRANSPARENT: | |
1625 | dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n"); | |
1626 | ndev->conn_type = NTB_CONN_TRANSPARENT; | |
1627 | /* | |
1628 | * This mode is default to USD/DSP. HW does not report | |
1629 | * properly in transparent mode as it has no knowledge of | |
1630 | * NTB. We will just force correct here. | |
1631 | */ | |
1632 | ndev->dev_type = NTB_DEV_USD; | |
ab760a0c DJ |
1633 | |
1634 | /* | |
1635 | * This is a way for transparent BAR to figure out if we | |
1636 | * are doing split BAR or not. There is no way for the hw | |
1637 | * on the transparent side to know and set the PPD. | |
1638 | */ | |
1639 | bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM); | |
1640 | bars = hweight32(bars_mask); | |
1641 | if (bars == (HSX_SPLITBAR_MAX_MW + 1)) | |
1642 | ndev->split_bar = 1; | |
1643 | ||
1db97f25 DJ |
1644 | break; |
1645 | default: | |
1646 | dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd); | |
1647 | return -ENODEV; | |
1648 | } | |
1649 | ||
ab760a0c DJ |
1650 | ntb_max_mw_detect(ndev); |
1651 | ||
1db97f25 DJ |
1652 | return 0; |
1653 | } | |
1654 | ||
1655 | static int ntb_atom_detect(struct ntb_device *ndev) | |
1656 | { | |
1657 | int rc; | |
1658 | u32 ppd; | |
1659 | ||
1660 | ndev->hw_type = BWD_HW; | |
ebaad132 | 1661 | ndev->limits.max_mw = BWD_MAX_MW; |
1db97f25 DJ |
1662 | |
1663 | rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd); | |
1664 | if (rc) | |
1665 | return rc; | |
1666 | ||
1667 | switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) { | |
1668 | case NTB_CONN_B2B: | |
1669 | dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); | |
1670 | ndev->conn_type = NTB_CONN_B2B; | |
1671 | break; | |
1672 | case NTB_CONN_RP: | |
1673 | default: | |
1674 | dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); | |
1675 | return -EINVAL; | |
1676 | } | |
1677 | ||
1678 | if (ppd & BWD_PPD_DEV_TYPE) | |
1679 | ndev->dev_type = NTB_DEV_DSD; | |
1680 | else | |
1681 | ndev->dev_type = NTB_DEV_USD; | |
1682 | ||
1683 | return 0; | |
1684 | } | |
1685 | ||
1686 | static int ntb_device_detect(struct ntb_device *ndev) | |
1687 | { | |
1688 | int rc; | |
1689 | ||
1690 | if (is_ntb_xeon(ndev)) | |
1691 | rc = ntb_xeon_detect(ndev); | |
1692 | else if (is_ntb_atom(ndev)) | |
1693 | rc = ntb_atom_detect(ndev); | |
1694 | else | |
1695 | rc = -ENODEV; | |
1696 | ||
1697 | dev_info(&ndev->pdev->dev, "Device Type = %s\n", | |
1698 | ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); | |
1699 | ||
1700 | return 0; | |
1701 | } | |
1702 | ||
78a61ab7 | 1703 | static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) |
fce8a7bb JM |
1704 | { |
1705 | struct ntb_device *ndev; | |
1706 | int rc, i; | |
1707 | ||
1708 | ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); | |
1709 | if (!ndev) | |
1710 | return -ENOMEM; | |
1711 | ||
1712 | ndev->pdev = pdev; | |
069684e8 DJ |
1713 | |
1714 | ntb_set_errata_flags(ndev); | |
1715 | ||
fce8a7bb JM |
1716 | ndev->link_status = NTB_LINK_DOWN; |
1717 | pci_set_drvdata(pdev, ndev); | |
1517a3f2 | 1718 | ntb_setup_debugfs(ndev); |
fce8a7bb JM |
1719 | |
1720 | rc = pci_enable_device(pdev); | |
1721 | if (rc) | |
1722 | goto err; | |
1723 | ||
1724 | pci_set_master(ndev->pdev); | |
1725 | ||
1db97f25 DJ |
1726 | rc = ntb_device_detect(ndev); |
1727 | if (rc) | |
1728 | goto err; | |
1729 | ||
ab760a0c DJ |
1730 | ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw), |
1731 | GFP_KERNEL); | |
1732 | if (!ndev->mw) { | |
1733 | rc = -ENOMEM; | |
fce8a7bb | 1734 | goto err1; |
ab760a0c DJ |
1735 | } |
1736 | ||
1737 | if (ndev->split_bar) | |
1738 | rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK, | |
1739 | KBUILD_MODNAME); | |
1740 | else | |
1741 | rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, | |
1742 | KBUILD_MODNAME); | |
1743 | ||
1744 | if (rc) | |
1745 | goto err2; | |
fce8a7bb JM |
1746 | |
1747 | ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); | |
1748 | if (!ndev->reg_base) { | |
1749 | dev_warn(&pdev->dev, "Cannot remap BAR 0\n"); | |
1750 | rc = -EIO; | |
ab760a0c | 1751 | goto err3; |
fce8a7bb JM |
1752 | } |
1753 | ||
ab760a0c | 1754 | for (i = 0; i < ndev->limits.max_mw; i++) { |
fce8a7bb | 1755 | ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i)); |
ab760a0c DJ |
1756 | |
1757 | /* | |
1758 | * with the errata we need to steal last of the memory | |
1759 | * windows for workarounds and they point to MMIO registers. | |
1760 | */ | |
1761 | if ((ndev->wa_flags & WA_SNB_ERR) && | |
1762 | (i == (ndev->limits.max_mw - 1))) { | |
1763 | ndev->mw[i].vbase = | |
1764 | ioremap_nocache(pci_resource_start(pdev, | |
1765 | MW_TO_BAR(i)), | |
1766 | ndev->mw[i].bar_sz); | |
1767 | } else { | |
1768 | ndev->mw[i].vbase = | |
1769 | ioremap_wc(pci_resource_start(pdev, | |
1770 | MW_TO_BAR(i)), | |
1771 | ndev->mw[i].bar_sz); | |
1772 | } | |
1773 | ||
113fc505 | 1774 | dev_info(&pdev->dev, "MW %d size %llu\n", i, |
ac477afb | 1775 | (unsigned long long) ndev->mw[i].bar_sz); |
fce8a7bb JM |
1776 | if (!ndev->mw[i].vbase) { |
1777 | dev_warn(&pdev->dev, "Cannot remap BAR %d\n", | |
1778 | MW_TO_BAR(i)); | |
1779 | rc = -EIO; | |
2f4eb6a8 | 1780 | goto err4; |
fce8a7bb JM |
1781 | } |
1782 | } | |
1783 | ||
1784 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1785 | if (rc) { | |
1786 | rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1787 | if (rc) | |
ab760a0c | 1788 | goto err4; |
fce8a7bb JM |
1789 | |
1790 | dev_warn(&pdev->dev, "Cannot DMA highmem\n"); | |
1791 | } | |
1792 | ||
1793 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); | |
1794 | if (rc) { | |
1795 | rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); | |
1796 | if (rc) | |
ab760a0c | 1797 | goto err4; |
fce8a7bb JM |
1798 | |
1799 | dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); | |
1800 | } | |
1801 | ||
1802 | rc = ntb_device_setup(ndev); | |
1803 | if (rc) | |
ab760a0c | 1804 | goto err4; |
fce8a7bb JM |
1805 | |
1806 | rc = ntb_create_callbacks(ndev); | |
1807 | if (rc) | |
ab760a0c | 1808 | goto err5; |
fce8a7bb JM |
1809 | |
1810 | rc = ntb_setup_interrupts(ndev); | |
1811 | if (rc) | |
ab760a0c | 1812 | goto err6; |
fce8a7bb JM |
1813 | |
1814 | /* The scratchpad registers keep the values between rmmod/insmod, | |
1815 | * blast them now | |
1816 | */ | |
1817 | for (i = 0; i < ndev->limits.max_spads; i++) { | |
1818 | ntb_write_local_spad(ndev, i, 0); | |
1819 | ntb_write_remote_spad(ndev, i, 0); | |
1820 | } | |
1821 | ||
1822 | rc = ntb_transport_init(pdev); | |
1823 | if (rc) | |
ab760a0c | 1824 | goto err7; |
fce8a7bb | 1825 | |
9fec60c4 | 1826 | ntb_hw_link_up(ndev); |
fce8a7bb JM |
1827 | |
1828 | return 0; | |
1829 | ||
ab760a0c | 1830 | err7: |
fce8a7bb | 1831 | ntb_free_interrupts(ndev); |
ab760a0c | 1832 | err6: |
fce8a7bb | 1833 | ntb_free_callbacks(ndev); |
ab760a0c | 1834 | err5: |
fce8a7bb | 1835 | ntb_device_free(ndev); |
ab760a0c | 1836 | err4: |
fce8a7bb JM |
1837 | for (i--; i >= 0; i--) |
1838 | iounmap(ndev->mw[i].vbase); | |
1839 | iounmap(ndev->reg_base); | |
ab760a0c DJ |
1840 | err3: |
1841 | if (ndev->split_bar) | |
1842 | pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK); | |
1843 | else | |
1844 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
fce8a7bb | 1845 | err2: |
ab760a0c | 1846 | kfree(ndev->mw); |
fce8a7bb JM |
1847 | err1: |
1848 | pci_disable_device(pdev); | |
1849 | err: | |
1517a3f2 | 1850 | ntb_free_debugfs(ndev); |
fce8a7bb JM |
1851 | kfree(ndev); |
1852 | ||
1853 | dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME); | |
1854 | return rc; | |
1855 | } | |
1856 | ||
78a61ab7 | 1857 | static void ntb_pci_remove(struct pci_dev *pdev) |
fce8a7bb JM |
1858 | { |
1859 | struct ntb_device *ndev = pci_get_drvdata(pdev); | |
1860 | int i; | |
fce8a7bb | 1861 | |
9fec60c4 | 1862 | ntb_hw_link_down(ndev); |
fce8a7bb JM |
1863 | |
1864 | ntb_transport_free(ndev->ntb_transport); | |
1865 | ||
1866 | ntb_free_interrupts(ndev); | |
1867 | ntb_free_callbacks(ndev); | |
1868 | ntb_device_free(ndev); | |
1869 | ||
ab760a0c DJ |
1870 | /* need to reset max_mw limits so we can unmap properly */ |
1871 | if (ndev->hw_type == SNB_HW) | |
1872 | ntb_max_mw_detect(ndev); | |
1873 | ||
1874 | for (i = 0; i < ndev->limits.max_mw; i++) | |
fce8a7bb JM |
1875 | iounmap(ndev->mw[i].vbase); |
1876 | ||
ab760a0c | 1877 | kfree(ndev->mw); |
fce8a7bb | 1878 | iounmap(ndev->reg_base); |
ab760a0c DJ |
1879 | if (ndev->split_bar) |
1880 | pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK); | |
1881 | else | |
1882 | pci_release_selected_regions(pdev, NTB_BAR_MASK); | |
fce8a7bb | 1883 | pci_disable_device(pdev); |
1517a3f2 | 1884 | ntb_free_debugfs(ndev); |
fce8a7bb JM |
1885 | kfree(ndev); |
1886 | } | |
1887 | ||
1888 | static struct pci_driver ntb_pci_driver = { | |
1889 | .name = KBUILD_MODNAME, | |
1890 | .id_table = ntb_pci_tbl, | |
1891 | .probe = ntb_pci_probe, | |
78a61ab7 | 1892 | .remove = ntb_pci_remove, |
fce8a7bb | 1893 | }; |
6465d02e | 1894 | |
fce8a7bb | 1895 | module_pci_driver(ntb_pci_driver); |