projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
NTB: Add support for AMD PCI-Express Non-Transparent Bridge
[deliverable/linux.git] / drivers / ntb / hw / amd / ntb_hw_amd.c
1 /*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of AMD Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * AMD PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Xiangliang Yu <Xiangliang.Yu@amd.com>
47 */
48
49 #include <linux/debugfs.h>
50 #include <linux/delay.h>
51 #include <linux/init.h>
52 #include <linux/interrupt.h>
53 #include <linux/module.h>
54 #include <linux/acpi.h>
55 #include <linux/pci.h>
56 #include <linux/random.h>
57 #include <linux/slab.h>
58 #include <linux/ntb.h>
59
60 #include "ntb_hw_amd.h"
61
62 #define NTB_NAME "ntb_hw_amd"
63 #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
64 #define NTB_VER "1.0"
65
66 MODULE_DESCRIPTION(NTB_DESC);
67 MODULE_VERSION(NTB_VER);
68 MODULE_LICENSE("Dual BSD/GPL");
69 MODULE_AUTHOR("AMD Inc.");
70
71 static const struct file_operations amd_ntb_debugfs_info;
72 static struct dentry *debugfs_dir;
73
74 static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
75 {
76 if (idx < 0 || idx > ndev->mw_count)
77 return -EINVAL;
78
79 return 1 << idx;
80 }
81
82 static int amd_ntb_mw_count(struct ntb_dev *ntb)
83 {
84 return ntb_ndev(ntb)->mw_count;
85 }
86
87 static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
88 phys_addr_t *base,
89 resource_size_t *size,
90 resource_size_t *align,
91 resource_size_t *align_size)
92 {
93 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
94 int bar;
95
96 bar = ndev_mw_to_bar(ndev, idx);
97 if (bar < 0)
98 return bar;
99
100 if (base)
101 *base = pci_resource_start(ndev->ntb.pdev, bar);
102
103 if (size)
104 *size = pci_resource_len(ndev->ntb.pdev, bar);
105
106 if (align)
107 *align = SZ_4K;
108
109 if (align_size)
110 *align_size = 1;
111
112 return 0;
113 }
114
115 static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
116 dma_addr_t addr, resource_size_t size)
117 {
118 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
119 unsigned long xlat_reg, limit_reg = 0;
120 resource_size_t mw_size;
121 void __iomem *mmio, *peer_mmio;
122 u64 base_addr, limit, reg_val;
123 int bar;
124
125 bar = ndev_mw_to_bar(ndev, idx);
126 if (bar < 0)
127 return bar;
128
129 mw_size = pci_resource_len(ndev->ntb.pdev, bar);
130
131 /* make sure the range fits in the usable mw size */
132 if (size > mw_size)
133 return -EINVAL;
134
135 mmio = ndev->self_mmio;
136 peer_mmio = ndev->peer_mmio;
137
138 base_addr = pci_resource_start(ndev->ntb.pdev, bar);
139
140 if (bar != 1) {
141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3);
142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3);
143
144 /* Set the limit if supported */
145 limit = base_addr + size;
146
147 /* set and verify setting the translation address */
148 write64(addr, peer_mmio + xlat_reg);
149 reg_val = read64(peer_mmio + xlat_reg);
150 if (reg_val != addr) {
151 write64(0, peer_mmio + xlat_reg);
152 return -EIO;
153 }
154
155 /* set and verify setting the limit */
156 write64(limit, mmio + limit_reg);
157 reg_val = read64(mmio + limit_reg);
158 if (reg_val != limit) {
159 write64(base_addr, mmio + limit_reg);
160 write64(0, peer_mmio + xlat_reg);
161 return -EIO;
162 }
163 } else {
164 xlat_reg = AMD_BAR1XLAT_OFFSET;
165 limit_reg = AMD_BAR1LMT_OFFSET;
166
167 /* split bar addr range must all be 32 bit */
168 if (addr & (~0ull << 32))
169 return -EINVAL;
170 if ((addr + size) & (~0ull << 32))
171 return -EINVAL;
172
173 /* Set the limit if supported */
174 limit = base_addr + size;
175
176 /* set and verify setting the translation address */
177 write64(addr, peer_mmio + xlat_reg);
178 reg_val = read64(peer_mmio + xlat_reg);
179 if (reg_val != addr) {
180 write64(0, peer_mmio + xlat_reg);
181 return -EIO;
182 }
183
184 /* set and verify setting the limit */
185 writel(limit, mmio + limit_reg);
186 reg_val = readl(mmio + limit_reg);
187 if (reg_val != limit) {
188 writel(base_addr, mmio + limit_reg);
189 writel(0, peer_mmio + xlat_reg);
190 return -EIO;
191 }
192 }
193
194 return 0;
195 }
196
197 static int amd_link_is_up(struct amd_ntb_dev *ndev)
198 {
199 if (!ndev->peer_sta)
200 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
201
202 /* If peer_sta is reset or D0 event, the ISR has
203 * started a timer to check link status of hardware.
204 * So here just clear status bit. And if peer_sta is
205 * D3 or PME_TO, D0/reset event will be happened when
206 * system wakeup/poweron, so do nothing here.
207 */
208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
210 else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
211 ndev->peer_sta = 0;
212
213 return 0;
214 }
215
216 static int amd_ntb_link_is_up(struct ntb_dev *ntb,
217 enum ntb_speed *speed,
218 enum ntb_width *width)
219 {
220 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
221 int ret = 0;
222
223 if (amd_link_is_up(ndev)) {
224 if (speed)
225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
226 if (width)
227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
228
229 dev_dbg(ndev_dev(ndev), "link is up.\n");
230
231 ret = 1;
232 } else {
233 if (speed)
234 *speed = NTB_SPEED_NONE;
235 if (width)
236 *width = NTB_WIDTH_NONE;
237
238 dev_dbg(ndev_dev(ndev), "link is down.\n");
239 }
240
241 return ret;
242 }
243
244 static int amd_ntb_link_enable(struct ntb_dev *ntb,
245 enum ntb_speed max_speed,
246 enum ntb_width max_width)
247 {
248 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
249 void __iomem *mmio = ndev->self_mmio;
250 u32 ntb_ctl;
251
252 /* Enable event interrupt */
253 ndev->int_mask &= ~AMD_EVENT_INTMASK;
254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
255
256 if (ndev->ntb.topo == NTB_TOPO_SEC)
257 return -EINVAL;
258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
259
260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
263
264 return 0;
265 }
266
267 static int amd_ntb_link_disable(struct ntb_dev *ntb)
268 {
269 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
270 void __iomem *mmio = ndev->self_mmio;
271 u32 ntb_ctl;
272
273 /* Disable event interrupt */
274 ndev->int_mask |= AMD_EVENT_INTMASK;
275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
276
277 if (ndev->ntb.topo == NTB_TOPO_SEC)
278 return -EINVAL;
279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
280
281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
284
285 return 0;
286 }
287
288 static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
289 {
290 return ntb_ndev(ntb)->db_valid_mask;
291 }
292
293 static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
294 {
295 return ntb_ndev(ntb)->db_count;
296 }
297
298 static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
299 {
300 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
301
302 if (db_vector < 0 || db_vector > ndev->db_count)
303 return 0;
304
305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
306 }
307
308 static u64 amd_ntb_db_read(struct ntb_dev *ntb)
309 {
310 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
311 void __iomem *mmio = ndev->self_mmio;
312
313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
314 }
315
316 static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
317 {
318 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
319 void __iomem *mmio = ndev->self_mmio;
320
321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
322
323 return 0;
324 }
325
326 static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
327 {
328 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
329 void __iomem *mmio = ndev->self_mmio;
330 unsigned long flags;
331
332 if (db_bits & ~ndev->db_valid_mask)
333 return -EINVAL;
334
335 spin_lock_irqsave(&ndev->db_mask_lock, flags);
336 ndev->db_mask |= db_bits;
337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
339
340 return 0;
341 }
342
343 static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
344 {
345 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
346 void __iomem *mmio = ndev->self_mmio;
347 unsigned long flags;
348
349 if (db_bits & ~ndev->db_valid_mask)
350 return -EINVAL;
351
352 spin_lock_irqsave(&ndev->db_mask_lock, flags);
353 ndev->db_mask &= ~db_bits;
354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
356
357 return 0;
358 }
359
360 static int amd_ntb_peer_db_addr(struct ntb_dev *ntb,
361 phys_addr_t *db_addr,
362 resource_size_t *db_size)
363 {
364 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
365
366 if (db_addr)
367 *db_addr = (phys_addr_t)(ndev->peer_mmio + AMD_DBREQ_OFFSET);
368 if (db_size)
369 *db_size = sizeof(u32);
370
371 return 0;
372 }
373
374 static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
375 {
376 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
377 void __iomem *mmio = ndev->self_mmio;
378
379 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
380
381 return 0;
382 }
383
384 static int amd_ntb_spad_count(struct ntb_dev *ntb)
385 {
386 return ntb_ndev(ntb)->spad_count;
387 }
388
389 static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
390 {
391 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
392 void __iomem *mmio = ndev->self_mmio;
393 u32 offset;
394
395 if (idx < 0 || idx >= ndev->spad_count)
396 return 0;
397
398 offset = ndev->self_spad + (idx << 2);
399 return readl(mmio + AMD_SPAD_OFFSET + offset);
400 }
401
402 static int amd_ntb_spad_write(struct ntb_dev *ntb,
403 int idx, u32 val)
404 {
405 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
406 void __iomem *mmio = ndev->self_mmio;
407 u32 offset;
408
409 if (idx < 0 || idx >= ndev->spad_count)
410 return -EINVAL;
411
412 offset = ndev->self_spad + (idx << 2);
413 writel(val, mmio + AMD_SPAD_OFFSET + offset);
414
415 return 0;
416 }
417
418 static int amd_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
419 phys_addr_t *spad_addr)
420 {
421 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
422
423 if (idx < 0 || idx >= ndev->spad_count)
424 return -EINVAL;
425
426 if (spad_addr)
427 *spad_addr = (phys_addr_t)(ndev->self_mmio + AMD_SPAD_OFFSET +
428 ndev->peer_spad + (idx << 2));
429 return 0;
430 }
431
432 static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
433 {
434 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
435 void __iomem *mmio = ndev->self_mmio;
436 u32 offset;
437
438 if (idx < 0 || idx >= ndev->spad_count)
439 return -EINVAL;
440
441 offset = ndev->peer_spad + (idx << 2);
442 return readl(mmio + AMD_SPAD_OFFSET + offset);
443 }
444
445 static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
446 int idx, u32 val)
447 {
448 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
449 void __iomem *mmio = ndev->self_mmio;
450 u32 offset;
451
452 if (idx < 0 || idx >= ndev->spad_count)
453 return -EINVAL;
454
455 offset = ndev->peer_spad + (idx << 2);
456 writel(val, mmio + AMD_SPAD_OFFSET + offset);
457
458 return 0;
459 }
460
461 static const struct ntb_dev_ops amd_ntb_ops = {
462 .mw_count = amd_ntb_mw_count,
463 .mw_get_range = amd_ntb_mw_get_range,
464 .mw_set_trans = amd_ntb_mw_set_trans,
465 .link_is_up = amd_ntb_link_is_up,
466 .link_enable = amd_ntb_link_enable,
467 .link_disable = amd_ntb_link_disable,
468 .db_valid_mask = amd_ntb_db_valid_mask,
469 .db_vector_count = amd_ntb_db_vector_count,
470 .db_vector_mask = amd_ntb_db_vector_mask,
471 .db_read = amd_ntb_db_read,
472 .db_clear = amd_ntb_db_clear,
473 .db_set_mask = amd_ntb_db_set_mask,
474 .db_clear_mask = amd_ntb_db_clear_mask,
475 .peer_db_addr = amd_ntb_peer_db_addr,
476 .peer_db_set = amd_ntb_peer_db_set,
477 .spad_count = amd_ntb_spad_count,
478 .spad_read = amd_ntb_spad_read,
479 .spad_write = amd_ntb_spad_write,
480 .peer_spad_addr = amd_ntb_peer_spad_addr,
481 .peer_spad_read = amd_ntb_peer_spad_read,
482 .peer_spad_write = amd_ntb_peer_spad_write,
483 };
484
485 static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
486 {
487 void __iomem *mmio = ndev->self_mmio;
488 int reg;
489
490 reg = readl(mmio + AMD_SMUACK_OFFSET);
491 reg |= bit;
492 writel(reg, mmio + AMD_SMUACK_OFFSET);
493
494 ndev->peer_sta |= bit;
495 }
496
497 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
498 {
499 void __iomem *mmio = ndev->self_mmio;
500 u32 status;
501
502 status = readl(mmio + AMD_INTSTAT_OFFSET);
503 if (!(status & AMD_EVENT_INTMASK))
504 return;
505
506 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec);
507
508 status &= AMD_EVENT_INTMASK;
509 switch (status) {
510 case AMD_PEER_FLUSH_EVENT:
511 dev_info(ndev_dev(ndev), "Flush is done.\n");
512 break;
513 case AMD_PEER_RESET_EVENT:
514 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
515
516 /* link down first */
517 ntb_link_event(&ndev->ntb);
518 /* polling peer status */
519 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
520
521 break;
522 case AMD_PEER_D3_EVENT:
523 case AMD_PEER_PMETO_EVENT:
524 amd_ack_smu(ndev, status);
525
526 /* link down */
527 ntb_link_event(&ndev->ntb);
528
529 break;
530 case AMD_PEER_D0_EVENT:
531 mmio = ndev->peer_mmio;
532 status = readl(mmio + AMD_PMESTAT_OFFSET);
533 /* check if this is WAKEUP event */
534 if (status & 0x1)
535 dev_info(ndev_dev(ndev), "Wakeup is done.\n");
536
537 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
538
539 /* start a timer to poll link status */
540 schedule_delayed_work(&ndev->hb_timer,
541 AMD_LINK_HB_TIMEOUT);
542 break;
543 default:
544 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status);
545 break;
546 }
547 }
548
549 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
550 {
551 dev_dbg(ndev_dev(ndev), "vec %d\n", vec);
552
553 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
554 amd_handle_event(ndev, vec);
555
556 if (vec < AMD_DB_CNT)
557 ntb_db_event(&ndev->ntb, vec);
558
559 return IRQ_HANDLED;
560 }
561
562 static irqreturn_t ndev_vec_isr(int irq, void *dev)
563 {
564 struct amd_ntb_vec *nvec = dev;
565
566 return ndev_interrupt(nvec->ndev, nvec->num);
567 }
568
569 static irqreturn_t ndev_irq_isr(int irq, void *dev)
570 {
571 struct amd_ntb_dev *ndev = dev;
572
573 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
574 }
575
576 static int ndev_init_isr(struct amd_ntb_dev *ndev,
577 int msix_min, int msix_max)
578 {
579 struct pci_dev *pdev;
580 int rc, i, msix_count, node;
581
582 pdev = ndev_pdev(ndev);
583
584 node = dev_to_node(&pdev->dev);
585
586 ndev->db_mask = ndev->db_valid_mask;
587
588 /* Try to set up msix irq */
589 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
590 GFP_KERNEL, node);
591 if (!ndev->vec)
592 goto err_msix_vec_alloc;
593
594 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
595 GFP_KERNEL, node);
596 if (!ndev->msix)
597 goto err_msix_alloc;
598
599 for (i = 0; i < msix_max; ++i)
600 ndev->msix[i].entry = i;
601
602 msix_count = pci_enable_msix_range(pdev, ndev->msix,
603 msix_min, msix_max);
604 if (msix_count < 0)
605 goto err_msix_enable;
606
607 /* NOTE: Disable MSIX if msix count is less than 16 because of
608 * hardware limitation.
609 */
610 if (msix_count < msix_min) {
611 pci_disable_msix(pdev);
612 goto err_msix_enable;
613 }
614
615 for (i = 0; i < msix_count; ++i) {
616 ndev->vec[i].ndev = ndev;
617 ndev->vec[i].num = i;
618 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
619 "ndev_vec_isr", &ndev->vec[i]);
620 if (rc)
621 goto err_msix_request;
622 }
623
624 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
625 ndev->db_count = msix_min;
626 ndev->msix_vec_count = msix_max;
627 return 0;
628
629 err_msix_request:
630 while (i-- > 0)
631 free_irq(ndev->msix[i].vector, ndev);
632 pci_disable_msix(pdev);
633 err_msix_enable:
634 kfree(ndev->msix);
635 err_msix_alloc:
636 kfree(ndev->vec);
637 err_msix_vec_alloc:
638 ndev->msix = NULL;
639 ndev->vec = NULL;
640
641 /* Try to set up msi irq */
642 rc = pci_enable_msi(pdev);
643 if (rc)
644 goto err_msi_enable;
645
646 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
647 "ndev_irq_isr", ndev);
648 if (rc)
649 goto err_msi_request;
650
651 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
652 ndev->db_count = 1;
653 ndev->msix_vec_count = 1;
654 return 0;
655
656 err_msi_request:
657 pci_disable_msi(pdev);
658 err_msi_enable:
659
660 /* Try to set up intx irq */
661 pci_intx(pdev, 1);
662
663 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
664 "ndev_irq_isr", ndev);
665 if (rc)
666 goto err_intx_request;
667
668 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
669 ndev->db_count = 1;
670 ndev->msix_vec_count = 1;
671 return 0;
672
673 err_intx_request:
674 return rc;
675 }
676
677 static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
678 {
679 struct pci_dev *pdev;
680 void __iomem *mmio = ndev->self_mmio;
681 int i;
682
683 pdev = ndev_pdev(ndev);
684
685 /* Mask all doorbell interrupts */
686 ndev->db_mask = ndev->db_valid_mask;
687 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
688
689 if (ndev->msix) {
690 i = ndev->msix_vec_count;
691 while (i--)
692 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
693 pci_disable_msix(pdev);
694 kfree(ndev->msix);
695 kfree(ndev->vec);
696 } else {
697 free_irq(pdev->irq, ndev);
698 if (pci_dev_msi_enabled(pdev))
699 pci_disable_msi(pdev);
700 else
701 pci_intx(pdev, 0);
702 }
703 }
704
705 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
706 size_t count, loff_t *offp)
707 {
708 struct amd_ntb_dev *ndev;
709 void __iomem *mmio;
710 char *buf;
711 size_t buf_size;
712 ssize_t ret, off;
713 union { u64 v64; u32 v32; u16 v16; } u;
714
715 ndev = filp->private_data;
716 mmio = ndev->self_mmio;
717
718 buf_size = min(count, 0x800ul);
719
720 buf = kmalloc(buf_size, GFP_KERNEL);
721 if (!buf)
722 return -ENOMEM;
723
724 off = 0;
725
726 off += scnprintf(buf + off, buf_size - off,
727 "NTB Device Information:\n");
728
729 off += scnprintf(buf + off, buf_size - off,
730 "Connection Topology -\t%s\n",
731 ntb_topo_string(ndev->ntb.topo));
732
733 off += scnprintf(buf + off, buf_size - off,
734 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
735
736 if (!amd_link_is_up(ndev)) {
737 off += scnprintf(buf + off, buf_size - off,
738 "Link Status -\t\tDown\n");
739 } else {
740 off += scnprintf(buf + off, buf_size - off,
741 "Link Status -\t\tUp\n");
742 off += scnprintf(buf + off, buf_size - off,
743 "Link Speed -\t\tPCI-E Gen %u\n",
744 NTB_LNK_STA_SPEED(ndev->lnk_sta));
745 off += scnprintf(buf + off, buf_size - off,
746 "Link Width -\t\tx%u\n",
747 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
748 }
749
750 off += scnprintf(buf + off, buf_size - off,
751 "Memory Window Count -\t%u\n", ndev->mw_count);
752 off += scnprintf(buf + off, buf_size - off,
753 "Scratchpad Count -\t%u\n", ndev->spad_count);
754 off += scnprintf(buf + off, buf_size - off,
755 "Doorbell Count -\t%u\n", ndev->db_count);
756 off += scnprintf(buf + off, buf_size - off,
757 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
758
759 off += scnprintf(buf + off, buf_size - off,
760 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
761
762 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
763 off += scnprintf(buf + off, buf_size - off,
764 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
765
766 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
767 off += scnprintf(buf + off, buf_size - off,
768 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
769
770 off += scnprintf(buf + off, buf_size - off,
771 "\nNTB Incoming XLAT:\n");
772
773 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
774 off += scnprintf(buf + off, buf_size - off,
775 "XLAT1 -\t\t%#018llx\n", u.v64);
776
777 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
778 off += scnprintf(buf + off, buf_size - off,
779 "XLAT23 -\t\t%#018llx\n", u.v64);
780
781 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
782 off += scnprintf(buf + off, buf_size - off,
783 "XLAT45 -\t\t%#018llx\n", u.v64);
784
785 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
786 off += scnprintf(buf + off, buf_size - off,
787 "LMT1 -\t\t\t%#06x\n", u.v32);
788
789 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
790 off += scnprintf(buf + off, buf_size - off,
791 "LMT23 -\t\t\t%#018llx\n", u.v64);
792
793 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
794 off += scnprintf(buf + off, buf_size - off,
795 "LMT45 -\t\t\t%#018llx\n", u.v64);
796
797 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
798 kfree(buf);
799 return ret;
800 }
801
802 static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
803 {
804 if (!debugfs_dir) {
805 ndev->debugfs_dir = NULL;
806 ndev->debugfs_info = NULL;
807 } else {
808 ndev->debugfs_dir =
809 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
810 if (!ndev->debugfs_dir)
811 ndev->debugfs_info = NULL;
812 else
813 ndev->debugfs_info =
814 debugfs_create_file("info", S_IRUSR,
815 ndev->debugfs_dir, ndev,
816 &amd_ntb_debugfs_info);
817 }
818 }
819
820 static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
821 {
822 debugfs_remove_recursive(ndev->debugfs_dir);
823 }
824
825 static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
826 struct pci_dev *pdev)
827 {
828 ndev->ntb.pdev = pdev;
829 ndev->ntb.topo = NTB_TOPO_NONE;
830 ndev->ntb.ops = &amd_ntb_ops;
831 ndev->int_mask = AMD_EVENT_INTMASK;
832 spin_lock_init(&ndev->db_mask_lock);
833 }
834
835 static int amd_poll_link(struct amd_ntb_dev *ndev)
836 {
837 void __iomem *mmio = ndev->peer_mmio;
838 u32 reg, stat;
839 int rc;
840
841 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
842 reg &= NTB_LIN_STA_ACTIVE_BIT;
843
844 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg);
845
846 if (reg == ndev->cntl_sta)
847 return 0;
848
849 ndev->cntl_sta = reg;
850
851 rc = pci_read_config_dword(ndev->ntb.pdev,
852 AMD_LINK_STATUS_OFFSET, &stat);
853 if (rc)
854 return 0;
855 ndev->lnk_sta = stat;
856
857 return 1;
858 }
859
860 static void amd_link_hb(struct work_struct *work)
861 {
862 struct amd_ntb_dev *ndev = hb_ndev(work);
863
864 if (amd_poll_link(ndev))
865 ntb_link_event(&ndev->ntb);
866
867 if (!amd_link_is_up(ndev))
868 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
869 }
870
871 static int amd_init_isr(struct amd_ntb_dev *ndev)
872 {
873 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
874 }
875
876 static void amd_init_side_info(struct amd_ntb_dev *ndev)
877 {
878 void __iomem *mmio = ndev->self_mmio;
879 unsigned int reg;
880
881 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
882 if (!(reg & AMD_SIDE_READY)) {
883 reg |= AMD_SIDE_READY;
884 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
885 }
886 }
887
888 static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
889 {
890 void __iomem *mmio = ndev->self_mmio;
891 unsigned int reg;
892
893 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
894 if (reg & AMD_SIDE_READY) {
895 reg &= ~AMD_SIDE_READY;
896 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
897 readl(mmio + AMD_SIDEINFO_OFFSET);
898 }
899 }
900
901 static int amd_init_ntb(struct amd_ntb_dev *ndev)
902 {
903 void __iomem *mmio = ndev->self_mmio;
904
905 ndev->mw_count = AMD_MW_CNT;
906 ndev->spad_count = AMD_SPADS_CNT;
907 ndev->db_count = AMD_DB_CNT;
908
909 switch (ndev->ntb.topo) {
910 case NTB_TOPO_PRI:
911 case NTB_TOPO_SEC:
912 ndev->spad_count >>= 1;
913 if (ndev->ntb.topo == NTB_TOPO_PRI) {
914 ndev->self_spad = 0;
915 ndev->peer_spad = 0x20;
916 } else {
917 ndev->self_spad = 0x20;
918 ndev->peer_spad = 0;
919 }
920
921 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
922 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
923
924 break;
925 default:
926 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n");
927 return -EINVAL;
928 }
929
930 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
931
932 /* Mask event interrupts */
933 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
934
935 return 0;
936 }
937
938 static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
939 {
940 void __iomem *mmio = ndev->self_mmio;
941 u32 info;
942
943 info = readl(mmio + AMD_SIDEINFO_OFFSET);
944 if (info & AMD_SIDE_MASK)
945 return NTB_TOPO_SEC;
946 else
947 return NTB_TOPO_PRI;
948 }
949
950 static int amd_init_dev(struct amd_ntb_dev *ndev)
951 {
952 struct pci_dev *pdev;
953 int rc = 0;
954
955 pdev = ndev_pdev(ndev);
956
957 ndev->ntb.topo = amd_get_topo(ndev);
958 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n",
959 ntb_topo_string(ndev->ntb.topo));
960
961 rc = amd_init_ntb(ndev);
962 if (rc)
963 return rc;
964
965 rc = amd_init_isr(ndev);
966 if (rc) {
967 dev_err(ndev_dev(ndev), "fail to init isr.\n");
968 return rc;
969 }
970
971 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
972
973 return 0;
974 }
975
976 static void amd_deinit_dev(struct amd_ntb_dev *ndev)
977 {
978 cancel_delayed_work_sync(&ndev->hb_timer);
979
980 ndev_deinit_isr(ndev);
981 }
982
983 static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
984 struct pci_dev *pdev)
985 {
986 int rc;
987
988 pci_set_drvdata(pdev, ndev);
989
990 rc = pci_enable_device(pdev);
991 if (rc)
992 goto err_pci_enable;
993
994 rc = pci_request_regions(pdev, NTB_NAME);
995 if (rc)
996 goto err_pci_regions;
997
998 pci_set_master(pdev);
999
1000 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1001 if (rc) {
1002 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1003 if (rc)
1004 goto err_dma_mask;
1005 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1006 }
1007
1008 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1009 if (rc) {
1010 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1011 if (rc)
1012 goto err_dma_mask;
1013 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1014 }
1015
1016 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1017 if (!ndev->self_mmio) {
1018 rc = -EIO;
1019 goto err_dma_mask;
1020 }
1021 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1022
1023 return 0;
1024
1025 err_dma_mask:
1026 pci_clear_master(pdev);
1027 err_pci_regions:
1028 pci_disable_device(pdev);
1029 err_pci_enable:
1030 pci_set_drvdata(pdev, NULL);
1031 return rc;
1032 }
1033
1034 static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1035 {
1036 struct pci_dev *pdev = ndev_pdev(ndev);
1037
1038 pci_iounmap(pdev, ndev->self_mmio);
1039
1040 pci_clear_master(pdev);
1041 pci_release_regions(pdev);
1042 pci_disable_device(pdev);
1043 pci_set_drvdata(pdev, NULL);
1044 }
1045
1046 static int amd_ntb_pci_probe(struct pci_dev *pdev,
1047 const struct pci_device_id *id)
1048 {
1049 struct amd_ntb_dev *ndev;
1050 int rc, node;
1051
1052 node = dev_to_node(&pdev->dev);
1053
1054 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1055 if (!ndev) {
1056 rc = -ENOMEM;
1057 goto err_ndev;
1058 }
1059
1060 ndev_init_struct(ndev, pdev);
1061
1062 rc = amd_ntb_init_pci(ndev, pdev);
1063 if (rc)
1064 goto err_init_pci;
1065
1066 rc = amd_init_dev(ndev);
1067 if (rc)
1068 goto err_init_dev;
1069
1070 /* write side info */
1071 amd_init_side_info(ndev);
1072
1073 amd_poll_link(ndev);
1074
1075 ndev_init_debugfs(ndev);
1076
1077 rc = ntb_register_device(&ndev->ntb);
1078 if (rc)
1079 goto err_register;
1080
1081 dev_info(&pdev->dev, "NTB device registered.\n");
1082
1083 return 0;
1084
1085 err_register:
1086 ndev_deinit_debugfs(ndev);
1087 amd_deinit_dev(ndev);
1088 err_init_dev:
1089 amd_ntb_deinit_pci(ndev);
1090 err_init_pci:
1091 kfree(ndev);
1092 err_ndev:
1093 return rc;
1094 }
1095
1096 static void amd_ntb_pci_remove(struct pci_dev *pdev)
1097 {
1098 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1099
1100 ntb_unregister_device(&ndev->ntb);
1101 ndev_deinit_debugfs(ndev);
1102 amd_deinit_side_info(ndev);
1103 amd_deinit_dev(ndev);
1104 amd_ntb_deinit_pci(ndev);
1105 kfree(ndev);
1106 }
1107
1108 static const struct file_operations amd_ntb_debugfs_info = {
1109 .owner = THIS_MODULE,
1110 .open = simple_open,
1111 .read = ndev_debugfs_read,
1112 };
1113
1114 static const struct pci_device_id amd_ntb_pci_tbl[] = {
1115 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
1116 {0}
1117 };
1118 MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1119
1120 static struct pci_driver amd_ntb_pci_driver = {
1121 .name = KBUILD_MODNAME,
1122 .id_table = amd_ntb_pci_tbl,
1123 .probe = amd_ntb_pci_probe,
1124 .remove = amd_ntb_pci_remove,
1125 };
1126
1127 static int __init amd_ntb_pci_driver_init(void)
1128 {
1129 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1130
1131 if (debugfs_initialized())
1132 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1133
1134 return pci_register_driver(&amd_ntb_pci_driver);
1135 }
1136 module_init(amd_ntb_pci_driver_init);
1137
1138 static void __exit amd_ntb_pci_driver_exit(void)
1139 {
1140 pci_unregister_driver(&amd_ntb_pci_driver);
1141 debugfs_remove_recursive(debugfs_dir);
1142 }
1143 module_exit(amd_ntb_pci_driver_exit);
Linux kernel - Deliverables
This page took 0.055554 seconds and 5 git commands to generate.