4 * Copyright (C) 2014 Renesas Electronics Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/err.h>
14 #include <linux/export.h>
15 #include <linux/interrupt.h>
17 #include <linux/iommu.h>
18 #include <linux/module.h>
20 #include <linux/platform_data/ipmmu-vmsa.h>
21 #include <linux/platform_device.h>
22 #include <linux/sizes.h>
23 #include <linux/slab.h>
25 #include <asm/dma-iommu.h>
26 #include <asm/pgalloc.h>
28 struct ipmmu_vmsa_device
{
31 struct list_head list
;
33 const struct ipmmu_vmsa_platform_data
*pdata
;
34 unsigned int num_utlbs
;
36 struct dma_iommu_mapping
*mapping
;
39 struct ipmmu_vmsa_domain
{
40 struct ipmmu_vmsa_device
*mmu
;
41 struct iommu_domain
*io_domain
;
43 unsigned int context_id
;
44 spinlock_t lock
; /* Protects mappings */
48 struct ipmmu_vmsa_archdata
{
49 struct ipmmu_vmsa_device
*mmu
;
51 unsigned int num_utlbs
;
54 static DEFINE_SPINLOCK(ipmmu_devices_lock
);
55 static LIST_HEAD(ipmmu_devices
);
57 #define TLB_LOOP_TIMEOUT 100 /* 100us */
59 /* -----------------------------------------------------------------------------
60 * Registers Definition
63 #define IM_NS_ALIAS_OFFSET 0x800
65 #define IM_CTX_SIZE 0x40
68 #define IMCTR_TRE (1 << 17)
69 #define IMCTR_AFE (1 << 16)
70 #define IMCTR_RTSEL_MASK (3 << 4)
71 #define IMCTR_RTSEL_SHIFT 4
72 #define IMCTR_TREN (1 << 3)
73 #define IMCTR_INTEN (1 << 2)
74 #define IMCTR_FLUSH (1 << 1)
75 #define IMCTR_MMUEN (1 << 0)
79 #define IMTTBCR 0x0008
80 #define IMTTBCR_EAE (1 << 31)
81 #define IMTTBCR_PMB (1 << 30)
82 #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
83 #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
84 #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
85 #define IMTTBCR_SH1_MASK (3 << 28)
86 #define IMTTBCR_ORGN1_NC (0 << 26)
87 #define IMTTBCR_ORGN1_WB_WA (1 << 26)
88 #define IMTTBCR_ORGN1_WT (2 << 26)
89 #define IMTTBCR_ORGN1_WB (3 << 26)
90 #define IMTTBCR_ORGN1_MASK (3 << 26)
91 #define IMTTBCR_IRGN1_NC (0 << 24)
92 #define IMTTBCR_IRGN1_WB_WA (1 << 24)
93 #define IMTTBCR_IRGN1_WT (2 << 24)
94 #define IMTTBCR_IRGN1_WB (3 << 24)
95 #define IMTTBCR_IRGN1_MASK (3 << 24)
96 #define IMTTBCR_TSZ1_MASK (7 << 16)
97 #define IMTTBCR_TSZ1_SHIFT 16
98 #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
99 #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
100 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
101 #define IMTTBCR_SH0_MASK (3 << 12)
102 #define IMTTBCR_ORGN0_NC (0 << 10)
103 #define IMTTBCR_ORGN0_WB_WA (1 << 10)
104 #define IMTTBCR_ORGN0_WT (2 << 10)
105 #define IMTTBCR_ORGN0_WB (3 << 10)
106 #define IMTTBCR_ORGN0_MASK (3 << 10)
107 #define IMTTBCR_IRGN0_NC (0 << 8)
108 #define IMTTBCR_IRGN0_WB_WA (1 << 8)
109 #define IMTTBCR_IRGN0_WT (2 << 8)
110 #define IMTTBCR_IRGN0_WB (3 << 8)
111 #define IMTTBCR_IRGN0_MASK (3 << 8)
112 #define IMTTBCR_SL0_LVL_2 (0 << 4)
113 #define IMTTBCR_SL0_LVL_1 (1 << 4)
114 #define IMTTBCR_TSZ0_MASK (7 << 0)
115 #define IMTTBCR_TSZ0_SHIFT O
117 #define IMBUSCR 0x000c
118 #define IMBUSCR_DVM (1 << 2)
119 #define IMBUSCR_BUSSEL_SYS (0 << 0)
120 #define IMBUSCR_BUSSEL_CCI (1 << 0)
121 #define IMBUSCR_BUSSEL_IMCAAR (2 << 0)
122 #define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0)
123 #define IMBUSCR_BUSSEL_MASK (3 << 0)
125 #define IMTTLBR0 0x0010
126 #define IMTTUBR0 0x0014
127 #define IMTTLBR1 0x0018
128 #define IMTTUBR1 0x001c
131 #define IMSTR_ERRLVL_MASK (3 << 12)
132 #define IMSTR_ERRLVL_SHIFT 12
133 #define IMSTR_ERRCODE_TLB_FORMAT (1 << 8)
134 #define IMSTR_ERRCODE_ACCESS_PERM (4 << 8)
135 #define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8)
136 #define IMSTR_ERRCODE_MASK (7 << 8)
137 #define IMSTR_MHIT (1 << 4)
138 #define IMSTR_ABORT (1 << 2)
139 #define IMSTR_PF (1 << 1)
140 #define IMSTR_TF (1 << 0)
142 #define IMMAIR0 0x0028
143 #define IMMAIR1 0x002c
144 #define IMMAIR_ATTR_MASK 0xff
145 #define IMMAIR_ATTR_DEVICE 0x04
146 #define IMMAIR_ATTR_NC 0x44
147 #define IMMAIR_ATTR_WBRWA 0xff
148 #define IMMAIR_ATTR_SHIFT(n) ((n) << 3)
149 #define IMMAIR_ATTR_IDX_NC 0
150 #define IMMAIR_ATTR_IDX_WBRWA 1
151 #define IMMAIR_ATTR_IDX_DEV 2
155 #define IMPCTR 0x0200
156 #define IMPSTR 0x0208
157 #define IMPEAR 0x020c
158 #define IMPMBA(n) (0x0280 + ((n) * 4))
159 #define IMPMBD(n) (0x02c0 + ((n) * 4))
161 #define IMUCTR(n) (0x0300 + ((n) * 16))
162 #define IMUCTR_FIXADDEN (1 << 31)
163 #define IMUCTR_FIXADD_MASK (0xff << 16)
164 #define IMUCTR_FIXADD_SHIFT 16
165 #define IMUCTR_TTSEL_MMU(n) ((n) << 4)
166 #define IMUCTR_TTSEL_PMB (8 << 4)
167 #define IMUCTR_TTSEL_MASK (15 << 4)
168 #define IMUCTR_FLUSH (1 << 1)
169 #define IMUCTR_MMUEN (1 << 0)
171 #define IMUASID(n) (0x0308 + ((n) * 16))
172 #define IMUASID_ASID8_MASK (0xff << 8)
173 #define IMUASID_ASID8_SHIFT 8
174 #define IMUASID_ASID0_MASK (0xff << 0)
175 #define IMUASID_ASID0_SHIFT 0
177 /* -----------------------------------------------------------------------------
182 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access,
183 * Long-descriptor format" that the NStable bit being set in a table descriptor
184 * will result in the NStable and NS bits of all child entries being ignored and
185 * considered as being set. The IPMMU seems not to comply with this, as it
186 * generates a secure access page fault if any of the NStable and NS bits isn't
187 * set when running in non-secure mode.
190 #define PMD_NSTABLE (_AT(pmdval_t, 1) << 63)
193 #define ARM_VMSA_PTE_XN (((pteval_t)3) << 53)
194 #define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52)
195 #define ARM_VMSA_PTE_AF (((pteval_t)1) << 10)
196 #define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8)
197 #define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8)
198 #define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8)
199 #define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8)
200 #define ARM_VMSA_PTE_NS (((pteval_t)1) << 5)
201 #define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0)
204 #define ARM_VMSA_PTE_nG (((pteval_t)1) << 11)
205 #define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6)
206 #define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6)
207 #define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6)
208 #define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2)
209 #define ARM_VMSA_PTE_ATTRINDX_SHIFT 2
211 #define ARM_VMSA_PTE_ATTRS_MASK \
212 (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \
213 ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \
214 ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK)
216 #define ARM_VMSA_PTE_CONT_ENTRIES 16
217 #define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES)
219 #define IPMMU_PTRS_PER_PTE 512
220 #define IPMMU_PTRS_PER_PMD 512
221 #define IPMMU_PTRS_PER_PGD 4
223 /* -----------------------------------------------------------------------------
227 static u32
ipmmu_read(struct ipmmu_vmsa_device
*mmu
, unsigned int offset
)
229 return ioread32(mmu
->base
+ offset
);
232 static void ipmmu_write(struct ipmmu_vmsa_device
*mmu
, unsigned int offset
,
235 iowrite32(data
, mmu
->base
+ offset
);
238 static u32
ipmmu_ctx_read(struct ipmmu_vmsa_domain
*domain
, unsigned int reg
)
240 return ipmmu_read(domain
->mmu
, domain
->context_id
* IM_CTX_SIZE
+ reg
);
243 static void ipmmu_ctx_write(struct ipmmu_vmsa_domain
*domain
, unsigned int reg
,
246 ipmmu_write(domain
->mmu
, domain
->context_id
* IM_CTX_SIZE
+ reg
, data
);
249 /* -----------------------------------------------------------------------------
250 * TLB and microTLB Management
253 /* Wait for any pending TLB invalidations to complete */
254 static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain
*domain
)
256 unsigned int count
= 0;
258 while (ipmmu_ctx_read(domain
, IMCTR
) & IMCTR_FLUSH
) {
260 if (++count
== TLB_LOOP_TIMEOUT
) {
261 dev_err_ratelimited(domain
->mmu
->dev
,
262 "TLB sync timed out -- MMU may be deadlocked\n");
269 static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain
*domain
)
273 reg
= ipmmu_ctx_read(domain
, IMCTR
);
275 ipmmu_ctx_write(domain
, IMCTR
, reg
);
277 ipmmu_tlb_sync(domain
);
281 * Enable MMU translation for the microTLB.
283 static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain
*domain
,
286 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
289 * TODO: Reference-count the microTLB as several bus masters can be
290 * connected to the same microTLB.
293 /* TODO: What should we set the ASID to ? */
294 ipmmu_write(mmu
, IMUASID(utlb
), 0);
295 /* TODO: Do we need to flush the microTLB ? */
296 ipmmu_write(mmu
, IMUCTR(utlb
),
297 IMUCTR_TTSEL_MMU(domain
->context_id
) | IMUCTR_FLUSH
|
302 * Disable MMU translation for the microTLB.
304 static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain
*domain
,
307 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
309 ipmmu_write(mmu
, IMUCTR(utlb
), 0);
312 static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device
*mmu
, void *addr
,
315 unsigned long offset
= (unsigned long)addr
& ~PAGE_MASK
;
318 * TODO: Add support for coherent walk through CCI with DVM and remove
321 dma_map_page(mmu
->dev
, virt_to_page(addr
), offset
, size
, DMA_TO_DEVICE
);
324 /* -----------------------------------------------------------------------------
325 * Domain/Context Management
328 static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain
*domain
)
334 * TODO: When adding support for multiple contexts, find an unused
337 domain
->context_id
= 0;
340 ipmmu_flush_pgtable(domain
->mmu
, domain
->pgd
,
341 IPMMU_PTRS_PER_PGD
* sizeof(*domain
->pgd
));
342 ttbr
= __pa(domain
->pgd
);
343 ipmmu_ctx_write(domain
, IMTTLBR0
, ttbr
);
344 ipmmu_ctx_write(domain
, IMTTUBR0
, ttbr
>> 32);
348 * We use long descriptors with inner-shareable WBWA tables and allocate
349 * the whole 32-bit VA space to TTBR0.
351 ipmmu_ctx_write(domain
, IMTTBCR
, IMTTBCR_EAE
|
352 IMTTBCR_SH0_INNER_SHAREABLE
| IMTTBCR_ORGN0_WB_WA
|
353 IMTTBCR_IRGN0_WB_WA
| IMTTBCR_SL0_LVL_1
);
357 * We need three attributes only, non-cacheable, write-back read/write
358 * allocate and device memory.
360 reg
= (IMMAIR_ATTR_NC
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC
))
361 | (IMMAIR_ATTR_WBRWA
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA
))
362 | (IMMAIR_ATTR_DEVICE
<< IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV
));
363 ipmmu_ctx_write(domain
, IMMAIR0
, reg
);
366 ipmmu_ctx_write(domain
, IMBUSCR
,
367 ipmmu_ctx_read(domain
, IMBUSCR
) &
368 ~(IMBUSCR_DVM
| IMBUSCR_BUSSEL_MASK
));
372 * Clear all interrupt flags.
374 ipmmu_ctx_write(domain
, IMSTR
, ipmmu_ctx_read(domain
, IMSTR
));
378 * Enable the MMU and interrupt generation. The long-descriptor
379 * translation table format doesn't use TEX remapping. Don't enable AF
380 * software management as we have no use for it. Flush the TLB as
381 * required when modifying the context registers.
383 ipmmu_ctx_write(domain
, IMCTR
, IMCTR_INTEN
| IMCTR_FLUSH
| IMCTR_MMUEN
);
388 static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain
*domain
)
391 * Disable the context. Flush the TLB as required when modifying the
394 * TODO: Is TLB flush really needed ?
396 ipmmu_ctx_write(domain
, IMCTR
, IMCTR_FLUSH
);
397 ipmmu_tlb_sync(domain
);
400 /* -----------------------------------------------------------------------------
404 static irqreturn_t
ipmmu_domain_irq(struct ipmmu_vmsa_domain
*domain
)
406 const u32 err_mask
= IMSTR_MHIT
| IMSTR_ABORT
| IMSTR_PF
| IMSTR_TF
;
407 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
411 status
= ipmmu_ctx_read(domain
, IMSTR
);
412 if (!(status
& err_mask
))
415 iova
= ipmmu_ctx_read(domain
, IMEAR
);
418 * Clear the error status flags. Unlike traditional interrupt flag
419 * registers that must be cleared by writing 1, this status register
420 * seems to require 0. The error address register must be read before,
421 * otherwise its value will be 0.
423 ipmmu_ctx_write(domain
, IMSTR
, 0);
425 /* Log fatal errors. */
426 if (status
& IMSTR_MHIT
)
427 dev_err_ratelimited(mmu
->dev
, "Multiple TLB hits @0x%08x\n",
429 if (status
& IMSTR_ABORT
)
430 dev_err_ratelimited(mmu
->dev
, "Page Table Walk Abort @0x%08x\n",
433 if (!(status
& (IMSTR_PF
| IMSTR_TF
)))
437 * Try to handle page faults and translation faults.
439 * TODO: We need to look up the faulty device based on the I/O VA. Use
440 * the IOMMU device for now.
442 if (!report_iommu_fault(domain
->io_domain
, mmu
->dev
, iova
, 0))
445 dev_err_ratelimited(mmu
->dev
,
446 "Unhandled fault: status 0x%08x iova 0x%08x\n",
452 static irqreturn_t
ipmmu_irq(int irq
, void *dev
)
454 struct ipmmu_vmsa_device
*mmu
= dev
;
455 struct iommu_domain
*io_domain
;
456 struct ipmmu_vmsa_domain
*domain
;
461 io_domain
= mmu
->mapping
->domain
;
462 domain
= io_domain
->priv
;
464 return ipmmu_domain_irq(domain
);
467 /* -----------------------------------------------------------------------------
468 * Page Table Management
471 #define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
473 static void ipmmu_free_ptes(pmd_t
*pmd
)
475 pgtable_t table
= pmd_pgtable(*pmd
);
479 static void ipmmu_free_pmds(pud_t
*pud
)
481 pmd_t
*pmd
= pmd_offset(pud
, 0);
485 for (i
= 0; i
< IPMMU_PTRS_PER_PMD
; ++i
) {
486 if (!pmd_table(*pmd
))
489 ipmmu_free_ptes(pmd
);
493 table
= pud_pgtable(*pud
);
497 static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain
*domain
)
499 pgd_t
*pgd
, *pgd_base
= domain
->pgd
;
503 * Recursively free the page tables for this domain. We don't care about
504 * speculative TLB filling, because the TLB will be nuked next time this
505 * context bank is re-allocated and no devices currently map to these
509 for (i
= 0; i
< IPMMU_PTRS_PER_PGD
; ++i
) {
512 ipmmu_free_pmds((pud_t
*)pgd
);
520 * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte)
521 * functions as they would flush the CPU TLB.
524 static pte_t
*ipmmu_alloc_pte(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
530 return pte_offset_kernel(pmd
, iova
);
532 pte
= (pte_t
*)get_zeroed_page(GFP_ATOMIC
);
536 ipmmu_flush_pgtable(mmu
, pte
, PAGE_SIZE
);
537 *pmd
= __pmd(__pa(pte
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
538 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
540 return pte
+ pte_index(iova
);
543 static pmd_t
*ipmmu_alloc_pmd(struct ipmmu_vmsa_device
*mmu
, pgd_t
*pgd
,
546 pud_t
*pud
= (pud_t
*)pgd
;
550 return pmd_offset(pud
, iova
);
552 pmd
= (pmd_t
*)get_zeroed_page(GFP_ATOMIC
);
556 ipmmu_flush_pgtable(mmu
, pmd
, PAGE_SIZE
);
557 *pud
= __pud(__pa(pmd
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
558 ipmmu_flush_pgtable(mmu
, pud
, sizeof(*pud
));
560 return pmd
+ pmd_index(iova
);
563 static u64
ipmmu_page_prot(unsigned int prot
, u64 type
)
565 u64 pgprot
= ARM_VMSA_PTE_nG
| ARM_VMSA_PTE_AF
566 | ARM_VMSA_PTE_SH_IS
| ARM_VMSA_PTE_AP_UNPRIV
567 | ARM_VMSA_PTE_NS
| type
;
569 if (!(prot
& IOMMU_WRITE
) && (prot
& IOMMU_READ
))
570 pgprot
|= ARM_VMSA_PTE_AP_RDONLY
;
572 if (prot
& IOMMU_CACHE
)
573 pgprot
|= IMMAIR_ATTR_IDX_WBRWA
<< ARM_VMSA_PTE_ATTRINDX_SHIFT
;
575 if (prot
& IOMMU_NOEXEC
)
576 pgprot
|= ARM_VMSA_PTE_XN
;
577 else if (!(prot
& (IOMMU_READ
| IOMMU_WRITE
)))
578 /* If no access create a faulting entry to avoid TLB fills. */
579 pgprot
&= ~ARM_VMSA_PTE_PAGE
;
584 static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
585 unsigned long iova
, unsigned long pfn
,
586 size_t size
, int prot
)
588 pteval_t pteval
= ipmmu_page_prot(prot
, ARM_VMSA_PTE_PAGE
);
589 unsigned int num_ptes
= 1;
593 pte
= ipmmu_alloc_pte(mmu
, pmd
, iova
);
600 * Install the page table entries. We can be called both for a single
601 * page or for a block of 16 physically contiguous pages. In the latter
602 * case set the PTE contiguous hint.
604 if (size
== SZ_64K
) {
605 pteval
|= ARM_VMSA_PTE_CONT
;
606 num_ptes
= ARM_VMSA_PTE_CONT_ENTRIES
;
609 for (i
= num_ptes
; i
; --i
)
610 *pte
++ = pfn_pte(pfn
++, __pgprot(pteval
));
612 ipmmu_flush_pgtable(mmu
, start
, sizeof(*pte
) * num_ptes
);
617 static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
,
618 unsigned long iova
, unsigned long pfn
,
621 pmdval_t pmdval
= ipmmu_page_prot(prot
, PMD_TYPE_SECT
);
623 *pmd
= pfn_pmd(pfn
, __pgprot(pmdval
));
624 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
629 static int ipmmu_create_mapping(struct ipmmu_vmsa_domain
*domain
,
630 unsigned long iova
, phys_addr_t paddr
,
631 size_t size
, int prot
)
633 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
634 pgd_t
*pgd
= domain
->pgd
;
643 if (size
& ~PAGE_MASK
)
646 if (paddr
& ~((1ULL << 40) - 1))
649 pfn
= __phys_to_pfn(paddr
);
650 pgd
+= pgd_index(iova
);
652 /* Update the page tables. */
653 spin_lock_irqsave(&domain
->lock
, flags
);
655 pmd
= ipmmu_alloc_pmd(mmu
, pgd
, iova
);
663 ret
= ipmmu_alloc_init_pmd(mmu
, pmd
, iova
, pfn
, prot
);
667 ret
= ipmmu_alloc_init_pte(mmu
, pmd
, iova
, pfn
, size
, prot
);
675 spin_unlock_irqrestore(&domain
->lock
, flags
);
678 ipmmu_tlb_invalidate(domain
);
683 static void ipmmu_clear_pud(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
)
685 pgtable_t table
= pud_pgtable(*pud
);
689 ipmmu_flush_pgtable(mmu
, pud
, sizeof(*pud
));
691 /* Free the page table. */
695 static void ipmmu_clear_pmd(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
,
703 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
705 /* Free the page table. */
706 if (pmd_table(pmdval
)) {
707 pgtable_t table
= pmd_pgtable(pmdval
);
712 /* Check whether the PUD is still needed. */
713 pmd
= pmd_offset(pud
, 0);
714 for (i
= 0; i
< IPMMU_PTRS_PER_PMD
; ++i
) {
715 if (!pmd_none(pmd
[i
]))
719 /* Clear the parent PUD. */
720 ipmmu_clear_pud(mmu
, pud
);
723 static void ipmmu_clear_pte(struct ipmmu_vmsa_device
*mmu
, pud_t
*pud
,
724 pmd_t
*pmd
, pte_t
*pte
, unsigned int num_ptes
)
729 for (i
= num_ptes
; i
; --i
)
732 ipmmu_flush_pgtable(mmu
, pte
, sizeof(*pte
) * num_ptes
);
734 /* Check whether the PMD is still needed. */
735 pte
= pte_offset_kernel(pmd
, 0);
736 for (i
= 0; i
< IPMMU_PTRS_PER_PTE
; ++i
) {
737 if (!pte_none(pte
[i
]))
741 /* Clear the parent PMD. */
742 ipmmu_clear_pmd(mmu
, pud
, pmd
);
745 static int ipmmu_split_pmd(struct ipmmu_vmsa_device
*mmu
, pmd_t
*pmd
)
752 pte
= (pte_t
*)get_zeroed_page(GFP_ATOMIC
);
756 /* Copy the PMD attributes. */
757 pteval
= (pmd_val(*pmd
) & ARM_VMSA_PTE_ATTRS_MASK
)
758 | ARM_VMSA_PTE_CONT
| ARM_VMSA_PTE_PAGE
;
763 for (i
= IPMMU_PTRS_PER_PTE
; i
; --i
)
764 *pte
++ = pfn_pte(pfn
++, __pgprot(pteval
));
766 ipmmu_flush_pgtable(mmu
, start
, PAGE_SIZE
);
767 *pmd
= __pmd(__pa(start
) | PMD_NSTABLE
| PMD_TYPE_TABLE
);
768 ipmmu_flush_pgtable(mmu
, pmd
, sizeof(*pmd
));
773 static void ipmmu_split_pte(struct ipmmu_vmsa_device
*mmu
, pte_t
*pte
)
777 for (i
= ARM_VMSA_PTE_CONT_ENTRIES
; i
; --i
)
778 pte
[i
-1] = __pte(pte_val(*pte
) & ~ARM_VMSA_PTE_CONT
);
780 ipmmu_flush_pgtable(mmu
, pte
, sizeof(*pte
) * ARM_VMSA_PTE_CONT_ENTRIES
);
783 static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain
*domain
,
784 unsigned long iova
, size_t size
)
786 struct ipmmu_vmsa_device
*mmu
= domain
->mmu
;
788 pgd_t
*pgd
= domain
->pgd
;
796 if (size
& ~PAGE_MASK
)
799 pgd
+= pgd_index(iova
);
802 spin_lock_irqsave(&domain
->lock
, flags
);
804 /* If there's no PUD or PMD we're done. */
808 pmd
= pmd_offset(pud
, iova
);
813 * When freeing a 2MB block just clear the PMD. In the unlikely case the
814 * block is mapped as individual pages this will free the corresponding
818 ipmmu_clear_pmd(mmu
, pud
, pmd
);
823 * If the PMD has been mapped as a section remap it as pages to allow
824 * freeing individual pages.
827 ipmmu_split_pmd(mmu
, pmd
);
829 pte
= pte_offset_kernel(pmd
, iova
);
832 * When freeing a 64kB block just clear the PTE entries. We don't have
833 * to care about the contiguous hint of the surrounding entries.
835 if (size
== SZ_64K
) {
836 ipmmu_clear_pte(mmu
, pud
, pmd
, pte
, ARM_VMSA_PTE_CONT_ENTRIES
);
841 * If the PTE has been mapped with the contiguous hint set remap it and
842 * its surrounding PTEs to allow unmapping a single page.
844 if (pte_val(*pte
) & ARM_VMSA_PTE_CONT
)
845 ipmmu_split_pte(mmu
, pte
);
848 ipmmu_clear_pte(mmu
, pud
, pmd
, pte
, 1);
851 spin_unlock_irqrestore(&domain
->lock
, flags
);
853 ipmmu_tlb_invalidate(domain
);
858 /* -----------------------------------------------------------------------------
862 static int ipmmu_domain_init(struct iommu_domain
*io_domain
)
864 struct ipmmu_vmsa_domain
*domain
;
866 domain
= kzalloc(sizeof(*domain
), GFP_KERNEL
);
870 spin_lock_init(&domain
->lock
);
872 domain
->pgd
= kzalloc(IPMMU_PTRS_PER_PGD
* sizeof(pgd_t
), GFP_KERNEL
);
878 io_domain
->priv
= domain
;
879 domain
->io_domain
= io_domain
;
884 static void ipmmu_domain_destroy(struct iommu_domain
*io_domain
)
886 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
889 * Free the domain resources. We assume that all devices have already
892 ipmmu_domain_destroy_context(domain
);
893 ipmmu_free_pgtables(domain
);
897 static int ipmmu_attach_device(struct iommu_domain
*io_domain
,
900 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
901 struct ipmmu_vmsa_device
*mmu
= archdata
->mmu
;
902 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
908 dev_err(dev
, "Cannot attach to IPMMU\n");
912 spin_lock_irqsave(&domain
->lock
, flags
);
915 /* The domain hasn't been used yet, initialize it. */
917 ret
= ipmmu_domain_init_context(domain
);
918 } else if (domain
->mmu
!= mmu
) {
920 * Something is wrong, we can't attach two devices using
921 * different IOMMUs to the same domain.
923 dev_err(dev
, "Can't attach IPMMU %s to domain on IPMMU %s\n",
924 dev_name(mmu
->dev
), dev_name(domain
->mmu
->dev
));
928 spin_unlock_irqrestore(&domain
->lock
, flags
);
933 for (i
= 0; i
< archdata
->num_utlbs
; ++i
)
934 ipmmu_utlb_enable(domain
, archdata
->utlbs
[i
]);
939 static void ipmmu_detach_device(struct iommu_domain
*io_domain
,
942 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
943 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
946 for (i
= 0; i
< archdata
->num_utlbs
; ++i
)
947 ipmmu_utlb_disable(domain
, archdata
->utlbs
[i
]);
950 * TODO: Optimize by disabling the context when no device is attached.
954 static int ipmmu_map(struct iommu_domain
*io_domain
, unsigned long iova
,
955 phys_addr_t paddr
, size_t size
, int prot
)
957 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
962 return ipmmu_create_mapping(domain
, iova
, paddr
, size
, prot
);
965 static size_t ipmmu_unmap(struct iommu_domain
*io_domain
, unsigned long iova
,
968 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
971 ret
= ipmmu_clear_mapping(domain
, iova
, size
);
972 return ret
? 0 : size
;
975 static phys_addr_t
ipmmu_iova_to_phys(struct iommu_domain
*io_domain
,
978 struct ipmmu_vmsa_domain
*domain
= io_domain
->priv
;
984 /* TODO: Is locking needed ? */
989 pgd
= *(domain
->pgd
+ pgd_index(iova
));
993 pud
= *pud_offset(&pgd
, iova
);
997 pmd
= *pmd_offset(&pud
, iova
);
1002 return __pfn_to_phys(pmd_pfn(pmd
)) | (iova
& ~PMD_MASK
);
1004 pte
= *(pmd_page_vaddr(pmd
) + pte_index(iova
));
1008 return __pfn_to_phys(pte_pfn(pte
)) | (iova
& ~PAGE_MASK
);
1011 static int ipmmu_find_utlbs(struct ipmmu_vmsa_device
*mmu
, struct device
*dev
,
1012 unsigned int **_utlbs
)
1014 unsigned int *utlbs
;
1019 const struct ipmmu_vmsa_master
*master
= mmu
->pdata
->masters
;
1020 const char *devname
= dev_name(dev
);
1023 for (i
= 0; i
< mmu
->pdata
->num_masters
; ++i
, ++master
) {
1024 if (strcmp(master
->name
, devname
) == 0) {
1025 utlbs
= kmalloc(sizeof(*utlbs
), GFP_KERNEL
);
1029 utlbs
[0] = master
->utlb
;
1039 count
= of_count_phandle_with_args(dev
->of_node
, "iommus",
1044 utlbs
= kcalloc(count
, sizeof(*utlbs
), GFP_KERNEL
);
1048 for (i
= 0; i
< count
; ++i
) {
1049 struct of_phandle_args args
;
1052 ret
= of_parse_phandle_with_args(dev
->of_node
, "iommus",
1053 "#iommu-cells", i
, &args
);
1057 of_node_put(args
.np
);
1059 if (args
.np
!= mmu
->dev
->of_node
|| args
.args_count
!= 1)
1062 utlbs
[i
] = args
.args
[0];
1074 static int ipmmu_add_device(struct device
*dev
)
1076 struct ipmmu_vmsa_archdata
*archdata
;
1077 struct ipmmu_vmsa_device
*mmu
;
1078 struct iommu_group
*group
= NULL
;
1079 unsigned int *utlbs
= NULL
;
1084 if (dev
->archdata
.iommu
) {
1085 dev_warn(dev
, "IOMMU driver already assigned to device %s\n",
1090 /* Find the master corresponding to the device. */
1091 spin_lock(&ipmmu_devices_lock
);
1093 list_for_each_entry(mmu
, &ipmmu_devices
, list
) {
1094 num_utlbs
= ipmmu_find_utlbs(mmu
, dev
, &utlbs
);
1097 * TODO Take a reference to the MMU to protect
1098 * against device removal.
1104 spin_unlock(&ipmmu_devices_lock
);
1109 for (i
= 0; i
< num_utlbs
; ++i
) {
1110 if (utlbs
[i
] >= mmu
->num_utlbs
) {
1116 /* Create a device group and add the device to it. */
1117 group
= iommu_group_alloc();
1118 if (IS_ERR(group
)) {
1119 dev_err(dev
, "Failed to allocate IOMMU group\n");
1120 ret
= PTR_ERR(group
);
1124 ret
= iommu_group_add_device(group
, dev
);
1125 iommu_group_put(group
);
1128 dev_err(dev
, "Failed to add device to IPMMU group\n");
1133 archdata
= kzalloc(sizeof(*archdata
), GFP_KERNEL
);
1139 archdata
->mmu
= mmu
;
1140 archdata
->utlbs
= utlbs
;
1141 archdata
->num_utlbs
= num_utlbs
;
1142 dev
->archdata
.iommu
= archdata
;
1145 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
1146 * VAs. This will allocate a corresponding IOMMU domain.
1149 * - Create one mapping per context (TLB).
1150 * - Make the mapping size configurable ? We currently use a 2GB mapping
1151 * at a 1GB offset to ensure that NULL VAs will fault.
1153 if (!mmu
->mapping
) {
1154 struct dma_iommu_mapping
*mapping
;
1156 mapping
= arm_iommu_create_mapping(&platform_bus_type
,
1158 if (IS_ERR(mapping
)) {
1159 dev_err(mmu
->dev
, "failed to create ARM IOMMU mapping\n");
1160 ret
= PTR_ERR(mapping
);
1164 mmu
->mapping
= mapping
;
1167 /* Attach the ARM VA mapping to the device. */
1168 ret
= arm_iommu_attach_device(dev
, mmu
->mapping
);
1170 dev_err(dev
, "Failed to attach device to VA mapping\n");
1177 arm_iommu_release_mapping(mmu
->mapping
);
1179 kfree(dev
->archdata
.iommu
);
1182 dev
->archdata
.iommu
= NULL
;
1184 if (!IS_ERR_OR_NULL(group
))
1185 iommu_group_remove_device(dev
);
1190 static void ipmmu_remove_device(struct device
*dev
)
1192 struct ipmmu_vmsa_archdata
*archdata
= dev
->archdata
.iommu
;
1194 arm_iommu_detach_device(dev
);
1195 iommu_group_remove_device(dev
);
1197 kfree(archdata
->utlbs
);
1200 dev
->archdata
.iommu
= NULL
;
1203 static const struct iommu_ops ipmmu_ops
= {
1204 .domain_init
= ipmmu_domain_init
,
1205 .domain_destroy
= ipmmu_domain_destroy
,
1206 .attach_dev
= ipmmu_attach_device
,
1207 .detach_dev
= ipmmu_detach_device
,
1209 .unmap
= ipmmu_unmap
,
1210 .map_sg
= default_iommu_map_sg
,
1211 .iova_to_phys
= ipmmu_iova_to_phys
,
1212 .add_device
= ipmmu_add_device
,
1213 .remove_device
= ipmmu_remove_device
,
1214 .pgsize_bitmap
= SZ_2M
| SZ_64K
| SZ_4K
,
1217 /* -----------------------------------------------------------------------------
1218 * Probe/remove and init
1221 static void ipmmu_device_reset(struct ipmmu_vmsa_device
*mmu
)
1225 /* Disable all contexts. */
1226 for (i
= 0; i
< 4; ++i
)
1227 ipmmu_write(mmu
, i
* IM_CTX_SIZE
+ IMCTR
, 0);
1230 static int ipmmu_probe(struct platform_device
*pdev
)
1232 struct ipmmu_vmsa_device
*mmu
;
1233 struct resource
*res
;
1237 if (!IS_ENABLED(CONFIG_OF
) && !pdev
->dev
.platform_data
) {
1238 dev_err(&pdev
->dev
, "missing platform data\n");
1242 mmu
= devm_kzalloc(&pdev
->dev
, sizeof(*mmu
), GFP_KERNEL
);
1244 dev_err(&pdev
->dev
, "cannot allocate device data\n");
1248 mmu
->dev
= &pdev
->dev
;
1249 mmu
->pdata
= pdev
->dev
.platform_data
;
1250 mmu
->num_utlbs
= 32;
1252 /* Map I/O memory and request IRQ. */
1253 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1254 mmu
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1255 if (IS_ERR(mmu
->base
))
1256 return PTR_ERR(mmu
->base
);
1259 * The IPMMU has two register banks, for secure and non-secure modes.
1260 * The bank mapped at the beginning of the IPMMU address space
1261 * corresponds to the running mode of the CPU. When running in secure
1262 * mode the non-secure register bank is also available at an offset.
1264 * Secure mode operation isn't clearly documented and is thus currently
1265 * not implemented in the driver. Furthermore, preliminary tests of
1266 * non-secure operation with the main register bank were not successful.
1267 * Offset the registers base unconditionally to point to the non-secure
1268 * alias space for now.
1270 mmu
->base
+= IM_NS_ALIAS_OFFSET
;
1272 irq
= platform_get_irq(pdev
, 0);
1274 dev_err(&pdev
->dev
, "no IRQ found\n");
1278 ret
= devm_request_irq(&pdev
->dev
, irq
, ipmmu_irq
, 0,
1279 dev_name(&pdev
->dev
), mmu
);
1281 dev_err(&pdev
->dev
, "failed to request IRQ %d\n", irq
);
1285 ipmmu_device_reset(mmu
);
1288 * We can't create the ARM mapping here as it requires the bus to have
1289 * an IOMMU, which only happens when bus_set_iommu() is called in
1290 * ipmmu_init() after the probe function returns.
1293 spin_lock(&ipmmu_devices_lock
);
1294 list_add(&mmu
->list
, &ipmmu_devices
);
1295 spin_unlock(&ipmmu_devices_lock
);
1297 platform_set_drvdata(pdev
, mmu
);
1302 static int ipmmu_remove(struct platform_device
*pdev
)
1304 struct ipmmu_vmsa_device
*mmu
= platform_get_drvdata(pdev
);
1306 spin_lock(&ipmmu_devices_lock
);
1307 list_del(&mmu
->list
);
1308 spin_unlock(&ipmmu_devices_lock
);
1310 arm_iommu_release_mapping(mmu
->mapping
);
1312 ipmmu_device_reset(mmu
);
1317 static const struct of_device_id ipmmu_of_ids
[] = {
1318 { .compatible
= "renesas,ipmmu-vmsa", },
1321 static struct platform_driver ipmmu_driver
= {
1323 .name
= "ipmmu-vmsa",
1324 .of_match_table
= of_match_ptr(ipmmu_of_ids
),
1326 .probe
= ipmmu_probe
,
1327 .remove
= ipmmu_remove
,
1330 static int __init
ipmmu_init(void)
1334 ret
= platform_driver_register(&ipmmu_driver
);
1338 if (!iommu_present(&platform_bus_type
))
1339 bus_set_iommu(&platform_bus_type
, &ipmmu_ops
);
1344 static void __exit
ipmmu_exit(void)
1346 return platform_driver_unregister(&ipmmu_driver
);
1349 subsys_initcall(ipmmu_init
);
1350 module_exit(ipmmu_exit
);
1352 MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
1353 MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
1354 MODULE_LICENSE("GPL v2");