1 /* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/kernel.h>
20 #include <linux/module.h>
21 #include <linux/platform_device.h>
22 #include <linux/errno.h>
24 #include <linux/interrupt.h>
25 #include <linux/list.h>
26 #include <linux/spinlock.h>
27 #include <linux/slab.h>
28 #include <linux/iommu.h>
29 #include <linux/clk.h>
31 #include <asm/cacheflush.h>
32 #include <asm/sizes.h>
34 #include <mach/iommu_hw-8xxx.h>
35 #include <mach/iommu.h>
37 #define MRC(reg, processor, op1, crn, crm, op2) \
38 __asm__ __volatile__ ( \
39 " mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \
42 #define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0)
43 #define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1)
45 static int msm_iommu_tex_class
[4];
47 DEFINE_SPINLOCK(msm_iommu_lock
);
50 unsigned long *pgtable
;
51 struct list_head list_attached
;
54 static int __enable_clocks(struct msm_iommu_drvdata
*drvdata
)
58 ret
= clk_enable(drvdata
->pclk
);
63 ret
= clk_enable(drvdata
->clk
);
65 clk_disable(drvdata
->pclk
);
71 static void __disable_clocks(struct msm_iommu_drvdata
*drvdata
)
74 clk_disable(drvdata
->clk
);
75 clk_disable(drvdata
->pclk
);
78 static int __flush_iotlb(struct iommu_domain
*domain
)
80 struct msm_priv
*priv
= domain
->priv
;
81 struct msm_iommu_drvdata
*iommu_drvdata
;
82 struct msm_iommu_ctx_drvdata
*ctx_drvdata
;
84 #ifndef CONFIG_IOMMU_PGTABLES_L2
85 unsigned long *fl_table
= priv
->pgtable
;
88 if (!list_empty(&priv
->list_attached
)) {
89 dmac_flush_range(fl_table
, fl_table
+ SZ_16K
);
91 for (i
= 0; i
< NUM_FL_PTE
; i
++)
92 if ((fl_table
[i
] & 0x03) == FL_TYPE_TABLE
) {
93 void *sl_table
= __va(fl_table
[i
] &
95 dmac_flush_range(sl_table
, sl_table
+ SZ_4K
);
100 list_for_each_entry(ctx_drvdata
, &priv
->list_attached
, attached_elm
) {
101 if (!ctx_drvdata
->pdev
|| !ctx_drvdata
->pdev
->dev
.parent
)
104 iommu_drvdata
= dev_get_drvdata(ctx_drvdata
->pdev
->dev
.parent
);
105 BUG_ON(!iommu_drvdata
);
107 ret
= __enable_clocks(iommu_drvdata
);
111 SET_CTX_TLBIALL(iommu_drvdata
->base
, ctx_drvdata
->num
, 0);
112 __disable_clocks(iommu_drvdata
);
118 static void __reset_context(void __iomem
*base
, int ctx
)
120 SET_BPRCOSH(base
, ctx
, 0);
121 SET_BPRCISH(base
, ctx
, 0);
122 SET_BPRCNSH(base
, ctx
, 0);
123 SET_BPSHCFG(base
, ctx
, 0);
124 SET_BPMTCFG(base
, ctx
, 0);
125 SET_ACTLR(base
, ctx
, 0);
126 SET_SCTLR(base
, ctx
, 0);
127 SET_FSRRESTORE(base
, ctx
, 0);
128 SET_TTBR0(base
, ctx
, 0);
129 SET_TTBR1(base
, ctx
, 0);
130 SET_TTBCR(base
, ctx
, 0);
131 SET_BFBCR(base
, ctx
, 0);
132 SET_PAR(base
, ctx
, 0);
133 SET_FAR(base
, ctx
, 0);
134 SET_CTX_TLBIALL(base
, ctx
, 0);
135 SET_TLBFLPTER(base
, ctx
, 0);
136 SET_TLBSLPTER(base
, ctx
, 0);
137 SET_TLBLKCR(base
, ctx
, 0);
138 SET_PRRR(base
, ctx
, 0);
139 SET_NMRR(base
, ctx
, 0);
142 static void __program_context(void __iomem
*base
, int ctx
, phys_addr_t pgtable
)
144 unsigned int prrr
, nmrr
;
145 __reset_context(base
, ctx
);
147 /* Set up HTW mode */
148 /* TLB miss configuration: perform HTW on miss */
149 SET_TLBMCFG(base
, ctx
, 0x3);
151 /* V2P configuration: HTW for access */
152 SET_V2PCFG(base
, ctx
, 0x3);
154 SET_TTBCR(base
, ctx
, 0);
155 SET_TTBR0_PA(base
, ctx
, (pgtable
>> 14));
157 /* Invalidate the TLB for this context */
158 SET_CTX_TLBIALL(base
, ctx
, 0);
160 /* Set interrupt number to "secure" interrupt */
161 SET_IRPTNDX(base
, ctx
, 0);
163 /* Enable context fault interrupt */
164 SET_CFEIE(base
, ctx
, 1);
166 /* Stall access on a context fault and let the handler deal with it */
167 SET_CFCFG(base
, ctx
, 1);
169 /* Redirect all cacheable requests to L2 slave port. */
170 SET_RCISH(base
, ctx
, 1);
171 SET_RCOSH(base
, ctx
, 1);
172 SET_RCNSH(base
, ctx
, 1);
174 /* Turn on TEX Remap */
175 SET_TRE(base
, ctx
, 1);
177 /* Set TEX remap attributes */
180 SET_PRRR(base
, ctx
, prrr
);
181 SET_NMRR(base
, ctx
, nmrr
);
183 /* Turn on BFB prefetch */
184 SET_BFBDFE(base
, ctx
, 1);
186 #ifdef CONFIG_IOMMU_PGTABLES_L2
187 /* Configure page tables as inner-cacheable and shareable to reduce
188 * the TLB miss penalty.
190 SET_TTBR0_SH(base
, ctx
, 1);
191 SET_TTBR1_SH(base
, ctx
, 1);
193 SET_TTBR0_NOS(base
, ctx
, 1);
194 SET_TTBR1_NOS(base
, ctx
, 1);
196 SET_TTBR0_IRGNH(base
, ctx
, 0); /* WB, WA */
197 SET_TTBR0_IRGNL(base
, ctx
, 1);
199 SET_TTBR1_IRGNH(base
, ctx
, 0); /* WB, WA */
200 SET_TTBR1_IRGNL(base
, ctx
, 1);
202 SET_TTBR0_ORGN(base
, ctx
, 1); /* WB, WA */
203 SET_TTBR1_ORGN(base
, ctx
, 1); /* WB, WA */
210 static int msm_iommu_domain_init(struct iommu_domain
*domain
)
212 struct msm_priv
*priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
217 INIT_LIST_HEAD(&priv
->list_attached
);
218 priv
->pgtable
= (unsigned long *)__get_free_pages(GFP_KERNEL
,
224 memset(priv
->pgtable
, 0, SZ_16K
);
233 static void msm_iommu_domain_destroy(struct iommu_domain
*domain
)
235 struct msm_priv
*priv
;
237 unsigned long *fl_table
;
240 spin_lock_irqsave(&msm_iommu_lock
, flags
);
245 fl_table
= priv
->pgtable
;
247 for (i
= 0; i
< NUM_FL_PTE
; i
++)
248 if ((fl_table
[i
] & 0x03) == FL_TYPE_TABLE
)
249 free_page((unsigned long) __va(((fl_table
[i
]) &
252 free_pages((unsigned long)priv
->pgtable
, get_order(SZ_16K
));
253 priv
->pgtable
= NULL
;
257 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
260 static int msm_iommu_attach_dev(struct iommu_domain
*domain
, struct device
*dev
)
262 struct msm_priv
*priv
;
263 struct msm_iommu_ctx_dev
*ctx_dev
;
264 struct msm_iommu_drvdata
*iommu_drvdata
;
265 struct msm_iommu_ctx_drvdata
*ctx_drvdata
;
266 struct msm_iommu_ctx_drvdata
*tmp_drvdata
;
270 spin_lock_irqsave(&msm_iommu_lock
, flags
);
279 iommu_drvdata
= dev_get_drvdata(dev
->parent
);
280 ctx_drvdata
= dev_get_drvdata(dev
);
281 ctx_dev
= dev
->platform_data
;
283 if (!iommu_drvdata
|| !ctx_drvdata
|| !ctx_dev
) {
288 if (!list_empty(&ctx_drvdata
->attached_elm
)) {
293 list_for_each_entry(tmp_drvdata
, &priv
->list_attached
, attached_elm
)
294 if (tmp_drvdata
== ctx_drvdata
) {
299 ret
= __enable_clocks(iommu_drvdata
);
303 __program_context(iommu_drvdata
->base
, ctx_dev
->num
,
304 __pa(priv
->pgtable
));
306 __disable_clocks(iommu_drvdata
);
307 list_add(&(ctx_drvdata
->attached_elm
), &priv
->list_attached
);
308 ret
= __flush_iotlb(domain
);
311 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
315 static void msm_iommu_detach_dev(struct iommu_domain
*domain
,
318 struct msm_priv
*priv
;
319 struct msm_iommu_ctx_dev
*ctx_dev
;
320 struct msm_iommu_drvdata
*iommu_drvdata
;
321 struct msm_iommu_ctx_drvdata
*ctx_drvdata
;
325 spin_lock_irqsave(&msm_iommu_lock
, flags
);
331 iommu_drvdata
= dev_get_drvdata(dev
->parent
);
332 ctx_drvdata
= dev_get_drvdata(dev
);
333 ctx_dev
= dev
->platform_data
;
335 if (!iommu_drvdata
|| !ctx_drvdata
|| !ctx_dev
)
338 ret
= __flush_iotlb(domain
);
342 ret
= __enable_clocks(iommu_drvdata
);
346 __reset_context(iommu_drvdata
->base
, ctx_dev
->num
);
347 __disable_clocks(iommu_drvdata
);
348 list_del_init(&ctx_drvdata
->attached_elm
);
351 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
354 static int msm_iommu_map(struct iommu_domain
*domain
, unsigned long va
,
355 phys_addr_t pa
, size_t len
, int prot
)
357 struct msm_priv
*priv
;
359 unsigned long *fl_table
;
360 unsigned long *fl_pte
;
361 unsigned long fl_offset
;
362 unsigned long *sl_table
;
363 unsigned long *sl_pte
;
364 unsigned long sl_offset
;
366 int ret
= 0, tex
, sh
;
368 spin_lock_irqsave(&msm_iommu_lock
, flags
);
370 sh
= (prot
& MSM_IOMMU_ATTR_SH
) ? 1 : 0;
371 tex
= msm_iommu_tex_class
[prot
& MSM_IOMMU_CP_MASK
];
373 if (tex
< 0 || tex
> NUM_TEX_CLASS
- 1) {
384 fl_table
= priv
->pgtable
;
386 if (len
!= SZ_16M
&& len
!= SZ_1M
&&
387 len
!= SZ_64K
&& len
!= SZ_4K
) {
388 pr_debug("Bad size: %d\n", len
);
394 pr_debug("Null page table\n");
399 if (len
== SZ_16M
|| len
== SZ_1M
) {
400 pgprot
= sh
? FL_SHARED
: 0;
401 pgprot
|= tex
& 0x01 ? FL_BUFFERABLE
: 0;
402 pgprot
|= tex
& 0x02 ? FL_CACHEABLE
: 0;
403 pgprot
|= tex
& 0x04 ? FL_TEX0
: 0;
405 pgprot
= sh
? SL_SHARED
: 0;
406 pgprot
|= tex
& 0x01 ? SL_BUFFERABLE
: 0;
407 pgprot
|= tex
& 0x02 ? SL_CACHEABLE
: 0;
408 pgprot
|= tex
& 0x04 ? SL_TEX0
: 0;
411 fl_offset
= FL_OFFSET(va
); /* Upper 12 bits */
412 fl_pte
= fl_table
+ fl_offset
; /* int pointers, 4 bytes */
416 for (i
= 0; i
< 16; i
++)
417 *(fl_pte
+i
) = (pa
& 0xFF000000) | FL_SUPERSECTION
|
418 FL_AP_READ
| FL_AP_WRITE
| FL_TYPE_SECT
|
419 FL_SHARED
| FL_NG
| pgprot
;
423 *fl_pte
= (pa
& 0xFFF00000) | FL_AP_READ
| FL_AP_WRITE
| FL_NG
|
424 FL_TYPE_SECT
| FL_SHARED
| pgprot
;
426 /* Need a 2nd level table */
427 if ((len
== SZ_4K
|| len
== SZ_64K
) && (*fl_pte
) == 0) {
429 sl
= (unsigned long *) __get_free_pages(GFP_ATOMIC
,
433 pr_debug("Could not allocate second level table\n");
438 memset(sl
, 0, SZ_4K
);
439 *fl_pte
= ((((int)__pa(sl
)) & FL_BASE_MASK
) | FL_TYPE_TABLE
);
442 sl_table
= (unsigned long *) __va(((*fl_pte
) & FL_BASE_MASK
));
443 sl_offset
= SL_OFFSET(va
);
444 sl_pte
= sl_table
+ sl_offset
;
448 *sl_pte
= (pa
& SL_BASE_MASK_SMALL
) | SL_AP0
| SL_AP1
| SL_NG
|
449 SL_SHARED
| SL_TYPE_SMALL
| pgprot
;
454 for (i
= 0; i
< 16; i
++)
455 *(sl_pte
+i
) = (pa
& SL_BASE_MASK_LARGE
) | SL_AP0
|
456 SL_NG
| SL_AP1
| SL_SHARED
| SL_TYPE_LARGE
| pgprot
;
459 ret
= __flush_iotlb(domain
);
461 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
465 static size_t msm_iommu_unmap(struct iommu_domain
*domain
, unsigned long va
,
468 struct msm_priv
*priv
;
470 unsigned long *fl_table
;
471 unsigned long *fl_pte
;
472 unsigned long fl_offset
;
473 unsigned long *sl_table
;
474 unsigned long *sl_pte
;
475 unsigned long sl_offset
;
478 spin_lock_irqsave(&msm_iommu_lock
, flags
);
487 fl_table
= priv
->pgtable
;
489 if (len
!= SZ_16M
&& len
!= SZ_1M
&&
490 len
!= SZ_64K
&& len
!= SZ_4K
) {
491 pr_debug("Bad length: %d\n", len
);
497 pr_debug("Null page table\n");
502 fl_offset
= FL_OFFSET(va
); /* Upper 12 bits */
503 fl_pte
= fl_table
+ fl_offset
; /* int pointers, 4 bytes */
506 pr_debug("First level PTE is 0\n");
511 /* Unmap supersection */
513 for (i
= 0; i
< 16; i
++)
519 sl_table
= (unsigned long *) __va(((*fl_pte
) & FL_BASE_MASK
));
520 sl_offset
= SL_OFFSET(va
);
521 sl_pte
= sl_table
+ sl_offset
;
524 for (i
= 0; i
< 16; i
++)
531 if (len
== SZ_4K
|| len
== SZ_64K
) {
534 for (i
= 0; i
< NUM_SL_PTE
; i
++)
538 free_page((unsigned long)sl_table
);
543 ret
= __flush_iotlb(domain
);
546 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
548 /* the IOMMU API requires us to return how many bytes were unmapped */
553 static phys_addr_t
msm_iommu_iova_to_phys(struct iommu_domain
*domain
,
556 struct msm_priv
*priv
;
557 struct msm_iommu_drvdata
*iommu_drvdata
;
558 struct msm_iommu_ctx_drvdata
*ctx_drvdata
;
565 spin_lock_irqsave(&msm_iommu_lock
, flags
);
568 if (list_empty(&priv
->list_attached
))
571 ctx_drvdata
= list_entry(priv
->list_attached
.next
,
572 struct msm_iommu_ctx_drvdata
, attached_elm
);
573 iommu_drvdata
= dev_get_drvdata(ctx_drvdata
->pdev
->dev
.parent
);
575 base
= iommu_drvdata
->base
;
576 ctx
= ctx_drvdata
->num
;
578 ret
= __enable_clocks(iommu_drvdata
);
582 /* Invalidate context TLB */
583 SET_CTX_TLBIALL(base
, ctx
, 0);
584 SET_V2PPR(base
, ctx
, va
& V2Pxx_VA
);
586 par
= GET_PAR(base
, ctx
);
588 /* We are dealing with a supersection */
589 if (GET_NOFAULT_SS(base
, ctx
))
590 ret
= (par
& 0xFF000000) | (va
& 0x00FFFFFF);
591 else /* Upper 20 bits from PAR, lower 12 from VA */
592 ret
= (par
& 0xFFFFF000) | (va
& 0x00000FFF);
594 if (GET_FAULT(base
, ctx
))
597 __disable_clocks(iommu_drvdata
);
599 spin_unlock_irqrestore(&msm_iommu_lock
, flags
);
603 static int msm_iommu_domain_has_cap(struct iommu_domain
*domain
,
609 static void print_ctx_regs(void __iomem
*base
, int ctx
)
611 unsigned int fsr
= GET_FSR(base
, ctx
);
612 pr_err("FAR = %08x PAR = %08x\n",
613 GET_FAR(base
, ctx
), GET_PAR(base
, ctx
));
614 pr_err("FSR = %08x [%s%s%s%s%s%s%s%s%s%s]\n", fsr
,
615 (fsr
& 0x02) ? "TF " : "",
616 (fsr
& 0x04) ? "AFF " : "",
617 (fsr
& 0x08) ? "APF " : "",
618 (fsr
& 0x10) ? "TLBMF " : "",
619 (fsr
& 0x20) ? "HTWDEEF " : "",
620 (fsr
& 0x40) ? "HTWSEEF " : "",
621 (fsr
& 0x80) ? "MHF " : "",
622 (fsr
& 0x10000) ? "SL " : "",
623 (fsr
& 0x40000000) ? "SS " : "",
624 (fsr
& 0x80000000) ? "MULTI " : "");
626 pr_err("FSYNR0 = %08x FSYNR1 = %08x\n",
627 GET_FSYNR0(base
, ctx
), GET_FSYNR1(base
, ctx
));
628 pr_err("TTBR0 = %08x TTBR1 = %08x\n",
629 GET_TTBR0(base
, ctx
), GET_TTBR1(base
, ctx
));
630 pr_err("SCTLR = %08x ACTLR = %08x\n",
631 GET_SCTLR(base
, ctx
), GET_ACTLR(base
, ctx
));
632 pr_err("PRRR = %08x NMRR = %08x\n",
633 GET_PRRR(base
, ctx
), GET_NMRR(base
, ctx
));
636 irqreturn_t
msm_iommu_fault_handler(int irq
, void *dev_id
)
638 struct msm_iommu_drvdata
*drvdata
= dev_id
;
643 spin_lock(&msm_iommu_lock
);
646 pr_err("Invalid device ID in context interrupt handler\n");
650 base
= drvdata
->base
;
652 pr_err("Unexpected IOMMU page fault!\n");
653 pr_err("base = %08x\n", (unsigned int) base
);
655 ret
= __enable_clocks(drvdata
);
659 for (i
= 0; i
< drvdata
->ncb
; i
++) {
660 fsr
= GET_FSR(base
, i
);
662 pr_err("Fault occurred in context %d.\n", i
);
663 pr_err("Interesting registers:\n");
664 print_ctx_regs(base
, i
);
665 SET_FSR(base
, i
, 0x4000000F);
668 __disable_clocks(drvdata
);
670 spin_unlock(&msm_iommu_lock
);
674 static struct iommu_ops msm_iommu_ops
= {
675 .domain_init
= msm_iommu_domain_init
,
676 .domain_destroy
= msm_iommu_domain_destroy
,
677 .attach_dev
= msm_iommu_attach_dev
,
678 .detach_dev
= msm_iommu_detach_dev
,
679 .map
= msm_iommu_map
,
680 .unmap
= msm_iommu_unmap
,
681 .iova_to_phys
= msm_iommu_iova_to_phys
,
682 .domain_has_cap
= msm_iommu_domain_has_cap
685 static int __init
get_tex_class(int icp
, int ocp
, int mt
, int nos
)
688 unsigned int prrr
= 0;
689 unsigned int nmrr
= 0;
690 int c_icp
, c_ocp
, c_mt
, c_nos
;
695 for (i
= 0; i
< NUM_TEX_CLASS
; i
++) {
696 c_nos
= PRRR_NOS(prrr
, i
);
697 c_mt
= PRRR_MT(prrr
, i
);
698 c_icp
= NMRR_ICP(nmrr
, i
);
699 c_ocp
= NMRR_OCP(nmrr
, i
);
701 if (icp
== c_icp
&& ocp
== c_ocp
&& c_mt
== mt
&& c_nos
== nos
)
708 static void __init
setup_iommu_tex_classes(void)
710 msm_iommu_tex_class
[MSM_IOMMU_ATTR_NONCACHED
] =
711 get_tex_class(CP_NONCACHED
, CP_NONCACHED
, MT_NORMAL
, 1);
713 msm_iommu_tex_class
[MSM_IOMMU_ATTR_CACHED_WB_WA
] =
714 get_tex_class(CP_WB_WA
, CP_WB_WA
, MT_NORMAL
, 1);
716 msm_iommu_tex_class
[MSM_IOMMU_ATTR_CACHED_WB_NWA
] =
717 get_tex_class(CP_WB_NWA
, CP_WB_NWA
, MT_NORMAL
, 1);
719 msm_iommu_tex_class
[MSM_IOMMU_ATTR_CACHED_WT
] =
720 get_tex_class(CP_WT
, CP_WT
, MT_NORMAL
, 1);
723 static int __init
msm_iommu_init(void)
725 setup_iommu_tex_classes();
726 bus_set_iommu(&platform_bus_type
, &msm_iommu_ops
);
730 subsys_initcall(msm_iommu_init
);
732 MODULE_LICENSE("GPL v2");
733 MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>");