Commit | Line | Data |
---|---|---|
7a31f6f4 | 1 | /* |
89184651 | 2 | * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. |
7a31f6f4 | 3 | * |
89184651 TR |
4 | * This program is free software; you can redistribute it and/or modify |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7a31f6f4 HD |
7 | */ |
8 | ||
804cb54c | 9 | #include <linux/bitops.h> |
d1313e78 | 10 | #include <linux/debugfs.h> |
bc5e6dea | 11 | #include <linux/err.h> |
7a31f6f4 | 12 | #include <linux/iommu.h> |
89184651 | 13 | #include <linux/kernel.h> |
0760e8fa | 14 | #include <linux/of.h> |
89184651 TR |
15 | #include <linux/of_device.h> |
16 | #include <linux/platform_device.h> | |
17 | #include <linux/slab.h> | |
306a7f91 TR |
18 | |
19 | #include <soc/tegra/ahb.h> | |
89184651 | 20 | #include <soc/tegra/mc.h> |
7a31f6f4 | 21 | |
89184651 TR |
22 | struct tegra_smmu { |
23 | void __iomem *regs; | |
24 | struct device *dev; | |
e6bc5933 | 25 | |
89184651 TR |
26 | struct tegra_mc *mc; |
27 | const struct tegra_smmu_soc *soc; | |
39abf8aa | 28 | |
804cb54c TR |
29 | unsigned long pfn_mask; |
30 | ||
89184651 TR |
31 | unsigned long *asids; |
32 | struct mutex lock; | |
39abf8aa | 33 | |
89184651 | 34 | struct list_head list; |
d1313e78 TR |
35 | |
36 | struct dentry *debugfs; | |
7a31f6f4 | 37 | }; |
7a31f6f4 | 38 | |
89184651 | 39 | struct tegra_smmu_as { |
d5f1a81c | 40 | struct iommu_domain domain; |
89184651 TR |
41 | struct tegra_smmu *smmu; |
42 | unsigned int use_count; | |
32924c76 | 43 | u32 *count; |
853520fa | 44 | struct page **pts; |
89184651 TR |
45 | struct page *pd; |
46 | unsigned id; | |
47 | u32 attr; | |
7a31f6f4 HD |
48 | }; |
49 | ||
d5f1a81c JR |
50 | static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom) |
51 | { | |
52 | return container_of(dom, struct tegra_smmu_as, domain); | |
53 | } | |
54 | ||
89184651 TR |
55 | static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, |
56 | unsigned long offset) | |
57 | { | |
58 | writel(value, smmu->regs + offset); | |
59 | } | |
7a31f6f4 | 60 | |
89184651 TR |
61 | static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) |
62 | { | |
63 | return readl(smmu->regs + offset); | |
64 | } | |
5a2c937a | 65 | |
89184651 TR |
66 | #define SMMU_CONFIG 0x010 |
67 | #define SMMU_CONFIG_ENABLE (1 << 0) | |
7a31f6f4 | 68 | |
89184651 TR |
69 | #define SMMU_TLB_CONFIG 0x14 |
70 | #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) | |
71 | #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) | |
72 | #define SMMU_TLB_CONFIG_ACTIVE_LINES(x) ((x) & 0x3f) | |
0760e8fa | 73 | |
89184651 TR |
74 | #define SMMU_PTC_CONFIG 0x18 |
75 | #define SMMU_PTC_CONFIG_ENABLE (1 << 29) | |
76 | #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) | |
77 | #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) | |
39abf8aa | 78 | |
89184651 TR |
79 | #define SMMU_PTB_ASID 0x01c |
80 | #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) | |
a3b24915 | 81 | |
89184651 TR |
82 | #define SMMU_PTB_DATA 0x020 |
83 | #define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr)) | |
7a31f6f4 | 84 | |
89184651 | 85 | #define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr)) |
7a31f6f4 | 86 | |
89184651 TR |
87 | #define SMMU_TLB_FLUSH 0x030 |
88 | #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) | |
89 | #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) | |
90 | #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) | |
91 | #define SMMU_TLB_FLUSH_ASID(x) (((x) & 0x7f) << 24) | |
92 | #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ | |
93 | SMMU_TLB_FLUSH_VA_MATCH_SECTION) | |
94 | #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ | |
95 | SMMU_TLB_FLUSH_VA_MATCH_GROUP) | |
96 | #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) | |
a6870e92 | 97 | |
89184651 TR |
98 | #define SMMU_PTC_FLUSH 0x034 |
99 | #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) | |
100 | #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) | |
a6870e92 | 101 | |
89184651 TR |
102 | #define SMMU_PTC_FLUSH_HI 0x9b8 |
103 | #define SMMU_PTC_FLUSH_HI_MASK 0x3 | |
7a31f6f4 | 104 | |
89184651 TR |
105 | /* per-SWGROUP SMMU_*_ASID register */ |
106 | #define SMMU_ASID_ENABLE (1 << 31) | |
107 | #define SMMU_ASID_MASK 0x7f | |
108 | #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) | |
a6870e92 | 109 | |
89184651 TR |
110 | /* page table definitions */ |
111 | #define SMMU_NUM_PDE 1024 | |
112 | #define SMMU_NUM_PTE 1024 | |
a6870e92 | 113 | |
89184651 TR |
114 | #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) |
115 | #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) | |
7a31f6f4 | 116 | |
89184651 TR |
117 | #define SMMU_PDE_SHIFT 22 |
118 | #define SMMU_PTE_SHIFT 12 | |
fe1229b9 | 119 | |
89184651 TR |
120 | #define SMMU_PD_READABLE (1 << 31) |
121 | #define SMMU_PD_WRITABLE (1 << 30) | |
122 | #define SMMU_PD_NONSECURE (1 << 29) | |
7a31f6f4 | 123 | |
89184651 TR |
124 | #define SMMU_PDE_READABLE (1 << 31) |
125 | #define SMMU_PDE_WRITABLE (1 << 30) | |
126 | #define SMMU_PDE_NONSECURE (1 << 29) | |
127 | #define SMMU_PDE_NEXT (1 << 28) | |
7a31f6f4 | 128 | |
89184651 TR |
129 | #define SMMU_PTE_READABLE (1 << 31) |
130 | #define SMMU_PTE_WRITABLE (1 << 30) | |
131 | #define SMMU_PTE_NONSECURE (1 << 29) | |
7a31f6f4 | 132 | |
89184651 TR |
133 | #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ |
134 | SMMU_PDE_NONSECURE) | |
135 | #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \ | |
136 | SMMU_PTE_NONSECURE) | |
7a31f6f4 | 137 | |
34d35f8c RK |
138 | static unsigned int iova_pd_index(unsigned long iova) |
139 | { | |
140 | return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); | |
141 | } | |
142 | ||
143 | static unsigned int iova_pt_index(unsigned long iova) | |
144 | { | |
145 | return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); | |
146 | } | |
147 | ||
89184651 TR |
148 | static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, |
149 | unsigned long offset) | |
7a31f6f4 | 150 | { |
89184651 TR |
151 | phys_addr_t phys = page ? page_to_phys(page) : 0; |
152 | u32 value; | |
153 | ||
154 | if (page) { | |
155 | offset &= ~(smmu->mc->soc->atom_size - 1); | |
156 | ||
157 | if (smmu->mc->soc->num_address_bits > 32) { | |
158 | #ifdef CONFIG_PHYS_ADDR_T_64BIT | |
159 | value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; | |
160 | #else | |
161 | value = 0; | |
162 | #endif | |
163 | smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); | |
7a31f6f4 | 164 | } |
7a31f6f4 | 165 | |
89184651 TR |
166 | value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; |
167 | } else { | |
168 | value = SMMU_PTC_FLUSH_TYPE_ALL; | |
7a31f6f4 | 169 | } |
89184651 TR |
170 | |
171 | smmu_writel(smmu, value, SMMU_PTC_FLUSH); | |
7a31f6f4 HD |
172 | } |
173 | ||
89184651 | 174 | static inline void smmu_flush_tlb(struct tegra_smmu *smmu) |
7a31f6f4 | 175 | { |
89184651 | 176 | smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); |
7a31f6f4 HD |
177 | } |
178 | ||
89184651 TR |
179 | static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, |
180 | unsigned long asid) | |
7a31f6f4 | 181 | { |
89184651 | 182 | u32 value; |
7a31f6f4 | 183 | |
89184651 TR |
184 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
185 | SMMU_TLB_FLUSH_VA_MATCH_ALL; | |
186 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
187 | } |
188 | ||
89184651 TR |
189 | static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, |
190 | unsigned long asid, | |
191 | unsigned long iova) | |
7a31f6f4 | 192 | { |
89184651 | 193 | u32 value; |
7a31f6f4 | 194 | |
89184651 TR |
195 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
196 | SMMU_TLB_FLUSH_VA_SECTION(iova); | |
197 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
198 | } |
199 | ||
89184651 TR |
200 | static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, |
201 | unsigned long asid, | |
202 | unsigned long iova) | |
7a31f6f4 | 203 | { |
89184651 | 204 | u32 value; |
7a31f6f4 | 205 | |
89184651 TR |
206 | value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) | |
207 | SMMU_TLB_FLUSH_VA_GROUP(iova); | |
208 | smmu_writel(smmu, value, SMMU_TLB_FLUSH); | |
7a31f6f4 HD |
209 | } |
210 | ||
89184651 | 211 | static inline void smmu_flush(struct tegra_smmu *smmu) |
7a31f6f4 | 212 | { |
89184651 | 213 | smmu_readl(smmu, SMMU_CONFIG); |
7a31f6f4 HD |
214 | } |
215 | ||
89184651 | 216 | static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) |
7a31f6f4 | 217 | { |
89184651 | 218 | unsigned long id; |
7a31f6f4 | 219 | |
89184651 | 220 | mutex_lock(&smmu->lock); |
7a31f6f4 | 221 | |
89184651 TR |
222 | id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); |
223 | if (id >= smmu->soc->num_asids) { | |
224 | mutex_unlock(&smmu->lock); | |
225 | return -ENOSPC; | |
7a31f6f4 | 226 | } |
7a31f6f4 | 227 | |
89184651 TR |
228 | set_bit(id, smmu->asids); |
229 | *idp = id; | |
230 | ||
231 | mutex_unlock(&smmu->lock); | |
232 | return 0; | |
7a31f6f4 HD |
233 | } |
234 | ||
89184651 | 235 | static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) |
7a31f6f4 | 236 | { |
89184651 TR |
237 | mutex_lock(&smmu->lock); |
238 | clear_bit(id, smmu->asids); | |
239 | mutex_unlock(&smmu->lock); | |
7a31f6f4 | 240 | } |
89184651 TR |
241 | |
242 | static bool tegra_smmu_capable(enum iommu_cap cap) | |
7a31f6f4 | 243 | { |
89184651 | 244 | return false; |
7a31f6f4 | 245 | } |
7a31f6f4 | 246 | |
d5f1a81c | 247 | static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) |
7a31f6f4 | 248 | { |
89184651 TR |
249 | struct tegra_smmu_as *as; |
250 | unsigned int i; | |
251 | uint32_t *pd; | |
7a31f6f4 | 252 | |
d5f1a81c JR |
253 | if (type != IOMMU_DOMAIN_UNMANAGED) |
254 | return NULL; | |
255 | ||
89184651 TR |
256 | as = kzalloc(sizeof(*as), GFP_KERNEL); |
257 | if (!as) | |
d5f1a81c | 258 | return NULL; |
7a31f6f4 | 259 | |
89184651 | 260 | as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; |
7a31f6f4 | 261 | |
89184651 TR |
262 | as->pd = alloc_page(GFP_KERNEL | __GFP_DMA); |
263 | if (!as->pd) { | |
264 | kfree(as); | |
d5f1a81c | 265 | return NULL; |
7a31f6f4 | 266 | } |
9e971a03 | 267 | |
32924c76 | 268 | as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); |
89184651 TR |
269 | if (!as->count) { |
270 | __free_page(as->pd); | |
271 | kfree(as); | |
d5f1a81c | 272 | return NULL; |
7a31f6f4 | 273 | } |
9e971a03 | 274 | |
853520fa RK |
275 | as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); |
276 | if (!as->pts) { | |
32924c76 | 277 | kfree(as->count); |
853520fa RK |
278 | __free_page(as->pd); |
279 | kfree(as); | |
280 | return NULL; | |
281 | } | |
282 | ||
89184651 TR |
283 | /* clear PDEs */ |
284 | pd = page_address(as->pd); | |
285 | SetPageReserved(as->pd); | |
9e971a03 | 286 | |
89184651 TR |
287 | for (i = 0; i < SMMU_NUM_PDE; i++) |
288 | pd[i] = 0; | |
9e971a03 | 289 | |
471d9144 | 290 | /* setup aperture */ |
7f65ef01 JR |
291 | as->domain.geometry.aperture_start = 0; |
292 | as->domain.geometry.aperture_end = 0xffffffff; | |
293 | as->domain.geometry.force_aperture = true; | |
f9a4f063 | 294 | |
d5f1a81c | 295 | return &as->domain; |
7a31f6f4 HD |
296 | } |
297 | ||
d5f1a81c | 298 | static void tegra_smmu_domain_free(struct iommu_domain *domain) |
7a31f6f4 | 299 | { |
d5f1a81c | 300 | struct tegra_smmu_as *as = to_smmu_as(domain); |
7a31f6f4 | 301 | |
89184651 TR |
302 | /* TODO: free page directory and page tables */ |
303 | ClearPageReserved(as->pd); | |
7a31f6f4 | 304 | |
89184651 | 305 | kfree(as); |
7a31f6f4 HD |
306 | } |
307 | ||
89184651 TR |
308 | static const struct tegra_smmu_swgroup * |
309 | tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) | |
7a31f6f4 | 310 | { |
89184651 TR |
311 | const struct tegra_smmu_swgroup *group = NULL; |
312 | unsigned int i; | |
7a31f6f4 | 313 | |
89184651 TR |
314 | for (i = 0; i < smmu->soc->num_swgroups; i++) { |
315 | if (smmu->soc->swgroups[i].swgroup == swgroup) { | |
316 | group = &smmu->soc->swgroups[i]; | |
317 | break; | |
318 | } | |
319 | } | |
7a31f6f4 | 320 | |
89184651 | 321 | return group; |
7a31f6f4 HD |
322 | } |
323 | ||
89184651 TR |
324 | static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, |
325 | unsigned int asid) | |
7a31f6f4 | 326 | { |
89184651 TR |
327 | const struct tegra_smmu_swgroup *group; |
328 | unsigned int i; | |
329 | u32 value; | |
7a31f6f4 | 330 | |
89184651 TR |
331 | for (i = 0; i < smmu->soc->num_clients; i++) { |
332 | const struct tegra_mc_client *client = &smmu->soc->clients[i]; | |
7a31f6f4 | 333 | |
89184651 TR |
334 | if (client->swgroup != swgroup) |
335 | continue; | |
7a31f6f4 | 336 | |
89184651 TR |
337 | value = smmu_readl(smmu, client->smmu.reg); |
338 | value |= BIT(client->smmu.bit); | |
339 | smmu_writel(smmu, value, client->smmu.reg); | |
340 | } | |
7a31f6f4 | 341 | |
89184651 TR |
342 | group = tegra_smmu_find_swgroup(smmu, swgroup); |
343 | if (group) { | |
344 | value = smmu_readl(smmu, group->reg); | |
345 | value &= ~SMMU_ASID_MASK; | |
346 | value |= SMMU_ASID_VALUE(asid); | |
347 | value |= SMMU_ASID_ENABLE; | |
348 | smmu_writel(smmu, value, group->reg); | |
349 | } | |
7a31f6f4 HD |
350 | } |
351 | ||
89184651 TR |
352 | static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, |
353 | unsigned int asid) | |
7a31f6f4 | 354 | { |
89184651 TR |
355 | const struct tegra_smmu_swgroup *group; |
356 | unsigned int i; | |
357 | u32 value; | |
7a31f6f4 | 358 | |
89184651 TR |
359 | group = tegra_smmu_find_swgroup(smmu, swgroup); |
360 | if (group) { | |
361 | value = smmu_readl(smmu, group->reg); | |
362 | value &= ~SMMU_ASID_MASK; | |
363 | value |= SMMU_ASID_VALUE(asid); | |
364 | value &= ~SMMU_ASID_ENABLE; | |
365 | smmu_writel(smmu, value, group->reg); | |
366 | } | |
7a31f6f4 | 367 | |
89184651 TR |
368 | for (i = 0; i < smmu->soc->num_clients; i++) { |
369 | const struct tegra_mc_client *client = &smmu->soc->clients[i]; | |
7a31f6f4 | 370 | |
89184651 TR |
371 | if (client->swgroup != swgroup) |
372 | continue; | |
7a31f6f4 | 373 | |
89184651 TR |
374 | value = smmu_readl(smmu, client->smmu.reg); |
375 | value &= ~BIT(client->smmu.bit); | |
376 | smmu_writel(smmu, value, client->smmu.reg); | |
377 | } | |
7a31f6f4 HD |
378 | } |
379 | ||
89184651 TR |
380 | static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, |
381 | struct tegra_smmu_as *as) | |
7a31f6f4 | 382 | { |
89184651 | 383 | u32 value; |
7a31f6f4 HD |
384 | int err; |
385 | ||
89184651 TR |
386 | if (as->use_count > 0) { |
387 | as->use_count++; | |
388 | return 0; | |
7a31f6f4 | 389 | } |
7a31f6f4 | 390 | |
89184651 TR |
391 | err = tegra_smmu_alloc_asid(smmu, &as->id); |
392 | if (err < 0) | |
393 | return err; | |
7a31f6f4 | 394 | |
89184651 TR |
395 | smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD); |
396 | smmu_flush_ptc(smmu, as->pd, 0); | |
397 | smmu_flush_tlb_asid(smmu, as->id); | |
7a31f6f4 | 398 | |
89184651 TR |
399 | smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); |
400 | value = SMMU_PTB_DATA_VALUE(as->pd, as->attr); | |
401 | smmu_writel(smmu, value, SMMU_PTB_DATA); | |
402 | smmu_flush(smmu); | |
7a31f6f4 | 403 | |
89184651 TR |
404 | as->smmu = smmu; |
405 | as->use_count++; | |
7a31f6f4 | 406 | |
89184651 | 407 | return 0; |
7a31f6f4 HD |
408 | } |
409 | ||
89184651 TR |
410 | static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, |
411 | struct tegra_smmu_as *as) | |
7a31f6f4 | 412 | { |
89184651 TR |
413 | if (--as->use_count > 0) |
414 | return; | |
415 | ||
416 | tegra_smmu_free_asid(smmu, as->id); | |
417 | as->smmu = NULL; | |
7a31f6f4 HD |
418 | } |
419 | ||
89184651 TR |
420 | static int tegra_smmu_attach_dev(struct iommu_domain *domain, |
421 | struct device *dev) | |
7a31f6f4 | 422 | { |
89184651 | 423 | struct tegra_smmu *smmu = dev->archdata.iommu; |
d5f1a81c | 424 | struct tegra_smmu_as *as = to_smmu_as(domain); |
89184651 TR |
425 | struct device_node *np = dev->of_node; |
426 | struct of_phandle_args args; | |
427 | unsigned int index = 0; | |
428 | int err = 0; | |
7a31f6f4 | 429 | |
89184651 TR |
430 | while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
431 | &args)) { | |
432 | unsigned int swgroup = args.args[0]; | |
d2453b2c | 433 | |
89184651 TR |
434 | if (args.np != smmu->dev->of_node) { |
435 | of_node_put(args.np); | |
d2453b2c | 436 | continue; |
89184651 | 437 | } |
d2453b2c | 438 | |
89184651 | 439 | of_node_put(args.np); |
d2453b2c | 440 | |
89184651 TR |
441 | err = tegra_smmu_as_prepare(smmu, as); |
442 | if (err < 0) | |
443 | return err; | |
444 | ||
445 | tegra_smmu_enable(smmu, swgroup, as->id); | |
446 | index++; | |
7a31f6f4 | 447 | } |
7a31f6f4 | 448 | |
89184651 TR |
449 | if (index == 0) |
450 | return -ENODEV; | |
7a31f6f4 | 451 | |
89184651 TR |
452 | return 0; |
453 | } | |
7a31f6f4 | 454 | |
89184651 TR |
455 | static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) |
456 | { | |
d5f1a81c | 457 | struct tegra_smmu_as *as = to_smmu_as(domain); |
89184651 TR |
458 | struct device_node *np = dev->of_node; |
459 | struct tegra_smmu *smmu = as->smmu; | |
460 | struct of_phandle_args args; | |
461 | unsigned int index = 0; | |
7a31f6f4 | 462 | |
89184651 TR |
463 | while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
464 | &args)) { | |
465 | unsigned int swgroup = args.args[0]; | |
7a31f6f4 | 466 | |
89184651 TR |
467 | if (args.np != smmu->dev->of_node) { |
468 | of_node_put(args.np); | |
469 | continue; | |
470 | } | |
23349902 | 471 | |
89184651 | 472 | of_node_put(args.np); |
7a31f6f4 | 473 | |
89184651 TR |
474 | tegra_smmu_disable(smmu, swgroup, as->id); |
475 | tegra_smmu_as_unprepare(smmu, as); | |
476 | index++; | |
477 | } | |
7a31f6f4 HD |
478 | } |
479 | ||
0b42c7c1 RK |
480 | static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) |
481 | { | |
482 | u32 *pt = page_address(pt_page); | |
483 | ||
484 | return pt + iova_pt_index(iova); | |
485 | } | |
486 | ||
487 | static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, | |
488 | struct page **pagep) | |
489 | { | |
490 | unsigned int pd_index = iova_pd_index(iova); | |
491 | struct page *pt_page; | |
0b42c7c1 | 492 | |
853520fa RK |
493 | pt_page = as->pts[pd_index]; |
494 | if (!pt_page) | |
0b42c7c1 RK |
495 | return NULL; |
496 | ||
0b42c7c1 RK |
497 | *pagep = pt_page; |
498 | ||
499 | return tegra_smmu_pte_offset(pt_page, iova); | |
500 | } | |
501 | ||
89184651 TR |
502 | static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, |
503 | struct page **pagep) | |
7a31f6f4 | 504 | { |
32924c76 | 505 | u32 *pd = page_address(as->pd), *pt; |
34d35f8c | 506 | unsigned int pde = iova_pd_index(iova); |
89184651 TR |
507 | struct tegra_smmu *smmu = as->smmu; |
508 | struct page *page; | |
509 | unsigned int i; | |
510 | ||
853520fa | 511 | if (!as->pts[pde]) { |
89184651 TR |
512 | page = alloc_page(GFP_KERNEL | __GFP_DMA); |
513 | if (!page) | |
514 | return NULL; | |
7a31f6f4 | 515 | |
89184651 TR |
516 | pt = page_address(page); |
517 | SetPageReserved(page); | |
7a31f6f4 | 518 | |
89184651 TR |
519 | for (i = 0; i < SMMU_NUM_PTE; i++) |
520 | pt[i] = 0; | |
7a31f6f4 | 521 | |
853520fa RK |
522 | as->pts[pde] = page; |
523 | ||
89184651 | 524 | smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); |
7a31f6f4 | 525 | |
89184651 | 526 | pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); |
7a31f6f4 | 527 | |
89184651 TR |
528 | smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4); |
529 | smmu_flush_ptc(smmu, as->pd, pde << 2); | |
530 | smmu_flush_tlb_section(smmu, as->id, iova); | |
531 | smmu_flush(smmu); | |
532 | } else { | |
853520fa | 533 | page = as->pts[pde]; |
7a31f6f4 HD |
534 | } |
535 | ||
89184651 | 536 | *pagep = page; |
7a31f6f4 | 537 | |
0b42c7c1 RK |
538 | pt = page_address(page); |
539 | ||
89184651 | 540 | /* Keep track of entries in this page table. */ |
0b42c7c1 | 541 | if (pt[iova_pt_index(iova)] == 0) |
32924c76 | 542 | as->count[pde]++; |
7a31f6f4 | 543 | |
0b42c7c1 | 544 | return tegra_smmu_pte_offset(page, iova); |
89184651 | 545 | } |
39abf8aa | 546 | |
b98e34f0 | 547 | static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) |
39abf8aa | 548 | { |
b98e34f0 | 549 | struct tegra_smmu *smmu = as->smmu; |
34d35f8c | 550 | unsigned int pde = iova_pd_index(iova); |
b98e34f0 | 551 | u32 *pd = page_address(as->pd); |
853520fa | 552 | struct page *page = as->pts[pde]; |
39abf8aa | 553 | |
89184651 TR |
554 | /* |
555 | * When no entries in this page table are used anymore, return the | |
556 | * memory page to the system. | |
557 | */ | |
32924c76 | 558 | if (--as->count[pde] == 0) { |
b98e34f0 | 559 | unsigned int offset = pde * sizeof(*pd); |
39abf8aa | 560 | |
b98e34f0 RK |
561 | /* Clear the page directory entry first */ |
562 | pd[pde] = 0; | |
563 | ||
564 | /* Flush the page directory entry */ | |
565 | smmu->soc->ops->flush_dcache(as->pd, offset, sizeof(*pd)); | |
566 | smmu_flush_ptc(smmu, as->pd, offset); | |
567 | smmu_flush_tlb_section(smmu, as->id, iova); | |
568 | smmu_flush(smmu); | |
569 | ||
570 | /* Finally, free the page */ | |
571 | ClearPageReserved(page); | |
572 | __free_page(page); | |
853520fa | 573 | as->pts[pde] = NULL; |
39abf8aa | 574 | } |
39abf8aa HD |
575 | } |
576 | ||
8482ee5e RK |
577 | static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, |
578 | u32 *pte, struct page *pte_page, u32 val) | |
579 | { | |
580 | struct tegra_smmu *smmu = as->smmu; | |
581 | unsigned long offset = offset_in_page(pte); | |
582 | ||
583 | *pte = val; | |
584 | ||
585 | smmu->soc->ops->flush_dcache(pte_page, offset, 4); | |
586 | smmu_flush_ptc(smmu, pte_page, offset); | |
587 | smmu_flush_tlb_group(smmu, as->id, iova); | |
588 | smmu_flush(smmu); | |
589 | } | |
590 | ||
89184651 TR |
591 | static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, |
592 | phys_addr_t paddr, size_t size, int prot) | |
39abf8aa | 593 | { |
d5f1a81c | 594 | struct tegra_smmu_as *as = to_smmu_as(domain); |
89184651 TR |
595 | struct page *page; |
596 | u32 *pte; | |
39abf8aa | 597 | |
89184651 TR |
598 | pte = as_get_pte(as, iova, &page); |
599 | if (!pte) | |
600 | return -ENOMEM; | |
39abf8aa | 601 | |
8482ee5e RK |
602 | tegra_smmu_set_pte(as, iova, pte, page, |
603 | __phys_to_pfn(paddr) | SMMU_PTE_ATTR); | |
39abf8aa | 604 | |
39abf8aa HD |
605 | return 0; |
606 | } | |
607 | ||
89184651 TR |
608 | static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, |
609 | size_t size) | |
39abf8aa | 610 | { |
d5f1a81c | 611 | struct tegra_smmu_as *as = to_smmu_as(domain); |
0b42c7c1 | 612 | struct page *pte_page; |
89184651 | 613 | u32 *pte; |
39abf8aa | 614 | |
0b42c7c1 | 615 | pte = tegra_smmu_pte_lookup(as, iova, &pte_page); |
b98e34f0 | 616 | if (!pte || !*pte) |
89184651 | 617 | return 0; |
39abf8aa | 618 | |
0b42c7c1 | 619 | tegra_smmu_set_pte(as, iova, pte, pte_page, 0); |
b98e34f0 RK |
620 | tegra_smmu_pte_put_use(as, iova); |
621 | ||
89184651 | 622 | return size; |
39abf8aa HD |
623 | } |
624 | ||
89184651 TR |
625 | static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, |
626 | dma_addr_t iova) | |
39abf8aa | 627 | { |
d5f1a81c | 628 | struct tegra_smmu_as *as = to_smmu_as(domain); |
0b42c7c1 | 629 | struct page *pte_page; |
89184651 TR |
630 | unsigned long pfn; |
631 | u32 *pte; | |
39abf8aa | 632 | |
0b42c7c1 | 633 | pte = tegra_smmu_pte_lookup(as, iova, &pte_page); |
9113785c RK |
634 | if (!pte || !*pte) |
635 | return 0; | |
636 | ||
804cb54c | 637 | pfn = *pte & as->smmu->pfn_mask; |
39abf8aa | 638 | |
89184651 | 639 | return PFN_PHYS(pfn); |
39abf8aa HD |
640 | } |
641 | ||
89184651 | 642 | static struct tegra_smmu *tegra_smmu_find(struct device_node *np) |
7a31f6f4 | 643 | { |
89184651 TR |
644 | struct platform_device *pdev; |
645 | struct tegra_mc *mc; | |
7a31f6f4 | 646 | |
89184651 TR |
647 | pdev = of_find_device_by_node(np); |
648 | if (!pdev) | |
649 | return NULL; | |
650 | ||
651 | mc = platform_get_drvdata(pdev); | |
652 | if (!mc) | |
653 | return NULL; | |
654 | ||
655 | return mc->smmu; | |
7a31f6f4 HD |
656 | } |
657 | ||
89184651 | 658 | static int tegra_smmu_add_device(struct device *dev) |
7a31f6f4 | 659 | { |
89184651 TR |
660 | struct device_node *np = dev->of_node; |
661 | struct of_phandle_args args; | |
662 | unsigned int index = 0; | |
7a31f6f4 | 663 | |
89184651 TR |
664 | while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, |
665 | &args) == 0) { | |
666 | struct tegra_smmu *smmu; | |
667 | ||
668 | smmu = tegra_smmu_find(args.np); | |
669 | if (smmu) { | |
670 | /* | |
671 | * Only a single IOMMU master interface is currently | |
672 | * supported by the Linux kernel, so abort after the | |
673 | * first match. | |
674 | */ | |
675 | dev->archdata.iommu = smmu; | |
676 | break; | |
677 | } | |
678 | ||
679 | index++; | |
680 | } | |
681 | ||
682 | return 0; | |
7a31f6f4 HD |
683 | } |
684 | ||
89184651 | 685 | static void tegra_smmu_remove_device(struct device *dev) |
7a31f6f4 | 686 | { |
89184651 TR |
687 | dev->archdata.iommu = NULL; |
688 | } | |
7a31f6f4 | 689 | |
89184651 TR |
690 | static const struct iommu_ops tegra_smmu_ops = { |
691 | .capable = tegra_smmu_capable, | |
d5f1a81c JR |
692 | .domain_alloc = tegra_smmu_domain_alloc, |
693 | .domain_free = tegra_smmu_domain_free, | |
89184651 TR |
694 | .attach_dev = tegra_smmu_attach_dev, |
695 | .detach_dev = tegra_smmu_detach_dev, | |
696 | .add_device = tegra_smmu_add_device, | |
697 | .remove_device = tegra_smmu_remove_device, | |
698 | .map = tegra_smmu_map, | |
699 | .unmap = tegra_smmu_unmap, | |
700 | .map_sg = default_iommu_map_sg, | |
701 | .iova_to_phys = tegra_smmu_iova_to_phys, | |
7a31f6f4 | 702 | |
89184651 TR |
703 | .pgsize_bitmap = SZ_4K, |
704 | }; | |
7a31f6f4 | 705 | |
89184651 TR |
706 | static void tegra_smmu_ahb_enable(void) |
707 | { | |
708 | static const struct of_device_id ahb_match[] = { | |
709 | { .compatible = "nvidia,tegra30-ahb", }, | |
710 | { } | |
711 | }; | |
712 | struct device_node *ahb; | |
7a31f6f4 | 713 | |
89184651 TR |
714 | ahb = of_find_matching_node(NULL, ahb_match); |
715 | if (ahb) { | |
716 | tegra_ahb_enable_smmu(ahb); | |
717 | of_node_put(ahb); | |
7a31f6f4 | 718 | } |
89184651 | 719 | } |
7a31f6f4 | 720 | |
d1313e78 TR |
721 | static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) |
722 | { | |
723 | struct tegra_smmu *smmu = s->private; | |
724 | unsigned int i; | |
725 | u32 value; | |
726 | ||
727 | seq_printf(s, "swgroup enabled ASID\n"); | |
728 | seq_printf(s, "------------------------\n"); | |
729 | ||
730 | for (i = 0; i < smmu->soc->num_swgroups; i++) { | |
731 | const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; | |
732 | const char *status; | |
733 | unsigned int asid; | |
734 | ||
735 | value = smmu_readl(smmu, group->reg); | |
736 | ||
737 | if (value & SMMU_ASID_ENABLE) | |
738 | status = "yes"; | |
739 | else | |
740 | status = "no"; | |
741 | ||
742 | asid = value & SMMU_ASID_MASK; | |
743 | ||
744 | seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, | |
745 | asid); | |
746 | } | |
747 | ||
748 | return 0; | |
749 | } | |
750 | ||
751 | static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file) | |
752 | { | |
753 | return single_open(file, tegra_smmu_swgroups_show, inode->i_private); | |
754 | } | |
755 | ||
756 | static const struct file_operations tegra_smmu_swgroups_fops = { | |
757 | .open = tegra_smmu_swgroups_open, | |
758 | .read = seq_read, | |
759 | .llseek = seq_lseek, | |
760 | .release = single_release, | |
761 | }; | |
762 | ||
763 | static int tegra_smmu_clients_show(struct seq_file *s, void *data) | |
764 | { | |
765 | struct tegra_smmu *smmu = s->private; | |
766 | unsigned int i; | |
767 | u32 value; | |
768 | ||
769 | seq_printf(s, "client enabled\n"); | |
770 | seq_printf(s, "--------------------\n"); | |
771 | ||
772 | for (i = 0; i < smmu->soc->num_clients; i++) { | |
773 | const struct tegra_mc_client *client = &smmu->soc->clients[i]; | |
774 | const char *status; | |
775 | ||
776 | value = smmu_readl(smmu, client->smmu.reg); | |
777 | ||
778 | if (value & BIT(client->smmu.bit)) | |
779 | status = "yes"; | |
780 | else | |
781 | status = "no"; | |
782 | ||
783 | seq_printf(s, "%-12s %s\n", client->name, status); | |
784 | } | |
785 | ||
786 | return 0; | |
787 | } | |
788 | ||
789 | static int tegra_smmu_clients_open(struct inode *inode, struct file *file) | |
790 | { | |
791 | return single_open(file, tegra_smmu_clients_show, inode->i_private); | |
792 | } | |
793 | ||
794 | static const struct file_operations tegra_smmu_clients_fops = { | |
795 | .open = tegra_smmu_clients_open, | |
796 | .read = seq_read, | |
797 | .llseek = seq_lseek, | |
798 | .release = single_release, | |
799 | }; | |
800 | ||
801 | static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) | |
802 | { | |
803 | smmu->debugfs = debugfs_create_dir("smmu", NULL); | |
804 | if (!smmu->debugfs) | |
805 | return; | |
806 | ||
807 | debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, | |
808 | &tegra_smmu_swgroups_fops); | |
809 | debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, | |
810 | &tegra_smmu_clients_fops); | |
811 | } | |
812 | ||
813 | static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) | |
814 | { | |
815 | debugfs_remove_recursive(smmu->debugfs); | |
816 | } | |
817 | ||
89184651 TR |
818 | struct tegra_smmu *tegra_smmu_probe(struct device *dev, |
819 | const struct tegra_smmu_soc *soc, | |
820 | struct tegra_mc *mc) | |
821 | { | |
822 | struct tegra_smmu *smmu; | |
823 | size_t size; | |
824 | u32 value; | |
825 | int err; | |
7a31f6f4 | 826 | |
89184651 TR |
827 | /* This can happen on Tegra20 which doesn't have an SMMU */ |
828 | if (!soc) | |
829 | return NULL; | |
0760e8fa | 830 | |
89184651 TR |
831 | smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); |
832 | if (!smmu) | |
833 | return ERR_PTR(-ENOMEM); | |
0760e8fa | 834 | |
89184651 TR |
835 | /* |
836 | * This is a bit of a hack. Ideally we'd want to simply return this | |
837 | * value. However the IOMMU registration process will attempt to add | |
838 | * all devices to the IOMMU when bus_set_iommu() is called. In order | |
839 | * not to rely on global variables to track the IOMMU instance, we | |
840 | * set it here so that it can be looked up from the .add_device() | |
841 | * callback via the IOMMU device's .drvdata field. | |
842 | */ | |
843 | mc->smmu = smmu; | |
0760e8fa | 844 | |
89184651 | 845 | size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); |
0760e8fa | 846 | |
89184651 TR |
847 | smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); |
848 | if (!smmu->asids) | |
849 | return ERR_PTR(-ENOMEM); | |
7a31f6f4 | 850 | |
89184651 | 851 | mutex_init(&smmu->lock); |
7a31f6f4 | 852 | |
89184651 TR |
853 | smmu->regs = mc->regs; |
854 | smmu->soc = soc; | |
855 | smmu->dev = dev; | |
856 | smmu->mc = mc; | |
7a31f6f4 | 857 | |
804cb54c TR |
858 | smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; |
859 | dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", | |
860 | mc->soc->num_address_bits, smmu->pfn_mask); | |
861 | ||
89184651 | 862 | value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); |
7a31f6f4 | 863 | |
89184651 TR |
864 | if (soc->supports_request_limit) |
865 | value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); | |
39abf8aa | 866 | |
89184651 | 867 | smmu_writel(smmu, value, SMMU_PTC_CONFIG); |
7a31f6f4 | 868 | |
89184651 TR |
869 | value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | |
870 | SMMU_TLB_CONFIG_ACTIVE_LINES(0x20); | |
7a31f6f4 | 871 | |
89184651 TR |
872 | if (soc->supports_round_robin_arbitration) |
873 | value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; | |
7a31f6f4 | 874 | |
89184651 | 875 | smmu_writel(smmu, value, SMMU_TLB_CONFIG); |
7a31f6f4 | 876 | |
89184651 TR |
877 | smmu_flush_ptc(smmu, NULL, 0); |
878 | smmu_flush_tlb(smmu); | |
879 | smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); | |
880 | smmu_flush(smmu); | |
881 | ||
882 | tegra_smmu_ahb_enable(); | |
7a31f6f4 | 883 | |
89184651 TR |
884 | err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); |
885 | if (err < 0) | |
886 | return ERR_PTR(err); | |
7a31f6f4 | 887 | |
d1313e78 TR |
888 | if (IS_ENABLED(CONFIG_DEBUG_FS)) |
889 | tegra_smmu_debugfs_init(smmu); | |
890 | ||
89184651 TR |
891 | return smmu; |
892 | } | |
d1313e78 TR |
893 | |
894 | void tegra_smmu_remove(struct tegra_smmu *smmu) | |
895 | { | |
896 | if (IS_ENABLED(CONFIG_DEBUG_FS)) | |
897 | tegra_smmu_debugfs_exit(smmu); | |
898 | } |