Commit | Line | Data |
---|---|---|
4a488a7a OG |
1 | /* |
2 | * Copyright 2014 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | */ | |
22 | ||
23 | #include <linux/amd-iommu.h> | |
24 | #include <linux/bsearch.h> | |
25 | #include <linux/pci.h> | |
26 | #include <linux/slab.h> | |
27 | #include "kfd_priv.h" | |
64c7f8cf | 28 | #include "kfd_device_queue_manager.h" |
4a488a7a | 29 | |
19f6d2a6 OG |
30 | #define MQD_SIZE_ALIGNED 768 |
31 | ||
4a488a7a OG |
32 | static const struct kfd_device_info kaveri_device_info = { |
33 | .max_pasid_bits = 16, | |
19f6d2a6 | 34 | .mqd_size_aligned = MQD_SIZE_ALIGNED |
4a488a7a OG |
35 | }; |
36 | ||
37 | struct kfd_deviceid { | |
38 | unsigned short did; | |
39 | const struct kfd_device_info *device_info; | |
40 | }; | |
41 | ||
42 | /* Please keep this sorted by increasing device id. */ | |
43 | static const struct kfd_deviceid supported_devices[] = { | |
44 | { 0x1304, &kaveri_device_info }, /* Kaveri */ | |
45 | { 0x1305, &kaveri_device_info }, /* Kaveri */ | |
46 | { 0x1306, &kaveri_device_info }, /* Kaveri */ | |
47 | { 0x1307, &kaveri_device_info }, /* Kaveri */ | |
48 | { 0x1309, &kaveri_device_info }, /* Kaveri */ | |
49 | { 0x130A, &kaveri_device_info }, /* Kaveri */ | |
50 | { 0x130B, &kaveri_device_info }, /* Kaveri */ | |
51 | { 0x130C, &kaveri_device_info }, /* Kaveri */ | |
52 | { 0x130D, &kaveri_device_info }, /* Kaveri */ | |
53 | { 0x130E, &kaveri_device_info }, /* Kaveri */ | |
54 | { 0x130F, &kaveri_device_info }, /* Kaveri */ | |
55 | { 0x1310, &kaveri_device_info }, /* Kaveri */ | |
56 | { 0x1311, &kaveri_device_info }, /* Kaveri */ | |
57 | { 0x1312, &kaveri_device_info }, /* Kaveri */ | |
58 | { 0x1313, &kaveri_device_info }, /* Kaveri */ | |
59 | { 0x1315, &kaveri_device_info }, /* Kaveri */ | |
60 | { 0x1316, &kaveri_device_info }, /* Kaveri */ | |
61 | { 0x1317, &kaveri_device_info }, /* Kaveri */ | |
62 | { 0x1318, &kaveri_device_info }, /* Kaveri */ | |
63 | { 0x131B, &kaveri_device_info }, /* Kaveri */ | |
64 | { 0x131C, &kaveri_device_info }, /* Kaveri */ | |
65 | { 0x131D, &kaveri_device_info }, /* Kaveri */ | |
66 | }; | |
67 | ||
68 | static const struct kfd_device_info *lookup_device_info(unsigned short did) | |
69 | { | |
70 | size_t i; | |
71 | ||
72 | for (i = 0; i < ARRAY_SIZE(supported_devices); i++) { | |
73 | if (supported_devices[i].did == did) { | |
74 | BUG_ON(supported_devices[i].device_info == NULL); | |
75 | return supported_devices[i].device_info; | |
76 | } | |
77 | } | |
78 | ||
79 | return NULL; | |
80 | } | |
81 | ||
82 | struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev) | |
83 | { | |
84 | struct kfd_dev *kfd; | |
85 | ||
86 | const struct kfd_device_info *device_info = | |
87 | lookup_device_info(pdev->device); | |
88 | ||
89 | if (!device_info) | |
90 | return NULL; | |
91 | ||
92 | kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); | |
93 | if (!kfd) | |
94 | return NULL; | |
95 | ||
96 | kfd->kgd = kgd; | |
97 | kfd->device_info = device_info; | |
98 | kfd->pdev = pdev; | |
19f6d2a6 | 99 | kfd->init_complete = false; |
4a488a7a OG |
100 | |
101 | return kfd; | |
102 | } | |
103 | ||
b17f068a OG |
104 | static bool device_iommu_pasid_init(struct kfd_dev *kfd) |
105 | { | |
106 | const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP | | |
107 | AMD_IOMMU_DEVICE_FLAG_PRI_SUP | | |
108 | AMD_IOMMU_DEVICE_FLAG_PASID_SUP; | |
109 | ||
110 | struct amd_iommu_device_info iommu_info; | |
111 | unsigned int pasid_limit; | |
112 | int err; | |
113 | ||
114 | err = amd_iommu_device_info(kfd->pdev, &iommu_info); | |
115 | if (err < 0) { | |
116 | dev_err(kfd_device, | |
117 | "error getting iommu info. is the iommu enabled?\n"); | |
118 | return false; | |
119 | } | |
120 | ||
121 | if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) { | |
122 | dev_err(kfd_device, "error required iommu flags ats(%i), pri(%i), pasid(%i)\n", | |
123 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0, | |
124 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0, | |
125 | (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) != 0); | |
126 | return false; | |
127 | } | |
128 | ||
129 | pasid_limit = min_t(unsigned int, | |
130 | (unsigned int)1 << kfd->device_info->max_pasid_bits, | |
131 | iommu_info.max_pasids); | |
132 | /* | |
133 | * last pasid is used for kernel queues doorbells | |
134 | * in the future the last pasid might be used for a kernel thread. | |
135 | */ | |
136 | pasid_limit = min_t(unsigned int, | |
137 | pasid_limit, | |
138 | kfd->doorbell_process_limit - 1); | |
139 | ||
140 | err = amd_iommu_init_device(kfd->pdev, pasid_limit); | |
141 | if (err < 0) { | |
142 | dev_err(kfd_device, "error initializing iommu device\n"); | |
143 | return false; | |
144 | } | |
145 | ||
146 | if (!kfd_set_pasid_limit(pasid_limit)) { | |
147 | dev_err(kfd_device, "error setting pasid limit\n"); | |
148 | amd_iommu_free_device(kfd->pdev); | |
149 | return false; | |
150 | } | |
151 | ||
152 | return true; | |
153 | } | |
154 | ||
155 | static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid) | |
156 | { | |
157 | struct kfd_dev *dev = kfd_device_by_pci_dev(pdev); | |
158 | ||
159 | if (dev) | |
160 | kfd_unbind_process_from_device(dev, pasid); | |
161 | } | |
162 | ||
4a488a7a OG |
163 | bool kgd2kfd_device_init(struct kfd_dev *kfd, |
164 | const struct kgd2kfd_shared_resources *gpu_resources) | |
165 | { | |
19f6d2a6 OG |
166 | unsigned int size; |
167 | ||
4a488a7a OG |
168 | kfd->shared_resources = *gpu_resources; |
169 | ||
19f6d2a6 OG |
170 | /* calculate max size of mqds needed for queues */ |
171 | size = max_num_of_processes * | |
172 | max_num_of_queues_per_process * | |
173 | kfd->device_info->mqd_size_aligned; | |
174 | ||
175 | /* add another 512KB for all other allocations on gart */ | |
176 | size += 512 * 1024; | |
177 | ||
178 | if (kfd2kgd->init_sa_manager(kfd->kgd, size)) { | |
179 | dev_err(kfd_device, | |
180 | "Error initializing sa manager for device (%x:%x)\n", | |
181 | kfd->pdev->vendor, kfd->pdev->device); | |
182 | goto out; | |
183 | } | |
184 | ||
185 | kfd_doorbell_init(kfd); | |
186 | ||
187 | if (kfd_topology_add_device(kfd) != 0) { | |
188 | dev_err(kfd_device, | |
189 | "Error adding device (%x:%x) to topology\n", | |
190 | kfd->pdev->vendor, kfd->pdev->device); | |
191 | goto kfd_topology_add_device_error; | |
192 | } | |
193 | ||
b17f068a OG |
194 | if (!device_iommu_pasid_init(kfd)) { |
195 | dev_err(kfd_device, | |
196 | "Error initializing iommuv2 for device (%x:%x)\n", | |
197 | kfd->pdev->vendor, kfd->pdev->device); | |
198 | goto device_iommu_pasid_error; | |
199 | } | |
200 | amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | |
201 | iommu_pasid_shutdown_callback); | |
5b5c4e40 | 202 | |
64c7f8cf BG |
203 | kfd->dqm = device_queue_manager_init(kfd); |
204 | if (!kfd->dqm) { | |
205 | dev_err(kfd_device, | |
206 | "Error initializing queue manager for device (%x:%x)\n", | |
207 | kfd->pdev->vendor, kfd->pdev->device); | |
208 | goto device_queue_manager_error; | |
209 | } | |
210 | ||
211 | if (kfd->dqm->start(kfd->dqm) != 0) { | |
212 | dev_err(kfd_device, | |
213 | "Error starting queuen manager for device (%x:%x)\n", | |
214 | kfd->pdev->vendor, kfd->pdev->device); | |
215 | goto dqm_start_error; | |
216 | } | |
217 | ||
4a488a7a OG |
218 | kfd->init_complete = true; |
219 | dev_info(kfd_device, "added device (%x:%x)\n", kfd->pdev->vendor, | |
220 | kfd->pdev->device); | |
221 | ||
64c7f8cf BG |
222 | pr_debug("kfd: Starting kfd with the following scheduling policy %d\n", |
223 | sched_policy); | |
224 | ||
19f6d2a6 OG |
225 | goto out; |
226 | ||
64c7f8cf BG |
227 | dqm_start_error: |
228 | device_queue_manager_uninit(kfd->dqm); | |
229 | device_queue_manager_error: | |
230 | amd_iommu_free_device(kfd->pdev); | |
b17f068a OG |
231 | device_iommu_pasid_error: |
232 | kfd_topology_remove_device(kfd); | |
19f6d2a6 OG |
233 | kfd_topology_add_device_error: |
234 | kfd2kgd->fini_sa_manager(kfd->kgd); | |
235 | dev_err(kfd_device, | |
236 | "device (%x:%x) NOT added due to errors\n", | |
237 | kfd->pdev->vendor, kfd->pdev->device); | |
238 | out: | |
239 | return kfd->init_complete; | |
4a488a7a OG |
240 | } |
241 | ||
242 | void kgd2kfd_device_exit(struct kfd_dev *kfd) | |
243 | { | |
b17f068a | 244 | if (kfd->init_complete) { |
64c7f8cf | 245 | device_queue_manager_uninit(kfd->dqm); |
b17f068a OG |
246 | amd_iommu_free_device(kfd->pdev); |
247 | kfd_topology_remove_device(kfd); | |
248 | } | |
5b5c4e40 | 249 | |
4a488a7a OG |
250 | kfree(kfd); |
251 | } | |
252 | ||
253 | void kgd2kfd_suspend(struct kfd_dev *kfd) | |
254 | { | |
255 | BUG_ON(kfd == NULL); | |
b17f068a | 256 | |
64c7f8cf BG |
257 | if (kfd->init_complete) { |
258 | kfd->dqm->stop(kfd->dqm); | |
b17f068a | 259 | amd_iommu_free_device(kfd->pdev); |
64c7f8cf | 260 | } |
4a488a7a OG |
261 | } |
262 | ||
263 | int kgd2kfd_resume(struct kfd_dev *kfd) | |
264 | { | |
b17f068a OG |
265 | unsigned int pasid_limit; |
266 | int err; | |
267 | ||
4a488a7a OG |
268 | BUG_ON(kfd == NULL); |
269 | ||
b17f068a OG |
270 | pasid_limit = kfd_get_pasid_limit(); |
271 | ||
272 | if (kfd->init_complete) { | |
273 | err = amd_iommu_init_device(kfd->pdev, pasid_limit); | |
274 | if (err < 0) | |
275 | return -ENXIO; | |
276 | amd_iommu_set_invalidate_ctx_cb(kfd->pdev, | |
277 | iommu_pasid_shutdown_callback); | |
64c7f8cf | 278 | kfd->dqm->start(kfd->dqm); |
b17f068a OG |
279 | } |
280 | ||
4a488a7a OG |
281 | return 0; |
282 | } | |
283 | ||
284 | void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry) | |
285 | { | |
286 | } |