projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'media_tree/vsp1' into generic-zpos-v8
[deliverable/linux.git] / drivers / gpu / drm / amd / amdgpu / gmc_v7_0.c
CommitLineData
a2e73f56
AD		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
24#include "drmP.h"
25#include "amdgpu.h"
26#include "cikd.h"
27#include "cik.h"
28#include "gmc_v7_0.h"
29#include "amdgpu_ucode.h"
30
31#include "bif/bif_4_1_d.h"
32#include "bif/bif_4_1_sh_mask.h"
33
34#include "gmc/gmc_7_1_d.h"
35#include "gmc/gmc_7_1_sh_mask.h"
36
37#include "oss/oss_2_0_d.h"
38#include "oss/oss_2_0_sh_mask.h"
39
40static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
41static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
313c45fd 42static int gmc_v7_0_wait_for_idle(void *handle);
a2e73f56 43
2269a395 44MODULE_FIRMWARE("radeon/bonaire_mc.bin");
a2e73f56 45MODULE_FIRMWARE("radeon/hawaii_mc.bin");
429c45de 46MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
a2e73f56 47
72b459c8
AD		 48static const u32 golden_settings_iceland_a11[] =
49{
50 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
51 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
52 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
53 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
54};
55
56static const u32 iceland_mgcg_cgcg_init[] =
57{
58 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
59};
60
61static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
62{
63 switch (adev->asic_type) {
64 case CHIP_TOPAZ:
65 amdgpu_program_register_sequence(adev,
66 iceland_mgcg_cgcg_init,
67 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
68 amdgpu_program_register_sequence(adev,
69 golden_settings_iceland_a11,
70 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
71 break;
72 default:
73 break;
74 }
75}
76
ccd73f24
AD		 77static void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
78 struct amdgpu_mode_mc_save *save)
a2e73f56
AD		 79{
80 u32 blackout;
81
82 if (adev->mode_info.num_crtc)
83 amdgpu_display_stop_mc_access(adev, save);
84
313c45fd 85 gmc_v7_0_wait_for_idle((void *)adev);
a2e73f56
AD		 86
87 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
88 if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
89 /* Block CPU access */
90 WREG32(mmBIF_FB_EN, 0);
91 /* blackout the MC */
92 blackout = REG_SET_FIELD(blackout,
93 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
94 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
95 }
96 /* wait for the MC to settle */
97 udelay(100);
98}
99
ccd73f24
AD		 100static void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
101 struct amdgpu_mode_mc_save *save)
a2e73f56
AD		 102{
103 u32 tmp;
104
105 /* unblackout the MC */
106 tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
107 tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
108 WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
109 /* allow CPU access */
110 tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
111 tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
112 WREG32(mmBIF_FB_EN, tmp);
113
114 if (adev->mode_info.num_crtc)
115 amdgpu_display_resume_mc_access(adev, save);
116}
117
118/**
119 * gmc_v7_0_init_microcode - load ucode images from disk
120 *
121 * @adev: amdgpu_device pointer
122 *
123 * Use the firmware interface to load the ucode images into
124 * the driver (not loaded into hw).
125 * Returns 0 on success, error on failure.
126 */
127static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
128{
129 const char *chip_name;
130 char fw_name[30];
131 int err;
132
133 DRM_DEBUG("\n");
134
135 switch (adev->asic_type) {
136 case CHIP_BONAIRE:
137 chip_name = "bonaire";
138 break;
139 case CHIP_HAWAII:
140 chip_name = "hawaii";
141 break;
429c45de
KW		 142 case CHIP_TOPAZ:
143 chip_name = "topaz";
144 break;
a2e73f56
AD		 145 case CHIP_KAVERI:
146 case CHIP_KABINI:
147 return 0;
148 default: BUG();
149 }
150
72b459c8 151 if (adev->asic_type == CHIP_TOPAZ)
429c45de
KW		 152 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
153 else
154 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
155
a2e73f56
AD		 156 err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
157 if (err)
158 goto out;
159 err = amdgpu_ucode_validate(adev->mc.fw);
160
161out:
162 if (err) {
163 printk(KERN_ERR
164 "cik_mc: Failed to load firmware \"%s\"\n",
165 fw_name);
166 release_firmware(adev->mc.fw);
167 adev->mc.fw = NULL;
168 }
169 return err;
170}
171
172/**
173 * gmc_v7_0_mc_load_microcode - load MC ucode into the hw
174 *
175 * @adev: amdgpu_device pointer
176 *
177 * Load the GDDR MC ucode into the hw (CIK).
178 * Returns 0 on success, error on failure.
179 */
180static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
181{
182 const struct mc_firmware_header_v1_0 *hdr;
183 const __le32 *fw_data = NULL;
184 const __le32 *io_mc_regs = NULL;
185 u32 running, blackout = 0;
186 int i, ucode_size, regs_size;
187
188 if (!adev->mc.fw)
189 return -EINVAL;
190
191 hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
192 amdgpu_ucode_print_mc_hdr(&hdr->header);
193
194 adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
195 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
196 io_mc_regs = (const __le32 *)
197 (adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
198 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
199 fw_data = (const __le32 *)
200 (adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
201
202 running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
203
204 if (running == 0) {
205 if (running) {
206 blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
207 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
208 }
209
210 /* reset the engine and set to writable */
211 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
212 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
213
214 /* load mc io regs */
215 for (i = 0; i < regs_size; i++) {
216 WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
217 WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
218 }
219 /* load the MC ucode */
220 for (i = 0; i < ucode_size; i++)
221 WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
222
223 /* put the engine back into the active state */
224 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
225 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
226 WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
227
228 /* wait for training to complete */
229 for (i = 0; i < adev->usec_timeout; i++) {
230 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
231 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
232 break;
233 udelay(1);
234 }
235 for (i = 0; i < adev->usec_timeout; i++) {
236 if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
237 MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
238 break;
239 udelay(1);
240 }
241
242 if (running)
243 WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
244 }
245
246 return 0;
247}
248
249static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
250 struct amdgpu_mc *mc)
251{
252 if (mc->mc_vram_size > 0xFFC0000000ULL) {
253 /* leave room for at least 1024M GTT */
254 dev_warn(adev->dev, "limiting VRAM\n");
255 mc->real_vram_size = 0xFFC0000000ULL;
256 mc->mc_vram_size = 0xFFC0000000ULL;
257 }
258 amdgpu_vram_location(adev, &adev->mc, 0);
259 adev->mc.gtt_base_align = 0;
260 amdgpu_gtt_location(adev, mc);
261}
262
263/**
264 * gmc_v7_0_mc_program - program the GPU memory controller
265 *
266 * @adev: amdgpu_device pointer
267 *
268 * Set the location of vram, gart, and AGP in the GPU's
269 * physical address space (CIK).
270 */
271static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
272{
273 struct amdgpu_mode_mc_save save;
274 u32 tmp;
275 int i, j;
276
277 /* Initialize HDP */
278 for (i = 0, j = 0; i < 32; i++, j += 0x6) {
279 WREG32((0xb05 + j), 0x00000000);
280 WREG32((0xb06 + j), 0x00000000);
281 WREG32((0xb07 + j), 0x00000000);
282 WREG32((0xb08 + j), 0x00000000);
283 WREG32((0xb09 + j), 0x00000000);
284 }
285 WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
286
287 if (adev->mode_info.num_crtc)
288 amdgpu_display_set_vga_render_state(adev, false);
289
290 gmc_v7_0_mc_stop(adev, &save);
313c45fd 291 if (gmc_v7_0_wait_for_idle((void *)adev)) {
a2e73f56
AD		 292 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
293 }
294 /* Update configuration */
295 WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
296 adev->mc.vram_start >> 12);
297 WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
298 adev->mc.vram_end >> 12);
299 WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
300 adev->vram_scratch.gpu_addr >> 12);
301 tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
302 tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
303 WREG32(mmMC_VM_FB_LOCATION, tmp);
304 /* XXX double check these! */
305 WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
306 WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
307 WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
308 WREG32(mmMC_VM_AGP_BASE, 0);
309 WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
310 WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
313c45fd 311 if (gmc_v7_0_wait_for_idle((void *)adev)) {
a2e73f56
AD		 312 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
313 }
314 gmc_v7_0_mc_resume(adev, &save);
315
316 WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
317
318 tmp = RREG32(mmHDP_MISC_CNTL);
13459bd0 319 tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
a2e73f56
AD		 320 WREG32(mmHDP_MISC_CNTL, tmp);
321
322 tmp = RREG32(mmHDP_HOST_PATH_CNTL);
323 WREG32(mmHDP_HOST_PATH_CNTL, tmp);
324}
325
326/**
327 * gmc_v7_0_mc_init - initialize the memory controller driver params
328 *
329 * @adev: amdgpu_device pointer
330 *
331 * Look up the amount of vram, vram width, and decide how to place
332 * vram and gart within the GPU's physical address space (CIK).
333 * Returns 0 for success.
334 */
335static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
336{
337 u32 tmp;
338 int chansize, numchan;
339
340 /* Get VRAM informations */
341 tmp = RREG32(mmMC_ARB_RAMCFG);
342 if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
343 chansize = 64;
344 } else {
345 chansize = 32;
346 }
347 tmp = RREG32(mmMC_SHARED_CHMAP);
348 switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
349 case 0:
350 default:
351 numchan = 1;
352 break;
353 case 1:
354 numchan = 2;
355 break;
356 case 2:
357 numchan = 4;
358 break;
359 case 3:
360 numchan = 8;
361 break;
362 case 4:
363 numchan = 3;
364 break;
365 case 5:
366 numchan = 6;
367 break;
368 case 6:
369 numchan = 10;
370 break;
371 case 7:
372 numchan = 12;
373 break;
374 case 8:
375 numchan = 16;
376 break;
377 }
378 adev->mc.vram_width = numchan * chansize;
379 /* Could aper size report 0 ? */
380 adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
381 adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
382 /* size in MB on si */
383 adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
384 adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
385 adev->mc.visible_vram_size = adev->mc.aper_size;
386
a1493cd5
AD		 387 /* In case the PCI BAR is larger than the actual amount of vram */
388 if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
389 adev->mc.visible_vram_size = adev->mc.real_vram_size;
390
a2e73f56
AD		 391 /* unless the user had overridden it, set the gart
392 * size equal to the 1024 or vram, whichever is larger.
393 */
394 if (amdgpu_gart_size == -1)
395 adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
396 else
397 adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
398
399 gmc_v7_0_vram_gtt_location(adev, &adev->mc);
400
401 return 0;
402}
403
404/*
405 * GART
406 * VMID 0 is the physical GPU addresses as used by the kernel.
407 * VMIDs 1-15 are used for userspace clients and are handled
408 * by the amdgpu vm/hsa code.
409 */
410
411/**
412 * gmc_v7_0_gart_flush_gpu_tlb - gart tlb flush callback
413 *
414 * @adev: amdgpu_device pointer
415 * @vmid: vm instance to flush
416 *
417 * Flush the TLB for the requested page table (CIK).
418 */
419static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
420 uint32_t vmid)
421{
422 /* flush hdp cache */
423 WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
424
425 /* bits 0-15 are the VM contexts0-15 */
426 WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
427}
428
429/**
430 * gmc_v7_0_gart_set_pte_pde - update the page tables using MMIO
431 *
432 * @adev: amdgpu_device pointer
433 * @cpu_pt_addr: cpu address of the page table
434 * @gpu_page_idx: entry in the page table to update
435 * @addr: dst addr to write into pte/pde
436 * @flags: access flags
437 *
438 * Update the page tables using the CPU.
439 */
440static int gmc_v7_0_gart_set_pte_pde(struct amdgpu_device *adev,
441 void *cpu_pt_addr,
442 uint32_t gpu_page_idx,
443 uint64_t addr,
444 uint32_t flags)
445{
446 void __iomem *ptr = (void *)cpu_pt_addr;
447 uint64_t value;
448
449 value = addr & 0xFFFFFFFFFFFFF000ULL;
450 value |= flags;
451 writeq(value, ptr + (gpu_page_idx * 8));
452
453 return 0;
454}
455
d9c13156
CK		 456/**
457 * gmc_v8_0_set_fault_enable_default - update VM fault handling
458 *
459 * @adev: amdgpu_device pointer
460 * @value: true redirects VM faults to the default page
461 */
462static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
463 bool value)
464{
465 u32 tmp;
466
467 tmp = RREG32(mmVM_CONTEXT1_CNTL);
468 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
469 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
470 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
471 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
472 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
473 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
474 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
475 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
476 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
477 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
478 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
479 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
480 WREG32(mmVM_CONTEXT1_CNTL, tmp);
481}
482
a2e73f56
AD		 483/**
484 * gmc_v7_0_gart_enable - gart enable
485 *
486 * @adev: amdgpu_device pointer
487 *
488 * This sets up the TLBs, programs the page tables for VMID0,
489 * sets up the hw for VMIDs 1-15 which are allocated on
490 * demand, and sets up the global locations for the LDS, GDS,
491 * and GPUVM for FSA64 clients (CIK).
492 * Returns 0 for success, errors for failure.
493 */
494static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
495{
496 int r, i;
497 u32 tmp;
498
499 if (adev->gart.robj == NULL) {
500 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
501 return -EINVAL;
502 }
503 r = amdgpu_gart_table_vram_pin(adev);
504 if (r)
505 return r;
506 /* Setup TLB control */
507 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
508 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
509 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
510 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
511 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
512 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
513 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
514 /* Setup L2 cache */
515 tmp = RREG32(mmVM_L2_CNTL);
516 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
517 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
518 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
519 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
520 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
521 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
a80b3047 522 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
a2e73f56
AD		 523 WREG32(mmVM_L2_CNTL, tmp);
524 tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
525 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
526 WREG32(mmVM_L2_CNTL2, tmp);
527 tmp = RREG32(mmVM_L2_CNTL3);
528 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
529 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
530 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
531 WREG32(mmVM_L2_CNTL3, tmp);
532 /* setup context0 */
533 WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
9c97b5ab 534 WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
a2e73f56
AD		 535 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
536 WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
537 (u32)(adev->dummy_page.addr >> 12));
538 WREG32(mmVM_CONTEXT0_CNTL2, 0);
539 tmp = RREG32(mmVM_CONTEXT0_CNTL);
540 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
541 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
542 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
543 WREG32(mmVM_CONTEXT0_CNTL, tmp);
544
545 WREG32(0x575, 0);
546 WREG32(0x576, 0);
547 WREG32(0x577, 0);
548
549 /* empty context1-15 */
550 /* FIXME start with 4G, once using 2 level pt switch to full
551 * vm size space
552 */
553 /* set vm size, must be a multiple of 4 */
554 WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
25a595e4 555 WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
a2e73f56
AD		 556 for (i = 1; i < 16; i++) {
557 if (i < 8)
558 WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
559 adev->gart.table_addr >> 12);
560 else
561 WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
562 adev->gart.table_addr >> 12);
563 }
564
565 /* enable context1-15 */
566 WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
567 (u32)(adev->dummy_page.addr >> 12));
568 WREG32(mmVM_CONTEXT1_CNTL2, 4);
569 tmp = RREG32(mmVM_CONTEXT1_CNTL);
570 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
571 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
a2e73f56
AD		 572 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
573 amdgpu_vm_block_size - 9);
574 WREG32(mmVM_CONTEXT1_CNTL, tmp);
d9c13156
CK		 575 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
576 gmc_v7_0_set_fault_enable_default(adev, false);
577 else
578 gmc_v7_0_set_fault_enable_default(adev, true);
a2e73f56
AD		 579
580 if (adev->asic_type == CHIP_KAVERI) {
581 tmp = RREG32(mmCHUB_CONTROL);
582 tmp &= ~BYPASS_VM;
583 WREG32(mmCHUB_CONTROL, tmp);
584 }
585
586 gmc_v7_0_gart_flush_gpu_tlb(adev, 0);
587 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
588 (unsigned)(adev->mc.gtt_size >> 20),
589 (unsigned long long)adev->gart.table_addr);
590 adev->gart.ready = true;
591 return 0;
592}
593
594static int gmc_v7_0_gart_init(struct amdgpu_device *adev)
595{
596 int r;
597
598 if (adev->gart.robj) {
599 WARN(1, "R600 PCIE GART already initialized\n");
600 return 0;
601 }
602 /* Initialize common gart structure */
603 r = amdgpu_gart_init(adev);
604 if (r)
605 return r;
606 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
607 return amdgpu_gart_table_vram_alloc(adev);
608}
609
610/**
611 * gmc_v7_0_gart_disable - gart disable
612 *
613 * @adev: amdgpu_device pointer
614 *
615 * This disables all VM page table (CIK).
616 */
617static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)
618{
619 u32 tmp;
620
621 /* Disable all tables */
622 WREG32(mmVM_CONTEXT0_CNTL, 0);
623 WREG32(mmVM_CONTEXT1_CNTL, 0);
624 /* Setup TLB control */
625 tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
626 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
627 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
628 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
629 WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
630 /* Setup L2 cache */
631 tmp = RREG32(mmVM_L2_CNTL);
632 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
633 WREG32(mmVM_L2_CNTL, tmp);
634 WREG32(mmVM_L2_CNTL2, 0);
635 amdgpu_gart_table_vram_unpin(adev);
636}
637
638/**
639 * gmc_v7_0_gart_fini - vm fini callback
640 *
641 * @adev: amdgpu_device pointer
642 *
643 * Tears down the driver GART/VM setup (CIK).
644 */
645static void gmc_v7_0_gart_fini(struct amdgpu_device *adev)
646{
647 amdgpu_gart_table_vram_free(adev);
648 amdgpu_gart_fini(adev);
649}
650
651/*
652 * vm
653 * VMID 0 is the physical GPU addresses as used by the kernel.
654 * VMIDs 1-15 are used for userspace clients and are handled
655 * by the amdgpu vm/hsa code.
656 */
657/**
658 * gmc_v7_0_vm_init - cik vm init callback
659 *
660 * @adev: amdgpu_device pointer
661 *
662 * Inits cik specific vm parameters (number of VMs, base of vram for
663 * VMIDs 1-15) (CIK).
664 * Returns 0 for success.
665 */
666static int gmc_v7_0_vm_init(struct amdgpu_device *adev)
667{
668 /*
669 * number of VMs
670 * VMID 0 is reserved for System
671 * amdgpu graphics/compute will use VMIDs 1-7
672 * amdkfd will use VMIDs 8-15
673 */
a9a78b32
CK		 674 adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS;
675 amdgpu_vm_manager_init(adev);
a2e73f56
AD		 676
677 /* base offset of vram pages */
2f7d10b3 678 if (adev->flags & AMD_IS_APU) {
a2e73f56
AD		 679 u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
680 tmp <<= 22;
681 adev->vm_manager.vram_base_offset = tmp;
682 } else
683 adev->vm_manager.vram_base_offset = 0;
684
685 return 0;
686}
687
688/**
689 * gmc_v7_0_vm_fini - cik vm fini callback
690 *
691 * @adev: amdgpu_device pointer
692 *
693 * Tear down any asic specific VM setup (CIK).
694 */
695static void gmc_v7_0_vm_fini(struct amdgpu_device *adev)
696{
697}
698
699/**
700 * gmc_v7_0_vm_decode_fault - print human readable fault info
701 *
702 * @adev: amdgpu_device pointer
703 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
704 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
705 *
706 * Print human readable fault information (CIK).
707 */
708static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
709 u32 status, u32 addr, u32 mc_client)
710{
711 u32 mc_id;
712 u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
713 u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
714 PROTECTIONS);
715 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
716 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
717
718 mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
719 MEMORY_CLIENT_ID);
720
721 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
722 protections, vmid, addr,
723 REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
724 MEMORY_CLIENT_RW) ?
725 "write" : "read", block, mc_client, mc_id);
726}
727
728
729static const u32 mc_cg_registers[] = {
730 mmMC_HUB_MISC_HUB_CG,
731 mmMC_HUB_MISC_SIP_CG,
732 mmMC_HUB_MISC_VM_CG,
733 mmMC_XPB_CLK_GAT,
734 mmATC_MISC_CG,
735 mmMC_CITF_MISC_WR_CG,
736 mmMC_CITF_MISC_RD_CG,
737 mmMC_CITF_MISC_VM_CG,
738 mmVM_L2_CG,
739};
740
741static const u32 mc_cg_ls_en[] = {
742 MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
743 MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
744 MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
745 MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
746 ATC_MISC_CG__MEM_LS_ENABLE_MASK,
747 MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
748 MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
749 MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
750 VM_L2_CG__MEM_LS_ENABLE_MASK,
751};
752
753static const u32 mc_cg_en[] = {
754 MC_HUB_MISC_HUB_CG__ENABLE_MASK,
755 MC_HUB_MISC_SIP_CG__ENABLE_MASK,
756 MC_HUB_MISC_VM_CG__ENABLE_MASK,
757 MC_XPB_CLK_GAT__ENABLE_MASK,
758 ATC_MISC_CG__ENABLE_MASK,
759 MC_CITF_MISC_WR_CG__ENABLE_MASK,
760 MC_CITF_MISC_RD_CG__ENABLE_MASK,
761 MC_CITF_MISC_VM_CG__ENABLE_MASK,
762 VM_L2_CG__ENABLE_MASK,
763};
764
765static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,
766 bool enable)
767{
768 int i;
769 u32 orig, data;
770
771 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
772 orig = data = RREG32(mc_cg_registers[i]);
e3b04bc7 773 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
a2e73f56
AD		 774 data |= mc_cg_ls_en[i];
775 else
776 data &= ~mc_cg_ls_en[i];
777 if (data != orig)
778 WREG32(mc_cg_registers[i], data);
779 }
780}
781
782static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,
783 bool enable)
784{
785 int i;
786 u32 orig, data;
787
788 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
789 orig = data = RREG32(mc_cg_registers[i]);
e3b04bc7 790 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
a2e73f56
AD		 791 data |= mc_cg_en[i];
792 else
793 data &= ~mc_cg_en[i];
794 if (data != orig)
795 WREG32(mc_cg_registers[i], data);
796 }
797}
798
799static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,
800 bool enable)
801{
802 u32 orig, data;
803
804 orig = data = RREG32_PCIE(ixPCIE_CNTL2);
805
e3b04bc7 806 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
a2e73f56
AD		 807 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
808 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
809 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
810 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
811 } else {
812 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
813 data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
814 data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
815 data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
816 }
817
818 if (orig != data)
819 WREG32_PCIE(ixPCIE_CNTL2, data);
820}
821
822static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,
823 bool enable)
824{
825 u32 orig, data;
826
827 orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
828
e3b04bc7 829 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
a2e73f56
AD		 830 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
831 else
832 data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
833
834 if (orig != data)
835 WREG32(mmHDP_HOST_PATH_CNTL, data);
836}
837
838static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,
839 bool enable)
840{
841 u32 orig, data;
842
843 orig = data = RREG32(mmHDP_MEM_POWER_LS);
844
e3b04bc7 845 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
a2e73f56
AD		 846 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
847 else
848 data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
849
850 if (orig != data)
851 WREG32(mmHDP_MEM_POWER_LS, data);
852}
853
81c59f54
KW		 854static int gmc_v7_0_convert_vram_type(int mc_seq_vram_type)
855{
856 switch (mc_seq_vram_type) {
857 case MC_SEQ_MISC0__MT__GDDR1:
858 return AMDGPU_VRAM_TYPE_GDDR1;
859 case MC_SEQ_MISC0__MT__DDR2:
860 return AMDGPU_VRAM_TYPE_DDR2;
861 case MC_SEQ_MISC0__MT__GDDR3:
862 return AMDGPU_VRAM_TYPE_GDDR3;
863 case MC_SEQ_MISC0__MT__GDDR4:
864 return AMDGPU_VRAM_TYPE_GDDR4;
865 case MC_SEQ_MISC0__MT__GDDR5:
866 return AMDGPU_VRAM_TYPE_GDDR5;
867 case MC_SEQ_MISC0__MT__HBM:
868 return AMDGPU_VRAM_TYPE_HBM;
869 case MC_SEQ_MISC0__MT__DDR3:
870 return AMDGPU_VRAM_TYPE_DDR3;
871 default:
872 return AMDGPU_VRAM_TYPE_UNKNOWN;
873 }
874}
875
5fc3aeeb 876static int gmc_v7_0_early_init(void *handle)
a2e73f56 877{
5fc3aeeb 878 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
879
a2e73f56
AD		 880 gmc_v7_0_set_gart_funcs(adev);
881 gmc_v7_0_set_irq_funcs(adev);
882
a2e73f56
AD		 883 return 0;
884}
885
140b519f
CK		 886static int gmc_v7_0_late_init(void *handle)
887{
888 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
889
afc45421
FC		 890 if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
891 return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
892 else
893 return 0;
140b519f
CK		 894}
895
5fc3aeeb 896static int gmc_v7_0_sw_init(void *handle)
a2e73f56
AD		 897{
898 int r;
899 int dma_bits;
5fc3aeeb 900 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56 901
d1518a1d
AD		 902 if (adev->flags & AMD_IS_APU) {
903 adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
904 } else {
905 u32 tmp = RREG32(mmMC_SEQ_MISC0);
906 tmp &= MC_SEQ_MISC0__MT__MASK;
907 adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp);
908 }
909
a2e73f56
AD		 910 r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault);
911 if (r)
912 return r;
913
914 r = amdgpu_irq_add_id(adev, 147, &adev->mc.vm_fault);
915 if (r)
916 return r;
917
918 /* Adjust VM size here.
919 * Currently set to 4GB ((1 << 20) 4k pages).
920 * Max GPUVM size for cayman and SI is 40 bits.
921 */
922 adev->vm_manager.max_pfn = amdgpu_vm_size << 18;
923
924 /* Set the internal MC address mask
925 * This is the max address of the GPU's
926 * internal address space.
927 */
928 adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
929
930 /* set DMA mask + need_dma32 flags.
931 * PCIE - can handle 40-bits.
932 * IGP - can handle 40-bits
933 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
934 */
935 adev->need_dma32 = false;
936 dma_bits = adev->need_dma32 ? 32 : 40;
937 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
938 if (r) {
939 adev->need_dma32 = true;
940 dma_bits = 32;
941 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
942 }
943 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
944 if (r) {
945 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
946 printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
947 }
948
949 r = gmc_v7_0_init_microcode(adev);
950 if (r) {
951 DRM_ERROR("Failed to load mc firmware!\n");
952 return r;
953 }
954
955 r = gmc_v7_0_mc_init(adev);
956 if (r)
957 return r;
958
959 /* Memory manager */
960 r = amdgpu_bo_init(adev);
961 if (r)
962 return r;
963
964 r = gmc_v7_0_gart_init(adev);
965 if (r)
966 return r;
967
968 if (!adev->vm_manager.enabled) {
969 r = gmc_v7_0_vm_init(adev);
970 if (r) {
971 dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
972 return r;
973 }
974 adev->vm_manager.enabled = true;
975 }
976
977 return r;
978}
979
5fc3aeeb 980static int gmc_v7_0_sw_fini(void *handle)
a2e73f56 981{
5fc3aeeb 982 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 983
984 if (adev->vm_manager.enabled) {
ea89f8c9 985 amdgpu_vm_manager_fini(adev);
a2e73f56
AD		 986 gmc_v7_0_vm_fini(adev);
987 adev->vm_manager.enabled = false;
988 }
989 gmc_v7_0_gart_fini(adev);
418aa0c2 990 amdgpu_gem_force_release(adev);
a2e73f56
AD		 991 amdgpu_bo_fini(adev);
992
993 return 0;
994}
995
5fc3aeeb 996static int gmc_v7_0_hw_init(void *handle)
a2e73f56
AD		 997{
998 int r;
5fc3aeeb 999 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56 1000
72b459c8
AD		 1001 gmc_v7_0_init_golden_registers(adev);
1002
a2e73f56
AD		 1003 gmc_v7_0_mc_program(adev);
1004
2f7d10b3 1005 if (!(adev->flags & AMD_IS_APU)) {
a2e73f56
AD		 1006 r = gmc_v7_0_mc_load_microcode(adev);
1007 if (r) {
1008 DRM_ERROR("Failed to load MC firmware!\n");
1009 return r;
1010 }
1011 }
1012
1013 r = gmc_v7_0_gart_enable(adev);
1014 if (r)
1015 return r;
1016
1017 return r;
1018}
1019
5fc3aeeb 1020static int gmc_v7_0_hw_fini(void *handle)
a2e73f56 1021{
5fc3aeeb 1022 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1023
140b519f 1024 amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
a2e73f56
AD		 1025 gmc_v7_0_gart_disable(adev);
1026
1027 return 0;
1028}
1029
5fc3aeeb 1030static int gmc_v7_0_suspend(void *handle)
a2e73f56 1031{
5fc3aeeb 1032 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 1033
1034 if (adev->vm_manager.enabled) {
a2e73f56
AD		 1035 gmc_v7_0_vm_fini(adev);
1036 adev->vm_manager.enabled = false;
1037 }
1038 gmc_v7_0_hw_fini(adev);
1039
1040 return 0;
1041}
1042
5fc3aeeb 1043static int gmc_v7_0_resume(void *handle)
a2e73f56
AD		 1044{
1045 int r;
5fc3aeeb 1046 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 1047
1048 r = gmc_v7_0_hw_init(adev);
1049 if (r)
1050 return r;
1051
1052 if (!adev->vm_manager.enabled) {
1053 r = gmc_v7_0_vm_init(adev);
1054 if (r) {
1055 dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
1056 return r;
1057 }
1058 adev->vm_manager.enabled = true;
1059 }
1060
1061 return r;
1062}
1063
5fc3aeeb 1064static bool gmc_v7_0_is_idle(void *handle)
a2e73f56 1065{
5fc3aeeb 1066 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 1067 u32 tmp = RREG32(mmSRBM_STATUS);
1068
1069 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1070 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1071 return false;
1072
1073 return true;
1074}
1075
5fc3aeeb 1076static int gmc_v7_0_wait_for_idle(void *handle)
a2e73f56
AD		 1077{
1078 unsigned i;
1079 u32 tmp;
5fc3aeeb 1080 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 1081
1082 for (i = 0; i < adev->usec_timeout; i++) {
1083 /* read MC_STATUS */
1084 tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1085 SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1086 SRBM_STATUS__MCC_BUSY_MASK |
1087 SRBM_STATUS__MCD_BUSY_MASK |
1088 SRBM_STATUS__VMC_BUSY_MASK);
1089 if (!tmp)
1090 return 0;
1091 udelay(1);
1092 }
1093 return -ETIMEDOUT;
1094
1095}
1096
5fc3aeeb 1097static int gmc_v7_0_soft_reset(void *handle)
a2e73f56 1098{
5fc3aeeb 1099 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56
AD		 1100 struct amdgpu_mode_mc_save save;
1101 u32 srbm_soft_reset = 0;
1102 u32 tmp = RREG32(mmSRBM_STATUS);
1103
1104 if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1105 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1106 SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1107
1108 if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1109 SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
2f7d10b3 1110 if (!(adev->flags & AMD_IS_APU))
a2e73f56
AD		 1111 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1112 SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1113 }
1114
1115 if (srbm_soft_reset) {
a2e73f56 1116 gmc_v7_0_mc_stop(adev, &save);
313c45fd 1117 if (gmc_v7_0_wait_for_idle((void *)adev)) {
a2e73f56
AD		 1118 dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1119 }
1120
1121
1122 tmp = RREG32(mmSRBM_SOFT_RESET);
1123 tmp |= srbm_soft_reset;
1124 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1125 WREG32(mmSRBM_SOFT_RESET, tmp);
1126 tmp = RREG32(mmSRBM_SOFT_RESET);
1127
1128 udelay(50);
1129
1130 tmp &= ~srbm_soft_reset;
1131 WREG32(mmSRBM_SOFT_RESET, tmp);
1132 tmp = RREG32(mmSRBM_SOFT_RESET);
1133
1134 /* Wait a little for things to settle down */
1135 udelay(50);
1136
1137 gmc_v7_0_mc_resume(adev, &save);
1138 udelay(50);
a2e73f56
AD		 1139 }
1140
1141 return 0;
1142}
1143
1144static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1145 struct amdgpu_irq_src *src,
1146 unsigned type,
1147 enum amdgpu_interrupt_state state)
1148{
1149 u32 tmp;
1150 u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1151 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1152 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1153 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1154 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1155 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1156
1157 switch (state) {
1158 case AMDGPU_IRQ_STATE_DISABLE:
1159 /* system context */
1160 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1161 tmp &= ~bits;
1162 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1163 /* VMs */
1164 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1165 tmp &= ~bits;
1166 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1167 break;
1168 case AMDGPU_IRQ_STATE_ENABLE:
1169 /* system context */
1170 tmp = RREG32(mmVM_CONTEXT0_CNTL);
1171 tmp |= bits;
1172 WREG32(mmVM_CONTEXT0_CNTL, tmp);
1173 /* VMs */
1174 tmp = RREG32(mmVM_CONTEXT1_CNTL);
1175 tmp |= bits;
1176 WREG32(mmVM_CONTEXT1_CNTL, tmp);
1177 break;
1178 default:
1179 break;
1180 }
1181
1182 return 0;
1183}
1184
1185static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
1186 struct amdgpu_irq_src *source,
1187 struct amdgpu_iv_entry *entry)
1188{
1189 u32 addr, status, mc_client;
1190
1191 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1192 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1193 mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
ce0c6bcd
CK		 1194 /* reset addr and status */
1195 WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1196
1197 if (!addr && !status)
1198 return 0;
1199
d9c13156
CK		 1200 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1201 gmc_v7_0_set_fault_enable_default(adev, false);
1202
a2e73f56
AD		 1203 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1204 entry->src_id, entry->src_data);
1205 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1206 addr);
1207 dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1208 status);
1209 gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
a2e73f56
AD		 1210
1211 return 0;
1212}
1213
5fc3aeeb 1214static int gmc_v7_0_set_clockgating_state(void *handle,
1215 enum amd_clockgating_state state)
a2e73f56
AD		 1216{
1217 bool gate = false;
5fc3aeeb 1218 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
a2e73f56 1219
5fc3aeeb 1220 if (state == AMD_CG_STATE_GATE)
a2e73f56
AD		 1221 gate = true;
1222
2f7d10b3 1223 if (!(adev->flags & AMD_IS_APU)) {
a2e73f56
AD		 1224 gmc_v7_0_enable_mc_mgcg(adev, gate);
1225 gmc_v7_0_enable_mc_ls(adev, gate);
1226 }
1227 gmc_v7_0_enable_bif_mgls(adev, gate);
1228 gmc_v7_0_enable_hdp_mgcg(adev, gate);
1229 gmc_v7_0_enable_hdp_ls(adev, gate);
1230
1231 return 0;
1232}
1233
5fc3aeeb 1234static int gmc_v7_0_set_powergating_state(void *handle,
1235 enum amd_powergating_state state)
a2e73f56
AD		 1236{
1237 return 0;
1238}
1239
5fc3aeeb 1240const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
88a907d6 1241 .name = "gmc_v7_0",
a2e73f56 1242 .early_init = gmc_v7_0_early_init,
140b519f 1243 .late_init = gmc_v7_0_late_init,
a2e73f56
AD		 1244 .sw_init = gmc_v7_0_sw_init,
1245 .sw_fini = gmc_v7_0_sw_fini,
1246 .hw_init = gmc_v7_0_hw_init,
1247 .hw_fini = gmc_v7_0_hw_fini,
1248 .suspend = gmc_v7_0_suspend,
1249 .resume = gmc_v7_0_resume,
1250 .is_idle = gmc_v7_0_is_idle,
1251 .wait_for_idle = gmc_v7_0_wait_for_idle,
1252 .soft_reset = gmc_v7_0_soft_reset,
a2e73f56
AD		 1253 .set_clockgating_state = gmc_v7_0_set_clockgating_state,
1254 .set_powergating_state = gmc_v7_0_set_powergating_state,
1255};
1256
1257static const struct amdgpu_gart_funcs gmc_v7_0_gart_funcs = {
1258 .flush_gpu_tlb = gmc_v7_0_gart_flush_gpu_tlb,
1259 .set_pte_pde = gmc_v7_0_gart_set_pte_pde,
1260};
1261
1262static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
1263 .set = gmc_v7_0_vm_fault_interrupt_state,
1264 .process = gmc_v7_0_process_interrupt,
1265};
1266
1267static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev)
1268{
1269 if (adev->gart.gart_funcs == NULL)
1270 adev->gart.gart_funcs = &gmc_v7_0_gart_funcs;
1271}
1272
1273static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1274{
1275 adev->mc.vm_fault.num_types = 1;
1276 adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
1277}
Linux kernel - Deliverables
This page took 0.165681 seconds and 5 git commands to generate.