projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: export error state ref handling
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
CommitLineData
76aaf220
DV		 1/*
2 * Copyright © 2010 Daniel Vetter
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
760285e7
DH		 25#include <drm/drmP.h>
26#include <drm/i915_drm.h>
76aaf220
DV		 27#include "i915_drv.h"
28#include "i915_trace.h"
29#include "intel_drv.h"
30
6670a5a5
BW		 31#define GEN6_PPGTT_PD_ENTRIES 512
32#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
33
26b1ff35
BW		 34/* PPGTT stuff */
35#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
36
37#define GEN6_PDE_VALID (1 << 0)
38/* gen6+ has bit 11-4 for physical addr bit 39-32 */
39#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
40
41#define GEN6_PTE_VALID (1 << 0)
42#define GEN6_PTE_UNCACHED (1 << 1)
43#define HSW_PTE_UNCACHED (0)
44#define GEN6_PTE_CACHE_LLC (2 << 1)
45#define GEN6_PTE_CACHE_LLC_MLC (3 << 1)
46#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
47
80a74f7f 48static gen6_gtt_pte_t gen6_pte_encode(dma_addr_t addr,
2d04befb 49 enum i915_cache_level level)
54d12527 50{
e7c2b58b 51 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
54d12527 52 pte |= GEN6_PTE_ADDR_ENCODE(addr);
e7210c3c
BW		 53
54 switch (level) {
55 case I915_CACHE_LLC_MLC:
9119708c 56 pte |= GEN6_PTE_CACHE_LLC_MLC;
e7210c3c
BW		 57 break;
58 case I915_CACHE_LLC:
59 pte |= GEN6_PTE_CACHE_LLC;
60 break;
61 case I915_CACHE_NONE:
9119708c 62 pte |= GEN6_PTE_UNCACHED;
e7210c3c
BW		 63 break;
64 default:
65 BUG();
66 }
67
54d12527
BW		 68 return pte;
69}
70
93c34e70
KG		 71#define BYT_PTE_WRITEABLE (1 << 1)
72#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
73
80a74f7f 74static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
93c34e70
KG		 75 enum i915_cache_level level)
76{
77 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
78 pte |= GEN6_PTE_ADDR_ENCODE(addr);
79
80 /* Mark the page as writeable. Other platforms don't have a
81 * setting for read-only/writable, so this matches that behavior.
82 */
83 pte |= BYT_PTE_WRITEABLE;
84
85 if (level != I915_CACHE_NONE)
86 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
87
88 return pte;
89}
90
80a74f7f 91static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
9119708c
KG		 92 enum i915_cache_level level)
93{
94 gen6_gtt_pte_t pte = GEN6_PTE_VALID;
95 pte |= GEN6_PTE_ADDR_ENCODE(addr);
96
97 if (level != I915_CACHE_NONE)
98 pte |= GEN6_PTE_CACHE_LLC;
99
100 return pte;
101}
102
3e302542 103static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
6197349b 104{
3e302542 105 struct drm_i915_private *dev_priv = ppgtt->dev->dev_private;
6197349b
BW		 106 gen6_gtt_pte_t __iomem *pd_addr;
107 uint32_t pd_entry;
108 int i;
109
0a732870 110 WARN_ON(ppgtt->pd_offset & 0x3f);
6197349b
BW		 111 pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
112 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
113 for (i = 0; i < ppgtt->num_pd_entries; i++) {
114 dma_addr_t pt_addr;
115
116 pt_addr = ppgtt->pt_dma_addr[i];
117 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
118 pd_entry |= GEN6_PDE_VALID;
119
120 writel(pd_entry, pd_addr + i);
121 }
122 readl(pd_addr);
3e302542
BW		 123}
124
125static int gen6_ppgtt_enable(struct drm_device *dev)
126{
127 drm_i915_private_t *dev_priv = dev->dev_private;
128 uint32_t pd_offset;
129 struct intel_ring_buffer *ring;
130 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
131 int i;
132
133 BUG_ON(ppgtt->pd_offset & 0x3f);
134
135 gen6_write_pdes(ppgtt);
6197349b
BW		 136
137 pd_offset = ppgtt->pd_offset;
138 pd_offset /= 64; /* in cachelines, */
139 pd_offset <<= 16;
140
141 if (INTEL_INFO(dev)->gen == 6) {
142 uint32_t ecochk, gab_ctl, ecobits;
143
144 ecobits = I915_READ(GAC_ECO_BITS);
3b9d7888
VS		 145 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
146 ECOBITS_PPGTT_CACHE64B);
6197349b
BW		 147
148 gab_ctl = I915_READ(GAB_CTL);
149 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
150
151 ecochk = I915_READ(GAM_ECOCHK);
152 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
153 ECOCHK_PPGTT_CACHE64B);
154 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
155 } else if (INTEL_INFO(dev)->gen >= 7) {
a6f429a5 156 uint32_t ecochk, ecobits;
a65c2fcd
VS		 157
158 ecobits = I915_READ(GAC_ECO_BITS);
159 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
160
a6f429a5
VS		 161 ecochk = I915_READ(GAM_ECOCHK);
162 if (IS_HASWELL(dev)) {
163 ecochk |= ECOCHK_PPGTT_WB_HSW;
164 } else {
165 ecochk |= ECOCHK_PPGTT_LLC_IVB;
166 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
167 }
168 I915_WRITE(GAM_ECOCHK, ecochk);
6197349b
BW		 169 /* GFX_MODE is per-ring on gen7+ */
170 }
171
172 for_each_ring(ring, dev_priv, i) {
173 if (INTEL_INFO(dev)->gen >= 7)
174 I915_WRITE(RING_MODE_GEN7(ring),
175 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
176
177 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
178 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
179 }
b7c36d25 180 return 0;
6197349b
BW		 181}
182
1d2a314c 183/* PPGTT support for Sandybdrige/Gen6 and later */
def886c3 184static void gen6_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
1d2a314c
DV		 185 unsigned first_entry,
186 unsigned num_entries)
187{
84f13560 188 struct drm_i915_private *dev_priv = ppgtt->dev->dev_private;
e7c2b58b 189 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
a15326a5 190 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
7bddb01f
DV		 191 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
192 unsigned last_pte, i;
1d2a314c 193
80a74f7f 194 scratch_pte = ppgtt->pte_encode(dev_priv->gtt.scratch.addr,
2d04befb 195 I915_CACHE_LLC);
1d2a314c 196
7bddb01f
DV		 197 while (num_entries) {
198 last_pte = first_pte + num_entries;
199 if (last_pte > I915_PPGTT_PT_ENTRIES)
200 last_pte = I915_PPGTT_PT_ENTRIES;
201
a15326a5 202 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
1d2a314c 203
7bddb01f
DV		 204 for (i = first_pte; i < last_pte; i++)
205 pt_vaddr[i] = scratch_pte;
1d2a314c
DV		 206
207 kunmap_atomic(pt_vaddr);
1d2a314c 208
7bddb01f
DV		 209 num_entries -= last_pte - first_pte;
210 first_pte = 0;
a15326a5 211 act_pt++;
7bddb01f 212 }
1d2a314c
DV		 213}
214
def886c3
DV		 215static void gen6_ppgtt_insert_entries(struct i915_hw_ppgtt *ppgtt,
216 struct sg_table *pages,
217 unsigned first_entry,
218 enum i915_cache_level cache_level)
219{
e7c2b58b 220 gen6_gtt_pte_t *pt_vaddr;
a15326a5 221 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
6e995e23
ID		 222 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
223 struct sg_page_iter sg_iter;
224
a15326a5 225 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
6e995e23
ID		 226 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
227 dma_addr_t page_addr;
228
2db76d7c 229 page_addr = sg_page_iter_dma_address(&sg_iter);
80a74f7f 230 pt_vaddr[act_pte] = ppgtt->pte_encode(page_addr, cache_level);
6e995e23
ID		 231 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
232 kunmap_atomic(pt_vaddr);
a15326a5
DV		 233 act_pt++;
234 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
6e995e23 235 act_pte = 0;
def886c3 236
def886c3 237 }
def886c3 238 }
6e995e23 239 kunmap_atomic(pt_vaddr);
def886c3
DV		 240}
241
3440d265 242static void gen6_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
1d2a314c 243{
3440d265
DV		 244 int i;
245
246 if (ppgtt->pt_dma_addr) {
247 for (i = 0; i < ppgtt->num_pd_entries; i++)
248 pci_unmap_page(ppgtt->dev->pdev,
249 ppgtt->pt_dma_addr[i],
250 4096, PCI_DMA_BIDIRECTIONAL);
251 }
252
253 kfree(ppgtt->pt_dma_addr);
254 for (i = 0; i < ppgtt->num_pd_entries; i++)
255 __free_page(ppgtt->pt_pages[i]);
256 kfree(ppgtt->pt_pages);
257 kfree(ppgtt);
258}
259
260static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
261{
262 struct drm_device *dev = ppgtt->dev;
1d2a314c 263 struct drm_i915_private *dev_priv = dev->dev_private;
1d2a314c 264 unsigned first_pd_entry_in_global_pt;
1d2a314c
DV		 265 int i;
266 int ret = -ENOMEM;
267
268 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
269 * entries. For aliasing ppgtt support we just steal them at the end for
270 * now. */
e1b73cba 271 first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
1d2a314c 272
9119708c
KG		 273 if (IS_HASWELL(dev)) {
274 ppgtt->pte_encode = hsw_pte_encode;
275 } else if (IS_VALLEYVIEW(dev)) {
93c34e70
KG		 276 ppgtt->pte_encode = byt_pte_encode;
277 } else {
278 ppgtt->pte_encode = gen6_pte_encode;
279 }
6670a5a5 280 ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
6197349b 281 ppgtt->enable = gen6_ppgtt_enable;
def886c3
DV		 282 ppgtt->clear_range = gen6_ppgtt_clear_range;
283 ppgtt->insert_entries = gen6_ppgtt_insert_entries;
3440d265 284 ppgtt->cleanup = gen6_ppgtt_cleanup;
1d2a314c
DV		 285 ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
286 GFP_KERNEL);
287 if (!ppgtt->pt_pages)
3440d265 288 return -ENOMEM;
1d2a314c
DV		 289
290 for (i = 0; i < ppgtt->num_pd_entries; i++) {
291 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
292 if (!ppgtt->pt_pages[i])
293 goto err_pt_alloc;
294 }
295
8d2e6308
BW		 296 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) *ppgtt->num_pd_entries,
297 GFP_KERNEL);
298 if (!ppgtt->pt_dma_addr)
299 goto err_pt_alloc;
1d2a314c 300
8d2e6308
BW		 301 for (i = 0; i < ppgtt->num_pd_entries; i++) {
302 dma_addr_t pt_addr;
211c568b 303
8d2e6308
BW		 304 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
305 PCI_DMA_BIDIRECTIONAL);
1d2a314c 306
8d2e6308
BW		 307 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
308 ret = -EIO;
309 goto err_pd_pin;
1d2a314c 310
211c568b 311 }
8d2e6308 312 ppgtt->pt_dma_addr[i] = pt_addr;
1d2a314c 313 }
1d2a314c 314
def886c3
DV		 315 ppgtt->clear_range(ppgtt, 0,
316 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
1d2a314c 317
e7c2b58b 318 ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
1d2a314c 319
1d2a314c
DV		 320 return 0;
321
322err_pd_pin:
323 if (ppgtt->pt_dma_addr) {
324 for (i--; i >= 0; i--)
325 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
326 4096, PCI_DMA_BIDIRECTIONAL);
327 }
328err_pt_alloc:
329 kfree(ppgtt->pt_dma_addr);
330 for (i = 0; i < ppgtt->num_pd_entries; i++) {
331 if (ppgtt->pt_pages[i])
332 __free_page(ppgtt->pt_pages[i]);
333 }
334 kfree(ppgtt->pt_pages);
3440d265
DV		 335
336 return ret;
337}
338
339static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
340{
341 struct drm_i915_private *dev_priv = dev->dev_private;
342 struct i915_hw_ppgtt *ppgtt;
343 int ret;
344
345 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
346 if (!ppgtt)
347 return -ENOMEM;
348
349 ppgtt->dev = dev;
350
3ed124b2
BW		 351 if (INTEL_INFO(dev)->gen < 8)
352 ret = gen6_ppgtt_init(ppgtt);
353 else
354 BUG();
355
3440d265
DV		 356 if (ret)
357 kfree(ppgtt);
358 else
359 dev_priv->mm.aliasing_ppgtt = ppgtt;
1d2a314c
DV		 360
361 return ret;
362}
363
364void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
365{
366 struct drm_i915_private *dev_priv = dev->dev_private;
367 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1d2a314c
DV		 368
369 if (!ppgtt)
370 return;
371
3440d265 372 ppgtt->cleanup(ppgtt);
5963cf04 373 dev_priv->mm.aliasing_ppgtt = NULL;
1d2a314c
DV		 374}
375
7bddb01f
DV		 376void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
377 struct drm_i915_gem_object *obj,
378 enum i915_cache_level cache_level)
379{
def886c3
DV		 380 ppgtt->insert_entries(ppgtt, obj->pages,
381 obj->gtt_space->start >> PAGE_SHIFT,
382 cache_level);
7bddb01f
DV		 383}
384
385void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
386 struct drm_i915_gem_object *obj)
387{
def886c3
DV		 388 ppgtt->clear_range(ppgtt,
389 obj->gtt_space->start >> PAGE_SHIFT,
390 obj->base.size >> PAGE_SHIFT);
7bddb01f
DV		 391}
392
a81cc00c
BW		 393extern int intel_iommu_gfx_mapped;
394/* Certain Gen5 chipsets require require idling the GPU before
395 * unmapping anything from the GTT when VT-d is enabled.
396 */
397static inline bool needs_idle_maps(struct drm_device *dev)
398{
399#ifdef CONFIG_INTEL_IOMMU
400 /* Query intel_iommu to see if we need the workaround. Presumably that
401 * was loaded first.
402 */
403 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
404 return true;
405#endif
406 return false;
407}
408
5c042287
BW		 409static bool do_idling(struct drm_i915_private *dev_priv)
410{
411 bool ret = dev_priv->mm.interruptible;
412
a81cc00c 413 if (unlikely(dev_priv->gtt.do_idle_maps)) {
5c042287 414 dev_priv->mm.interruptible = false;
b2da9fe5 415 if (i915_gpu_idle(dev_priv->dev)) {
5c042287
BW		 416 DRM_ERROR("Couldn't idle GPU\n");
417 /* Wait a bit, in hopes it avoids the hang */
418 udelay(10);
419 }
420 }
421
422 return ret;
423}
424
425static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
426{
a81cc00c 427 if (unlikely(dev_priv->gtt.do_idle_maps))
5c042287
BW		 428 dev_priv->mm.interruptible = interruptible;
429}
430
76aaf220
DV		 431void i915_gem_restore_gtt_mappings(struct drm_device *dev)
432{
433 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 434 struct drm_i915_gem_object *obj;
76aaf220 435
bee4a186 436 /* First fill our portion of the GTT with scratch pages */
7faf1ab2
DV		 437 dev_priv->gtt.gtt_clear_range(dev, dev_priv->gtt.start / PAGE_SIZE,
438 dev_priv->gtt.total / PAGE_SIZE);
bee4a186 439
35c20a60 440 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
a8e93126 441 i915_gem_clflush_object(obj);
74163907 442 i915_gem_gtt_bind_object(obj, obj->cache_level);
76aaf220
DV		 443 }
444
e76e9aeb 445 i915_gem_chipset_flush(dev);
76aaf220 446}
7c2e6fdf 447
74163907 448int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
7c2e6fdf 449{
9da3da66 450 if (obj->has_dma_mapping)
74163907 451 return 0;
9da3da66
CW		 452
453 if (!dma_map_sg(&obj->base.dev->pdev->dev,
454 obj->pages->sgl, obj->pages->nents,
455 PCI_DMA_BIDIRECTIONAL))
456 return -ENOSPC;
457
458 return 0;
7c2e6fdf
DV		 459}
460
e76e9aeb
BW		 461/*
462 * Binds an object into the global gtt with the specified cache level. The object
463 * will be accessible to the GPU via commands whose operands reference offsets
464 * within the global GTT as well as accessible by the GPU through the GMADR
465 * mapped BAR (dev_priv->mm.gtt->gtt).
466 */
7faf1ab2
DV		 467static void gen6_ggtt_insert_entries(struct drm_device *dev,
468 struct sg_table *st,
469 unsigned int first_entry,
470 enum i915_cache_level level)
e76e9aeb 471{
e76e9aeb 472 struct drm_i915_private *dev_priv = dev->dev_private;
e7c2b58b
BW		 473 gen6_gtt_pte_t __iomem *gtt_entries =
474 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
6e995e23
ID		 475 int i = 0;
476 struct sg_page_iter sg_iter;
e76e9aeb
BW		 477 dma_addr_t addr;
478
6e995e23 479 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2db76d7c 480 addr = sg_page_iter_dma_address(&sg_iter);
80a74f7f 481 iowrite32(dev_priv->gtt.pte_encode(addr, level),
2d04befb 482 &gtt_entries[i]);
6e995e23 483 i++;
e76e9aeb
BW		 484 }
485
e76e9aeb
BW		 486 /* XXX: This serves as a posting read to make sure that the PTE has
487 * actually been updated. There is some concern that even though
488 * registers and PTEs are within the same BAR that they are potentially
489 * of NUMA access patterns. Therefore, even with the way we assume
490 * hardware should work, we must keep this posting read for paranoia.
491 */
492 if (i != 0)
960e3e42 493 WARN_ON(readl(&gtt_entries[i-1])
80a74f7f 494 != dev_priv->gtt.pte_encode(addr, level));
0f9b91c7
BW		 495
496 /* This next bit makes the above posting read even more important. We
497 * want to flush the TLBs only after we're certain all the PTE updates
498 * have finished.
499 */
500 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
501 POSTING_READ(GFX_FLSH_CNTL_GEN6);
e76e9aeb
BW		 502}
503
7faf1ab2
DV		 504static void gen6_ggtt_clear_range(struct drm_device *dev,
505 unsigned int first_entry,
506 unsigned int num_entries)
507{
508 struct drm_i915_private *dev_priv = dev->dev_private;
e7c2b58b
BW		 509 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
510 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
a54c0c27 511 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
7faf1ab2
DV		 512 int i;
513
514 if (WARN(num_entries > max_entries,
515 "First entry = %d; Num entries = %d (max=%d)\n",
516 first_entry, num_entries, max_entries))
517 num_entries = max_entries;
518
80a74f7f 519 scratch_pte = dev_priv->gtt.pte_encode(dev_priv->gtt.scratch.addr,
2d04befb 520 I915_CACHE_LLC);
7faf1ab2
DV		 521 for (i = 0; i < num_entries; i++)
522 iowrite32(scratch_pte, &gtt_base[i]);
523 readl(gtt_base);
524}
525
526
527static void i915_ggtt_insert_entries(struct drm_device *dev,
528 struct sg_table *st,
529 unsigned int pg_start,
530 enum i915_cache_level cache_level)
531{
532 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
533 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
534
535 intel_gtt_insert_sg_entries(st, pg_start, flags);
536
537}
538
539static void i915_ggtt_clear_range(struct drm_device *dev,
540 unsigned int first_entry,
541 unsigned int num_entries)
542{
543 intel_gtt_clear_range(first_entry, num_entries);
544}
545
546
74163907
DV		 547void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
548 enum i915_cache_level cache_level)
d5bd1449
CW		 549{
550 struct drm_device *dev = obj->base.dev;
7faf1ab2
DV		 551 struct drm_i915_private *dev_priv = dev->dev_private;
552
553 dev_priv->gtt.gtt_insert_entries(dev, obj->pages,
554 obj->gtt_space->start >> PAGE_SHIFT,
555 cache_level);
d5bd1449 556
74898d7e 557 obj->has_global_gtt_mapping = 1;
d5bd1449
CW		 558}
559
05394f39 560void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
74163907 561{
7faf1ab2
DV		 562 struct drm_device *dev = obj->base.dev;
563 struct drm_i915_private *dev_priv = dev->dev_private;
564
565 dev_priv->gtt.gtt_clear_range(obj->base.dev,
566 obj->gtt_space->start >> PAGE_SHIFT,
567 obj->base.size >> PAGE_SHIFT);
74898d7e
DV		 568
569 obj->has_global_gtt_mapping = 0;
74163907
DV		 570}
571
572void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
7c2e6fdf 573{
5c042287
BW		 574 struct drm_device *dev = obj->base.dev;
575 struct drm_i915_private *dev_priv = dev->dev_private;
576 bool interruptible;
577
578 interruptible = do_idling(dev_priv);
579
9da3da66
CW		 580 if (!obj->has_dma_mapping)
581 dma_unmap_sg(&dev->pdev->dev,
582 obj->pages->sgl, obj->pages->nents,
583 PCI_DMA_BIDIRECTIONAL);
5c042287
BW		 584
585 undo_idling(dev_priv, interruptible);
7c2e6fdf 586}
644ec02b 587
42d6ab48
CW		 588static void i915_gtt_color_adjust(struct drm_mm_node *node,
589 unsigned long color,
590 unsigned long *start,
591 unsigned long *end)
592{
593 if (node->color != color)
594 *start += 4096;
595
596 if (!list_empty(&node->node_list)) {
597 node = list_entry(node->node_list.next,
598 struct drm_mm_node,
599 node_list);
600 if (node->allocated && node->color != color)
601 *end -= 4096;
602 }
603}
d7e5008f
BW		 604void i915_gem_setup_global_gtt(struct drm_device *dev,
605 unsigned long start,
606 unsigned long mappable_end,
607 unsigned long end)
644ec02b 608{
e78891ca
BW		 609 /* Let GEM Manage all of the aperture.
610 *
611 * However, leave one page at the end still bound to the scratch page.
612 * There are a number of places where the hardware apparently prefetches
613 * past the end of the object, and we've seen multiple hangs with the
614 * GPU head pointer stuck in a batchbuffer bound at the last page of the
615 * aperture. One page should be enough to keep any prefetching inside
616 * of the aperture.
617 */
644ec02b 618 drm_i915_private_t *dev_priv = dev->dev_private;
ed2f3452
CW		 619 struct drm_mm_node *entry;
620 struct drm_i915_gem_object *obj;
621 unsigned long hole_start, hole_end;
644ec02b 622
35451cb6
BW		 623 BUG_ON(mappable_end > end);
624
ed2f3452 625 /* Subtract the guard page ... */
d1dd20a9 626 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
42d6ab48
CW		 627 if (!HAS_LLC(dev))
628 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust;
644ec02b 629
ed2f3452 630 /* Mark any preallocated objects as occupied */
35c20a60 631 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
ed2f3452
CW		 632 DRM_DEBUG_KMS("reserving preallocated space: %x + %zx\n",
633 obj->gtt_offset, obj->base.size);
634
635 BUG_ON(obj->gtt_space != I915_GTT_RESERVED);
636 obj->gtt_space = drm_mm_create_block(&dev_priv->mm.gtt_space,
637 obj->gtt_offset,
638 obj->base.size,
639 false);
640 obj->has_global_gtt_mapping = 1;
641 }
642
5d4545ae 643 dev_priv->gtt.start = start;
5d4545ae 644 dev_priv->gtt.total = end - start;
644ec02b 645
ed2f3452
CW		 646 /* Clear any non-preallocated blocks */
647 drm_mm_for_each_hole(entry, &dev_priv->mm.gtt_space,
648 hole_start, hole_end) {
649 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
650 hole_start, hole_end);
7faf1ab2
DV		 651 dev_priv->gtt.gtt_clear_range(dev, hole_start / PAGE_SIZE,
652 (hole_end-hole_start) / PAGE_SIZE);
ed2f3452
CW		 653 }
654
655 /* And finally clear the reserved guard page */
7faf1ab2 656 dev_priv->gtt.gtt_clear_range(dev, end / PAGE_SIZE - 1, 1);
e76e9aeb
BW		 657}
658
d7e5008f
BW		 659static bool
660intel_enable_ppgtt(struct drm_device *dev)
661{
662 if (i915_enable_ppgtt >= 0)
663 return i915_enable_ppgtt;
664
665#ifdef CONFIG_INTEL_IOMMU
666 /* Disable ppgtt on SNB if VT-d is on. */
667 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
668 return false;
669#endif
670
671 return true;
672}
673
674void i915_gem_init_global_gtt(struct drm_device *dev)
675{
676 struct drm_i915_private *dev_priv = dev->dev_private;
677 unsigned long gtt_size, mappable_size;
d7e5008f 678
a54c0c27 679 gtt_size = dev_priv->gtt.total;
93d18799 680 mappable_size = dev_priv->gtt.mappable_end;
d7e5008f
BW		 681
682 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
e78891ca 683 int ret;
3eb1c005
BW		 684
685 if (INTEL_INFO(dev)->gen <= 7) {
686 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
687 * aperture accordingly when using aliasing ppgtt. */
6670a5a5 688 gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
3eb1c005 689 }
d7e5008f
BW		 690
691 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
692
693 ret = i915_gem_init_aliasing_ppgtt(dev);
e78891ca 694 if (!ret)
d7e5008f 695 return;
e78891ca
BW		 696
697 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
698 drm_mm_takedown(&dev_priv->mm.gtt_space);
6670a5a5 699 gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
d7e5008f 700 }
e78891ca 701 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
e76e9aeb
BW		 702}
703
704static int setup_scratch_page(struct drm_device *dev)
705{
706 struct drm_i915_private *dev_priv = dev->dev_private;
707 struct page *page;
708 dma_addr_t dma_addr;
709
710 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
711 if (page == NULL)
712 return -ENOMEM;
713 get_page(page);
714 set_pages_uc(page, 1);
715
716#ifdef CONFIG_INTEL_IOMMU
717 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
718 PCI_DMA_BIDIRECTIONAL);
719 if (pci_dma_mapping_error(dev->pdev, dma_addr))
720 return -EINVAL;
721#else
722 dma_addr = page_to_phys(page);
723#endif
67167240
BW		 724 dev_priv->gtt.scratch.page = page;
725 dev_priv->gtt.scratch.addr = dma_addr;
e76e9aeb
BW		 726
727 return 0;
728}
729
730static void teardown_scratch_page(struct drm_device *dev)
731{
732 struct drm_i915_private *dev_priv = dev->dev_private;
67167240
BW		 733 set_pages_wb(dev_priv->gtt.scratch.page, 1);
734 pci_unmap_page(dev->pdev, dev_priv->gtt.scratch.addr,
e76e9aeb 735 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
67167240
BW		 736 put_page(dev_priv->gtt.scratch.page);
737 __free_page(dev_priv->gtt.scratch.page);
e76e9aeb
BW		 738}
739
740static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
741{
742 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
743 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
744 return snb_gmch_ctl << 20;
745}
746
baa09f5f 747static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
e76e9aeb
BW		 748{
749 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
750 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
751 return snb_gmch_ctl << 25; /* 32 MB units */
752}
753
baa09f5f
BW		 754static int gen6_gmch_probe(struct drm_device *dev,
755 size_t *gtt_total,
41907ddc
BW		 756 size_t *stolen,
757 phys_addr_t *mappable_base,
758 unsigned long *mappable_end)
e76e9aeb
BW		 759{
760 struct drm_i915_private *dev_priv = dev->dev_private;
761 phys_addr_t gtt_bus_addr;
baa09f5f 762 unsigned int gtt_size;
e76e9aeb 763 u16 snb_gmch_ctl;
e76e9aeb
BW		 764 int ret;
765
41907ddc
BW		 766 *mappable_base = pci_resource_start(dev->pdev, 2);
767 *mappable_end = pci_resource_len(dev->pdev, 2);
768
baa09f5f
BW		 769 /* 64/512MB is the current min/max we actually know of, but this is just
770 * a coarse sanity check.
e76e9aeb 771 */
41907ddc 772 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
baa09f5f
BW		 773 DRM_ERROR("Unknown GMADR size (%lx)\n",
774 dev_priv->gtt.mappable_end);
775 return -ENXIO;
e76e9aeb
BW		 776 }
777
e76e9aeb
BW		 778 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
779 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
e76e9aeb 780 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
baa09f5f 781 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
e76e9aeb 782
c4ae25ec 783 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
e7c2b58b 784 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
e76e9aeb 785
a93e4161
BW		 786 /* For Modern GENs the PTEs and register space are split in the BAR */
787 gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
788 (pci_resource_len(dev->pdev, 0) / 2);
789
baa09f5f 790 dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
5d4545ae 791 if (!dev_priv->gtt.gsm) {
e76e9aeb 792 DRM_ERROR("Failed to map the gtt page table\n");
baa09f5f 793 return -ENOMEM;
e76e9aeb
BW		 794 }
795
baa09f5f
BW		 796 ret = setup_scratch_page(dev);
797 if (ret)
798 DRM_ERROR("Scratch setup failed\n");
e76e9aeb 799
7faf1ab2
DV		 800 dev_priv->gtt.gtt_clear_range = gen6_ggtt_clear_range;
801 dev_priv->gtt.gtt_insert_entries = gen6_ggtt_insert_entries;
802
e76e9aeb
BW		 803 return ret;
804}
805
d93c6233 806static void gen6_gmch_remove(struct drm_device *dev)
e76e9aeb
BW		 807{
808 struct drm_i915_private *dev_priv = dev->dev_private;
5d4545ae 809 iounmap(dev_priv->gtt.gsm);
baa09f5f 810 teardown_scratch_page(dev_priv->dev);
644ec02b 811}
baa09f5f
BW		 812
813static int i915_gmch_probe(struct drm_device *dev,
814 size_t *gtt_total,
41907ddc
BW		 815 size_t *stolen,
816 phys_addr_t *mappable_base,
817 unsigned long *mappable_end)
baa09f5f
BW		 818{
819 struct drm_i915_private *dev_priv = dev->dev_private;
820 int ret;
821
baa09f5f
BW		 822 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
823 if (!ret) {
824 DRM_ERROR("failed to set up gmch\n");
825 return -EIO;
826 }
827
41907ddc 828 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
baa09f5f
BW		 829
830 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
831 dev_priv->gtt.gtt_clear_range = i915_ggtt_clear_range;
832 dev_priv->gtt.gtt_insert_entries = i915_ggtt_insert_entries;
833
834 return 0;
835}
836
837static void i915_gmch_remove(struct drm_device *dev)
838{
839 intel_gmch_remove();
840}
841
842int i915_gem_gtt_init(struct drm_device *dev)
843{
844 struct drm_i915_private *dev_priv = dev->dev_private;
845 struct i915_gtt *gtt = &dev_priv->gtt;
baa09f5f
BW		 846 int ret;
847
baa09f5f 848 if (INTEL_INFO(dev)->gen <= 5) {
b2f21b4d
BW		 849 gtt->gtt_probe = i915_gmch_probe;
850 gtt->gtt_remove = i915_gmch_remove;
baa09f5f 851 } else {
b2f21b4d
BW		 852 gtt->gtt_probe = gen6_gmch_probe;
853 gtt->gtt_remove = gen6_gmch_remove;
854 if (IS_HASWELL(dev))
855 gtt->pte_encode = hsw_pte_encode;
856 else if (IS_VALLEYVIEW(dev))
857 gtt->pte_encode = byt_pte_encode;
858 else
859 gtt->pte_encode = gen6_pte_encode;
baa09f5f
BW		 860 }
861
b2f21b4d
BW		 862 ret = gtt->gtt_probe(dev, &gtt->total, &gtt->stolen_size,
863 &gtt->mappable_base, &gtt->mappable_end);
a54c0c27 864 if (ret)
baa09f5f 865 return ret;
baa09f5f 866
baa09f5f 867 /* GMADR is the PCI mmio aperture into the global GTT. */
b2f21b4d
BW		 868 DRM_INFO("Memory usable by graphics device = %zdM\n", gtt->total >> 20);
869 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
870 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
baa09f5f
BW		 871
872 return 0;
873}
Linux kernel - Deliverables
This page took 0.20997 seconds and 5 git commands to generate.