projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915/gen8: Dynamic page table allocations
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
CommitLineData
76aaf220
DV		 1/*
2 * Copyright © 2010 Daniel Vetter
c4ac524c 3 * Copyright © 2011-2014 Intel Corporation
76aaf220
DV		 4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
0e46ce2e 26#include <linux/seq_file.h>
760285e7
DH		 27#include <drm/drmP.h>
28#include <drm/i915_drm.h>
76aaf220 29#include "i915_drv.h"
5dda8fa3 30#include "i915_vgpu.h"
76aaf220
DV		 31#include "i915_trace.h"
32#include "intel_drv.h"
33
45f8f69a
TU		 34/**
35 * DOC: Global GTT views
36 *
37 * Background and previous state
38 *
39 * Historically objects could exists (be bound) in global GTT space only as
40 * singular instances with a view representing all of the object's backing pages
41 * in a linear fashion. This view will be called a normal view.
42 *
43 * To support multiple views of the same object, where the number of mapped
44 * pages is not equal to the backing store, or where the layout of the pages
45 * is not linear, concept of a GGTT view was added.
46 *
47 * One example of an alternative view is a stereo display driven by a single
48 * image. In this case we would have a framebuffer looking like this
49 * (2x2 pages):
50 *
51 * 12
52 * 34
53 *
54 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
55 * rendering. In contrast, fed to the display engine would be an alternative
56 * view which could look something like this:
57 *
58 * 1212
59 * 3434
60 *
61 * In this example both the size and layout of pages in the alternative view is
62 * different from the normal view.
63 *
64 * Implementation and usage
65 *
66 * GGTT views are implemented using VMAs and are distinguished via enum
67 * i915_ggtt_view_type and struct i915_ggtt_view.
68 *
69 * A new flavour of core GEM functions which work with GGTT bound objects were
ec7adb6e
JL		 70 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
71 * renaming in large amounts of code. They take the struct i915_ggtt_view
72 * parameter encapsulating all metadata required to implement a view.
45f8f69a
TU		 73 *
74 * As a helper for callers which are only interested in the normal view,
75 * globally const i915_ggtt_view_normal singleton instance exists. All old core
76 * GEM API functions, the ones not taking the view parameter, are operating on,
77 * or with the normal GGTT view.
78 *
79 * Code wanting to add or use a new GGTT view needs to:
80 *
81 * 1. Add a new enum with a suitable name.
82 * 2. Extend the metadata in the i915_ggtt_view structure if required.
83 * 3. Add support to i915_get_vma_pages().
84 *
85 * New views are required to build a scatter-gather table from within the
86 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
87 * exists for the lifetime of an VMA.
88 *
89 * Core API is designed to have copy semantics which means that passed in
90 * struct i915_ggtt_view does not need to be persistent (left around after
91 * calling the core API functions).
92 *
93 */
94
fe14d5f4 95const struct i915_ggtt_view i915_ggtt_view_normal;
9abc4648
JL		 96const struct i915_ggtt_view i915_ggtt_view_rotated = {
97 .type = I915_GGTT_VIEW_ROTATED
98};
fe14d5f4 99
ee0ce478
VS		 100static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
101static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
a2319c08 102
cfa7c862
DV		 103static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
104{
1893a71b
CW		 105 bool has_aliasing_ppgtt;
106 bool has_full_ppgtt;
107
108 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
109 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
1893a71b 110
71ba2d64
YZ		 111 if (intel_vgpu_active(dev))
112 has_full_ppgtt = false; /* emulation is too hard */
113
70ee45e1
DL		 114 /*
115 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
116 * execlists, the sole mechanism available to submit work.
117 */
118 if (INTEL_INFO(dev)->gen < 9 &&
119 (enable_ppgtt == 0 || !has_aliasing_ppgtt))
cfa7c862
DV		 120 return 0;
121
122 if (enable_ppgtt == 1)
123 return 1;
124
1893a71b 125 if (enable_ppgtt == 2 && has_full_ppgtt)
cfa7c862
DV		 126 return 2;
127
93a25a9e
DV		 128#ifdef CONFIG_INTEL_IOMMU
129 /* Disable ppgtt on SNB if VT-d is on. */
130 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
131 DRM_INFO("Disabling PPGTT because VT-d is on\n");
cfa7c862 132 return 0;
93a25a9e
DV		 133 }
134#endif
135
62942ed7 136 /* Early VLV doesn't have this */
ca2aed6c
VS		 137 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
138 dev->pdev->revision < 0xb) {
62942ed7
JB		 139 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
140 return 0;
141 }
142
2f82bbdf
MT		 143 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
144 return 2;
145 else
146 return has_aliasing_ppgtt ? 1 : 0;
93a25a9e
DV		 147}
148
6f65e29a
BW		 149static void ppgtt_bind_vma(struct i915_vma *vma,
150 enum i915_cache_level cache_level,
151 u32 flags);
152static void ppgtt_unbind_vma(struct i915_vma *vma);
153
07749ef3
MT		 154static inline gen8_pte_t gen8_pte_encode(dma_addr_t addr,
155 enum i915_cache_level level,
156 bool valid)
94ec8f61 157{
07749ef3 158 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
94ec8f61 159 pte |= addr;
63c42e56
BW		 160
161 switch (level) {
162 case I915_CACHE_NONE:
fbe5d36e 163 pte |= PPAT_UNCACHED_INDEX;
63c42e56
BW		 164 break;
165 case I915_CACHE_WT:
166 pte |= PPAT_DISPLAY_ELLC_INDEX;
167 break;
168 default:
169 pte |= PPAT_CACHED_INDEX;
170 break;
171 }
172
94ec8f61
BW		 173 return pte;
174}
175
07749ef3
MT		 176static inline gen8_pde_t gen8_pde_encode(struct drm_device *dev,
177 dma_addr_t addr,
178 enum i915_cache_level level)
b1fe6673 179{
07749ef3 180 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
b1fe6673
BW		 181 pde |= addr;
182 if (level != I915_CACHE_NONE)
183 pde |= PPAT_CACHED_PDE_INDEX;
184 else
185 pde |= PPAT_UNCACHED_INDEX;
186 return pde;
187}
188
07749ef3
MT		 189static gen6_pte_t snb_pte_encode(dma_addr_t addr,
190 enum i915_cache_level level,
191 bool valid, u32 unused)
54d12527 192{
07749ef3 193 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
54d12527 194 pte |= GEN6_PTE_ADDR_ENCODE(addr);
e7210c3c
BW		 195
196 switch (level) {
350ec881
CW		 197 case I915_CACHE_L3_LLC:
198 case I915_CACHE_LLC:
199 pte |= GEN6_PTE_CACHE_LLC;
200 break;
201 case I915_CACHE_NONE:
202 pte |= GEN6_PTE_UNCACHED;
203 break;
204 default:
5f77eeb0 205 MISSING_CASE(level);
350ec881
CW		 206 }
207
208 return pte;
209}
210
07749ef3
MT		 211static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
212 enum i915_cache_level level,
213 bool valid, u32 unused)
350ec881 214{
07749ef3 215 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
350ec881
CW		 216 pte |= GEN6_PTE_ADDR_ENCODE(addr);
217
218 switch (level) {
219 case I915_CACHE_L3_LLC:
220 pte |= GEN7_PTE_CACHE_L3_LLC;
e7210c3c
BW		 221 break;
222 case I915_CACHE_LLC:
223 pte |= GEN6_PTE_CACHE_LLC;
224 break;
225 case I915_CACHE_NONE:
9119708c 226 pte |= GEN6_PTE_UNCACHED;
e7210c3c
BW		 227 break;
228 default:
5f77eeb0 229 MISSING_CASE(level);
e7210c3c
BW		 230 }
231
54d12527
BW		 232 return pte;
233}
234
07749ef3
MT		 235static gen6_pte_t byt_pte_encode(dma_addr_t addr,
236 enum i915_cache_level level,
237 bool valid, u32 flags)
93c34e70 238{
07749ef3 239 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
93c34e70
KG		 240 pte |= GEN6_PTE_ADDR_ENCODE(addr);
241
24f3a8cf
AG		 242 if (!(flags & PTE_READ_ONLY))
243 pte |= BYT_PTE_WRITEABLE;
93c34e70
KG		 244
245 if (level != I915_CACHE_NONE)
246 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
247
248 return pte;
249}
250
07749ef3
MT		 251static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
252 enum i915_cache_level level,
253 bool valid, u32 unused)
9119708c 254{
07749ef3 255 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
0d8ff15e 256 pte |= HSW_PTE_ADDR_ENCODE(addr);
9119708c
KG		 257
258 if (level != I915_CACHE_NONE)
87a6b688 259 pte |= HSW_WB_LLC_AGE3;
9119708c
KG		 260
261 return pte;
262}
263
07749ef3
MT		 264static gen6_pte_t iris_pte_encode(dma_addr_t addr,
265 enum i915_cache_level level,
266 bool valid, u32 unused)
4d15c145 267{
07749ef3 268 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
4d15c145
BW		 269 pte |= HSW_PTE_ADDR_ENCODE(addr);
270
651d794f
CW		 271 switch (level) {
272 case I915_CACHE_NONE:
273 break;
274 case I915_CACHE_WT:
c51e9701 275 pte |= HSW_WT_ELLC_LLC_AGE3;
651d794f
CW		 276 break;
277 default:
c51e9701 278 pte |= HSW_WB_ELLC_LLC_AGE3;
651d794f
CW		 279 break;
280 }
4d15c145
BW		 281
282 return pte;
283}
284
678d96fb
BW		 285#define i915_dma_unmap_single(px, dev) \
286 __i915_dma_unmap_single((px)->daddr, dev)
287
288static inline void __i915_dma_unmap_single(dma_addr_t daddr,
289 struct drm_device *dev)
290{
291 struct device *device = &dev->pdev->dev;
292
293 dma_unmap_page(device, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
294}
295
296/**
297 * i915_dma_map_single() - Create a dma mapping for a page table/dir/etc.
298 * @px: Page table/dir/etc to get a DMA map for
299 * @dev: drm device
300 *
301 * Page table allocations are unified across all gens. They always require a
302 * single 4k allocation, as well as a DMA mapping. If we keep the structs
303 * symmetric here, the simple macro covers us for every page table type.
304 *
305 * Return: 0 if success.
306 */
307#define i915_dma_map_single(px, dev) \
308 i915_dma_map_page_single((px)->page, (dev), &(px)->daddr)
309
310static inline int i915_dma_map_page_single(struct page *page,
311 struct drm_device *dev,
312 dma_addr_t *daddr)
313{
314 struct device *device = &dev->pdev->dev;
315
316 *daddr = dma_map_page(device, page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
1266cdb1
MT		 317 if (dma_mapping_error(device, *daddr))
318 return -ENOMEM;
319
320 return 0;
678d96fb
BW		 321}
322
ec565b3c 323static void unmap_and_free_pt(struct i915_page_table *pt,
678d96fb 324 struct drm_device *dev)
06fda602
BW		 325{
326 if (WARN_ON(!pt->page))
327 return;
678d96fb
BW		 328
329 i915_dma_unmap_single(pt, dev);
06fda602 330 __free_page(pt->page);
678d96fb 331 kfree(pt->used_ptes);
06fda602
BW		 332 kfree(pt);
333}
334
5a8e9943 335static void gen8_initialize_pt(struct i915_address_space *vm,
e5815a2e 336 struct i915_page_table *pt)
5a8e9943
MT		 337{
338 gen8_pte_t *pt_vaddr, scratch_pte;
339 int i;
340
341 pt_vaddr = kmap_atomic(pt->page);
342 scratch_pte = gen8_pte_encode(vm->scratch.addr,
343 I915_CACHE_LLC, true);
344
345 for (i = 0; i < GEN8_PTES; i++)
346 pt_vaddr[i] = scratch_pte;
347
348 if (!HAS_LLC(vm->dev))
349 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
350 kunmap_atomic(pt_vaddr);
351}
352
ec565b3c 353static struct i915_page_table *alloc_pt_single(struct drm_device *dev)
06fda602 354{
ec565b3c 355 struct i915_page_table *pt;
678d96fb
BW		 356 const size_t count = INTEL_INFO(dev)->gen >= 8 ?
357 GEN8_PTES : GEN6_PTES;
358 int ret = -ENOMEM;
06fda602
BW		 359
360 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
361 if (!pt)
362 return ERR_PTR(-ENOMEM);
363
678d96fb
BW		 364 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
365 GFP_KERNEL);
366
367 if (!pt->used_ptes)
368 goto fail_bitmap;
369
4933d519 370 pt->page = alloc_page(GFP_KERNEL);
678d96fb
BW		 371 if (!pt->page)
372 goto fail_page;
373
374 ret = i915_dma_map_single(pt, dev);
375 if (ret)
376 goto fail_dma;
06fda602
BW		 377
378 return pt;
678d96fb
BW		 379
380fail_dma:
381 __free_page(pt->page);
382fail_page:
383 kfree(pt->used_ptes);
384fail_bitmap:
385 kfree(pt);
386
387 return ERR_PTR(ret);
06fda602
BW		 388}
389
390/**
391 * alloc_pt_range() - Allocate a multiple page tables
392 * @pd: The page directory which will have at least @count entries
393 * available to point to the allocated page tables.
394 * @pde: First page directory entry for which we are allocating.
395 * @count: Number of pages to allocate.
719cd21c 396 * @dev: DRM device.
06fda602
BW		 397 *
398 * Allocates multiple page table pages and sets the appropriate entries in the
399 * page table structure within the page directory. Function cleans up after
400 * itself on any failures.
401 *
402 * Return: 0 if allocation succeeded.
403 */
ec565b3c 404static int alloc_pt_range(struct i915_page_directory *pd, uint16_t pde, size_t count,
4933d519 405 struct drm_device *dev)
06fda602
BW		 406{
407 int i, ret;
408
409 /* 512 is the max page tables per page_directory on any platform. */
07749ef3 410 if (WARN_ON(pde + count > I915_PDES))
06fda602
BW		 411 return -EINVAL;
412
413 for (i = pde; i < pde + count; i++) {
ec565b3c 414 struct i915_page_table *pt = alloc_pt_single(dev);
06fda602
BW		 415
416 if (IS_ERR(pt)) {
417 ret = PTR_ERR(pt);
418 goto err_out;
419 }
420 WARN(pd->page_table[i],
686135da 421 "Leaking page directory entry %d (%p)\n",
06fda602
BW		 422 i, pd->page_table[i]);
423 pd->page_table[i] = pt;
424 }
425
426 return 0;
427
428err_out:
429 while (i-- > pde)
06dc68d6 430 unmap_and_free_pt(pd->page_table[i], dev);
06fda602
BW		 431 return ret;
432}
433
e5815a2e
MT		 434static void unmap_and_free_pd(struct i915_page_directory *pd,
435 struct drm_device *dev)
06fda602
BW		 436{
437 if (pd->page) {
e5815a2e 438 i915_dma_unmap_single(pd, dev);
06fda602 439 __free_page(pd->page);
33c8819f 440 kfree(pd->used_pdes);
06fda602
BW		 441 kfree(pd);
442 }
443}
444
e5815a2e 445static struct i915_page_directory *alloc_pd_single(struct drm_device *dev)
06fda602 446{
ec565b3c 447 struct i915_page_directory *pd;
33c8819f 448 int ret = -ENOMEM;
06fda602
BW		 449
450 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
451 if (!pd)
452 return ERR_PTR(-ENOMEM);
453
33c8819f
MT		 454 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
455 sizeof(*pd->used_pdes), GFP_KERNEL);
456 if (!pd->used_pdes)
457 goto free_pd;
458
5a8e9943 459 pd->page = alloc_page(GFP_KERNEL);
33c8819f
MT		 460 if (!pd->page)
461 goto free_bitmap;
06fda602 462
e5815a2e 463 ret = i915_dma_map_single(pd, dev);
33c8819f
MT		 464 if (ret)
465 goto free_page;
e5815a2e 466
06fda602 467 return pd;
33c8819f
MT		 468
469free_page:
470 __free_page(pd->page);
471free_bitmap:
472 kfree(pd->used_pdes);
473free_pd:
474 kfree(pd);
475
476 return ERR_PTR(ret);
06fda602
BW		 477}
478
94e409c1 479/* Broadwell Page Directory Pointer Descriptors */
7cb6d7ac
MT		 480static int gen8_write_pdp(struct intel_engine_cs *ring,
481 unsigned entry,
482 dma_addr_t addr)
94e409c1
BW		 483{
484 int ret;
485
486 BUG_ON(entry >= 4);
487
488 ret = intel_ring_begin(ring, 6);
489 if (ret)
490 return ret;
491
492 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
493 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
7cb6d7ac 494 intel_ring_emit(ring, upper_32_bits(addr));
94e409c1
BW		 495 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
496 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
7cb6d7ac 497 intel_ring_emit(ring, lower_32_bits(addr));
94e409c1
BW		 498 intel_ring_advance(ring);
499
500 return 0;
501}
502
eeb9488e 503static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
6689c167 504 struct intel_engine_cs *ring)
94e409c1 505{
eeb9488e 506 int i, ret;
94e409c1 507
7cb6d7ac
MT		 508 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
509 struct i915_page_directory *pd = ppgtt->pdp.page_directory[i];
510 dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr;
511 /* The page directory might be NULL, but we need to clear out
512 * whatever the previous context might have used. */
513 ret = gen8_write_pdp(ring, i, pd_daddr);
eeb9488e
BW		 514 if (ret)
515 return ret;
94e409c1 516 }
d595bd4b 517
eeb9488e 518 return 0;
94e409c1
BW		 519}
520
459108b8 521static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 522 uint64_t start,
523 uint64_t length,
459108b8
BW		 524 bool use_scratch)
525{
526 struct i915_hw_ppgtt *ppgtt =
527 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 528 gen8_pte_t *pt_vaddr, scratch_pte;
7ad47cf2
BW		 529 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
530 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
531 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
782f1495 532 unsigned num_entries = length >> PAGE_SHIFT;
459108b8
BW		 533 unsigned last_pte, i;
534
535 scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
536 I915_CACHE_LLC, use_scratch);
537
538 while (num_entries) {
ec565b3c
MT		 539 struct i915_page_directory *pd;
540 struct i915_page_table *pt;
06fda602
BW		 541 struct page *page_table;
542
543 if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))
544 continue;
545
546 pd = ppgtt->pdp.page_directory[pdpe];
547
548 if (WARN_ON(!pd->page_table[pde]))
549 continue;
550
551 pt = pd->page_table[pde];
552
553 if (WARN_ON(!pt->page))
554 continue;
555
556 page_table = pt->page;
459108b8 557
7ad47cf2 558 last_pte = pte + num_entries;
07749ef3
MT		 559 if (last_pte > GEN8_PTES)
560 last_pte = GEN8_PTES;
459108b8
BW		 561
562 pt_vaddr = kmap_atomic(page_table);
563
7ad47cf2 564 for (i = pte; i < last_pte; i++) {
459108b8 565 pt_vaddr[i] = scratch_pte;
7ad47cf2
BW		 566 num_entries--;
567 }
459108b8 568
fd1ab8f4
RB		 569 if (!HAS_LLC(ppgtt->base.dev))
570 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
459108b8
BW		 571 kunmap_atomic(pt_vaddr);
572
7ad47cf2 573 pte = 0;
07749ef3 574 if (++pde == I915_PDES) {
7ad47cf2
BW		 575 pdpe++;
576 pde = 0;
577 }
459108b8
BW		 578 }
579}
580
9df15b49
BW		 581static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
582 struct sg_table *pages,
782f1495 583 uint64_t start,
24f3a8cf 584 enum i915_cache_level cache_level, u32 unused)
9df15b49
BW		 585{
586 struct i915_hw_ppgtt *ppgtt =
587 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 588 gen8_pte_t *pt_vaddr;
7ad47cf2
BW		 589 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
590 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
591 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
9df15b49
BW		 592 struct sg_page_iter sg_iter;
593
6f1cc993 594 pt_vaddr = NULL;
7ad47cf2 595
9df15b49 596 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
76643600 597 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
7ad47cf2
BW		 598 break;
599
d7b3de91 600 if (pt_vaddr == NULL) {
ec565b3c
MT		 601 struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
602 struct i915_page_table *pt = pd->page_table[pde];
06fda602 603 struct page *page_table = pt->page;
d7b3de91
BW		 604
605 pt_vaddr = kmap_atomic(page_table);
606 }
9df15b49 607
7ad47cf2 608 pt_vaddr[pte] =
6f1cc993
CW		 609 gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
610 cache_level, true);
07749ef3 611 if (++pte == GEN8_PTES) {
fd1ab8f4
RB		 612 if (!HAS_LLC(ppgtt->base.dev))
613 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
9df15b49 614 kunmap_atomic(pt_vaddr);
6f1cc993 615 pt_vaddr = NULL;
07749ef3 616 if (++pde == I915_PDES) {
7ad47cf2
BW		 617 pdpe++;
618 pde = 0;
619 }
620 pte = 0;
9df15b49
BW		 621 }
622 }
fd1ab8f4
RB		 623 if (pt_vaddr) {
624 if (!HAS_LLC(ppgtt->base.dev))
625 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
6f1cc993 626 kunmap_atomic(pt_vaddr);
fd1ab8f4 627 }
9df15b49
BW		 628}
629
69876bed
MT		 630static void __gen8_do_map_pt(gen8_pde_t * const pde,
631 struct i915_page_table *pt,
632 struct drm_device *dev)
633{
634 gen8_pde_t entry =
635 gen8_pde_encode(dev, pt->daddr, I915_CACHE_LLC);
636 *pde = entry;
637}
638
639static void gen8_initialize_pd(struct i915_address_space *vm,
640 struct i915_page_directory *pd)
641{
642 struct i915_hw_ppgtt *ppgtt =
643 container_of(vm, struct i915_hw_ppgtt, base);
644 gen8_pde_t *page_directory;
645 struct i915_page_table *pt;
646 int i;
647
648 page_directory = kmap_atomic(pd->page);
649 pt = ppgtt->scratch_pt;
650 for (i = 0; i < I915_PDES; i++)
651 /* Map the PDE to the page table */
652 __gen8_do_map_pt(page_directory + i, pt, vm->dev);
653
654 if (!HAS_LLC(vm->dev))
655 drm_clflush_virt_range(page_directory, PAGE_SIZE);
e5815a2e
MT		 656 kunmap_atomic(page_directory);
657}
658
ec565b3c 659static void gen8_free_page_tables(struct i915_page_directory *pd, struct drm_device *dev)
7ad47cf2
BW		 660{
661 int i;
662
06fda602 663 if (!pd->page)
7ad47cf2
BW		 664 return;
665
33c8819f 666 for_each_set_bit(i, pd->used_pdes, I915_PDES) {
06fda602
BW		 667 if (WARN_ON(!pd->page_table[i]))
668 continue;
7ad47cf2 669
06dc68d6 670 unmap_and_free_pt(pd->page_table[i], dev);
06fda602
BW		 671 pd->page_table[i] = NULL;
672 }
d7b3de91
BW		 673}
674
675static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
b45a6715
BW		 676{
677 int i;
678
33c8819f 679 for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
06fda602
BW		 680 if (WARN_ON(!ppgtt->pdp.page_directory[i]))
681 continue;
682
06dc68d6 683 gen8_free_page_tables(ppgtt->pdp.page_directory[i], ppgtt->base.dev);
e5815a2e 684 unmap_and_free_pd(ppgtt->pdp.page_directory[i], ppgtt->base.dev);
7ad47cf2 685 }
69876bed 686
e5815a2e 687 unmap_and_free_pd(ppgtt->scratch_pd, ppgtt->base.dev);
69876bed 688 unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
b45a6715
BW		 689}
690
37aca44a
BW		 691static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
692{
693 struct i915_hw_ppgtt *ppgtt =
694 container_of(vm, struct i915_hw_ppgtt, base);
37aca44a 695
b45a6715 696 gen8_ppgtt_free(ppgtt);
37aca44a
BW		 697}
698
d7b2633d
MT		 699/**
700 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
701 * @ppgtt: Master ppgtt structure.
702 * @pd: Page directory for this address range.
703 * @start: Starting virtual address to begin allocations.
704 * @length Size of the allocations.
705 * @new_pts: Bitmap set by function with new allocations. Likely used by the
706 * caller to free on error.
707 *
708 * Allocate the required number of page tables. Extremely similar to
709 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
710 * the page directory boundary (instead of the page directory pointer). That
711 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
712 * possible, and likely that the caller will need to use multiple calls of this
713 * function to achieve the appropriate allocation.
714 *
715 * Return: 0 if success; negative error code otherwise.
716 */
e5815a2e
MT		 717static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt,
718 struct i915_page_directory *pd,
5441f0cb 719 uint64_t start,
d7b2633d
MT		 720 uint64_t length,
721 unsigned long *new_pts)
bf2b4ed2 722{
e5815a2e 723 struct drm_device *dev = ppgtt->base.dev;
d7b2633d 724 struct i915_page_table *pt;
5441f0cb
MT		 725 uint64_t temp;
726 uint32_t pde;
bf2b4ed2 727
d7b2633d
MT		 728 gen8_for_each_pde(pt, pd, start, length, temp, pde) {
729 /* Don't reallocate page tables */
730 if (pt) {
731 /* Scratch is never allocated this way */
732 WARN_ON(pt == ppgtt->scratch_pt);
733 continue;
734 }
735
736 pt = alloc_pt_single(dev);
737 if (IS_ERR(pt))
5441f0cb
MT		 738 goto unwind_out;
739
d7b2633d
MT		 740 gen8_initialize_pt(&ppgtt->base, pt);
741 pd->page_table[pde] = pt;
742 set_bit(pde, new_pts);
7ad47cf2
BW		 743 }
744
bf2b4ed2 745 return 0;
7ad47cf2
BW		 746
747unwind_out:
d7b2633d 748 for_each_set_bit(pde, new_pts, I915_PDES)
e5815a2e 749 unmap_and_free_pt(pd->page_table[pde], dev);
7ad47cf2 750
d7b3de91 751 return -ENOMEM;
bf2b4ed2
BW		 752}
753
d7b2633d
MT		 754/**
755 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
756 * @ppgtt: Master ppgtt structure.
757 * @pdp: Page directory pointer for this address range.
758 * @start: Starting virtual address to begin allocations.
759 * @length Size of the allocations.
760 * @new_pds Bitmap set by function with new allocations. Likely used by the
761 * caller to free on error.
762 *
763 * Allocate the required number of page directories starting at the pde index of
764 * @start, and ending at the pde index @start + @length. This function will skip
765 * over already allocated page directories within the range, and only allocate
766 * new ones, setting the appropriate pointer within the pdp as well as the
767 * correct position in the bitmap @new_pds.
768 *
769 * The function will only allocate the pages within the range for a give page
770 * directory pointer. In other words, if @start + @length straddles a virtually
771 * addressed PDP boundary (512GB for 4k pages), there will be more allocations
772 * required by the caller, This is not currently possible, and the BUG in the
773 * code will prevent it.
774 *
775 * Return: 0 if success; negative error code otherwise.
776 */
c488dbba
MT		 777static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
778 struct i915_page_directory_pointer *pdp,
69876bed 779 uint64_t start,
d7b2633d
MT		 780 uint64_t length,
781 unsigned long *new_pds)
bf2b4ed2 782{
e5815a2e 783 struct drm_device *dev = ppgtt->base.dev;
d7b2633d 784 struct i915_page_directory *pd;
69876bed
MT		 785 uint64_t temp;
786 uint32_t pdpe;
787
d7b2633d
MT		 788 WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
789
69876bed
MT		 790 /* FIXME: PPGTT container_of won't work for 64b */
791 WARN_ON((start + length) > 0x800000000ULL);
792
d7b2633d
MT		 793 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
794 if (pd)
795 continue;
33c8819f 796
d7b2633d
MT		 797 pd = alloc_pd_single(dev);
798 if (IS_ERR(pd))
d7b3de91 799 goto unwind_out;
69876bed 800
d7b2633d
MT		 801 gen8_initialize_pd(&ppgtt->base, pd);
802 pdp->page_directory[pdpe] = pd;
803 set_bit(pdpe, new_pds);
d7b3de91
BW		 804 }
805
bf2b4ed2 806 return 0;
d7b3de91
BW		 807
808unwind_out:
d7b2633d 809 for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
e5815a2e 810 unmap_and_free_pd(pdp->page_directory[pdpe], dev);
d7b3de91
BW		 811
812 return -ENOMEM;
bf2b4ed2
BW		 813}
814
d7b2633d
MT		 815static void
816free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
817{
818 int i;
819
820 for (i = 0; i < GEN8_LEGACY_PDPES; i++)
821 kfree(new_pts[i]);
822 kfree(new_pts);
823 kfree(new_pds);
824}
825
826/* Fills in the page directory bitmap, and the array of page tables bitmap. Both
827 * of these are based on the number of PDPEs in the system.
828 */
829static
830int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
831 unsigned long ***new_pts)
832{
833 int i;
834 unsigned long *pds;
835 unsigned long **pts;
836
837 pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
838 if (!pds)
839 return -ENOMEM;
840
841 pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
842 if (!pts) {
843 kfree(pds);
844 return -ENOMEM;
845 }
846
847 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
848 pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
849 sizeof(unsigned long), GFP_KERNEL);
850 if (!pts[i])
851 goto err_out;
852 }
853
854 *new_pds = pds;
855 *new_pts = pts;
856
857 return 0;
858
859err_out:
860 free_gen8_temp_bitmaps(pds, pts);
861 return -ENOMEM;
862}
863
e5815a2e
MT		 864static int gen8_alloc_va_range(struct i915_address_space *vm,
865 uint64_t start,
866 uint64_t length)
bf2b4ed2 867{
e5815a2e
MT		 868 struct i915_hw_ppgtt *ppgtt =
869 container_of(vm, struct i915_hw_ppgtt, base);
d7b2633d 870 unsigned long *new_page_dirs, **new_page_tables;
5441f0cb 871 struct i915_page_directory *pd;
33c8819f
MT		 872 const uint64_t orig_start = start;
873 const uint64_t orig_length = length;
5441f0cb
MT		 874 uint64_t temp;
875 uint32_t pdpe;
bf2b4ed2
BW		 876 int ret;
877
d7b2633d
MT		 878#ifndef CONFIG_64BIT
879 /* Disallow 64b address on 32b platforms. Nothing is wrong with doing
880 * this in hardware, but a lot of the drm code is not prepared to handle
881 * 64b offset on 32b platforms.
882 * This will be addressed when 48b PPGTT is added */
883 if (start + length > 0x100000000ULL)
884 return -E2BIG;
885#endif
886
887 /* Wrap is never okay since we can only represent 48b, and we don't
888 * actually use the other side of the canonical address space.
889 */
890 if (WARN_ON(start + length < start))
891 return -ERANGE;
892
893 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
bf2b4ed2
BW		 894 if (ret)
895 return ret;
896
d7b2633d
MT		 897 /* Do the allocations first so we can easily bail out */
898 ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
899 new_page_dirs);
900 if (ret) {
901 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
902 return ret;
903 }
904
905 /* For every page directory referenced, allocate page tables */
5441f0cb 906 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
d7b2633d
MT		 907 ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length,
908 new_page_tables[pdpe]);
5441f0cb
MT		 909 if (ret)
910 goto err_out;
5441f0cb
MT		 911 }
912
33c8819f
MT		 913 start = orig_start;
914 length = orig_length;
915
d7b2633d
MT		 916 /* Allocations have completed successfully, so set the bitmaps, and do
917 * the mappings. */
33c8819f 918 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
d7b2633d 919 gen8_pde_t *const page_directory = kmap_atomic(pd->page);
33c8819f
MT		 920 struct i915_page_table *pt;
921 uint64_t pd_len = gen8_clamp_pd(start, length);
922 uint64_t pd_start = start;
923 uint32_t pde;
924
d7b2633d
MT		 925 /* Every pd should be allocated, we just did that above. */
926 WARN_ON(!pd);
927
928 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
929 /* Same reasoning as pd */
930 WARN_ON(!pt);
931 WARN_ON(!pd_len);
932 WARN_ON(!gen8_pte_count(pd_start, pd_len));
933
934 /* Set our used ptes within the page table */
935 bitmap_set(pt->used_ptes,
936 gen8_pte_index(pd_start),
937 gen8_pte_count(pd_start, pd_len));
938
939 /* Our pde is now pointing to the pagetable, pt */
33c8819f 940 set_bit(pde, pd->used_pdes);
d7b2633d
MT		 941
942 /* Map the PDE to the page table */
943 __gen8_do_map_pt(page_directory + pde, pt, vm->dev);
944
945 /* NB: We haven't yet mapped ptes to pages. At this
946 * point we're still relying on insert_entries() */
33c8819f 947 }
d7b2633d
MT		 948
949 if (!HAS_LLC(vm->dev))
950 drm_clflush_virt_range(page_directory, PAGE_SIZE);
951
952 kunmap_atomic(page_directory);
953
33c8819f
MT		 954 set_bit(pdpe, ppgtt->pdp.used_pdpes);
955 }
956
d7b2633d 957 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
d7b3de91 958 return 0;
bf2b4ed2 959
d7b3de91 960err_out:
d7b2633d
MT		 961 while (pdpe--) {
962 for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES)
963 unmap_and_free_pt(ppgtt->pdp.page_directory[pdpe]->page_table[temp], vm->dev);
964 }
965
966 for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
967 unmap_and_free_pd(ppgtt->pdp.page_directory[pdpe], vm->dev);
968
969 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
bf2b4ed2
BW		 970 return ret;
971}
972
eb0b44ad 973/*
f3a964b9
BW		 974 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
975 * with a net effect resembling a 2-level page table in normal x86 terms. Each
976 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
977 * space.
37aca44a 978 *
f3a964b9 979 */
d7b2633d 980static int gen8_ppgtt_init_common(struct i915_hw_ppgtt *ppgtt, uint64_t size)
37aca44a 981{
69876bed
MT		 982 ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev);
983 if (IS_ERR(ppgtt->scratch_pt))
984 return PTR_ERR(ppgtt->scratch_pt);
985
e5815a2e 986 ppgtt->scratch_pd = alloc_pd_single(ppgtt->base.dev);
7cb6d7ac
MT		 987 if (IS_ERR(ppgtt->scratch_pd))
988 return PTR_ERR(ppgtt->scratch_pd);
989
69876bed 990 gen8_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);
7cb6d7ac 991 gen8_initialize_pd(&ppgtt->base, ppgtt->scratch_pd);
69876bed 992
d7b2633d
MT		 993 ppgtt->base.start = 0;
994 ppgtt->base.total = size;
995 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
996 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
997
998 ppgtt->switch_mm = gen8_mm_switch;
999
1000 return 0;
1001}
1002
1003static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1004{
1005 struct drm_device *dev = ppgtt->base.dev;
1006 struct drm_i915_private *dev_priv = dev->dev_private;
1007 uint64_t start = 0, size = dev_priv->gtt.base.total;
1008 int ret;
1009
1010 ret = gen8_ppgtt_init_common(ppgtt, dev_priv->gtt.base.total);
1011 if (ret)
1012 return ret;
1013
1014 /* Aliasing PPGTT has to always work and be mapped because of the way we
1015 * use RESTORE_INHIBIT in the context switch. This will be fixed
1016 * eventually. */
e5815a2e 1017 ret = gen8_alloc_va_range(&ppgtt->base, start, size);
7cb6d7ac 1018 if (ret) {
e5815a2e 1019 unmap_and_free_pd(ppgtt->scratch_pd, ppgtt->base.dev);
7cb6d7ac 1020 unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
bf2b4ed2 1021 return ret;
7cb6d7ac 1022 }
f3a964b9 1023
d7b2633d
MT		 1024 ppgtt->base.allocate_va_range = NULL;
1025 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1026 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
37aca44a 1027
d7b2633d
MT		 1028 return 0;
1029}
1030
1031static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1032{
1033 struct drm_device *dev = ppgtt->base.dev;
1034 struct drm_i915_private *dev_priv = dev->dev_private;
1035 int ret;
1036
1037 ret = gen8_ppgtt_init_common(ppgtt, dev_priv->gtt.base.total);
1038 if (ret)
1039 return ret;
1040
1041 ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1042 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
2934368e 1043
28cf5415 1044 return 0;
37aca44a
BW		 1045}
1046
87d60b63
BW		 1047static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1048{
87d60b63 1049 struct i915_address_space *vm = &ppgtt->base;
09942c65 1050 struct i915_page_table *unused;
07749ef3 1051 gen6_pte_t scratch_pte;
87d60b63 1052 uint32_t pd_entry;
09942c65
MT		 1053 uint32_t pte, pde, temp;
1054 uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
87d60b63 1055
24f3a8cf 1056 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
87d60b63 1057
09942c65 1058 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
87d60b63 1059 u32 expected;
07749ef3 1060 gen6_pte_t *pt_vaddr;
06fda602 1061 dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr;
09942c65 1062 pd_entry = readl(ppgtt->pd_addr + pde);
87d60b63
BW		 1063 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1064
1065 if (pd_entry != expected)
1066 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1067 pde,
1068 pd_entry,
1069 expected);
1070 seq_printf(m, "\tPDE: %x\n", pd_entry);
1071
06fda602 1072 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page);
07749ef3 1073 for (pte = 0; pte < GEN6_PTES; pte+=4) {
87d60b63 1074 unsigned long va =
07749ef3 1075 (pde * PAGE_SIZE * GEN6_PTES) +
87d60b63
BW		 1076 (pte * PAGE_SIZE);
1077 int i;
1078 bool found = false;
1079 for (i = 0; i < 4; i++)
1080 if (pt_vaddr[pte + i] != scratch_pte)
1081 found = true;
1082 if (!found)
1083 continue;
1084
1085 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1086 for (i = 0; i < 4; i++) {
1087 if (pt_vaddr[pte + i] != scratch_pte)
1088 seq_printf(m, " %08x", pt_vaddr[pte + i]);
1089 else
1090 seq_puts(m, " SCRATCH ");
1091 }
1092 seq_puts(m, "\n");
1093 }
1094 kunmap_atomic(pt_vaddr);
1095 }
1096}
1097
678d96fb 1098/* Write pde (index) from the page directory @pd to the page table @pt */
ec565b3c
MT		 1099static void gen6_write_pde(struct i915_page_directory *pd,
1100 const int pde, struct i915_page_table *pt)
6197349b 1101{
678d96fb
BW		 1102 /* Caller needs to make sure the write completes if necessary */
1103 struct i915_hw_ppgtt *ppgtt =
1104 container_of(pd, struct i915_hw_ppgtt, pd);
1105 u32 pd_entry;
6197349b 1106
678d96fb
BW		 1107 pd_entry = GEN6_PDE_ADDR_ENCODE(pt->daddr);
1108 pd_entry |= GEN6_PDE_VALID;
6197349b 1109
678d96fb
BW		 1110 writel(pd_entry, ppgtt->pd_addr + pde);
1111}
6197349b 1112
678d96fb
BW		 1113/* Write all the page tables found in the ppgtt structure to incrementing page
1114 * directories. */
1115static void gen6_write_page_range(struct drm_i915_private *dev_priv,
ec565b3c 1116 struct i915_page_directory *pd,
678d96fb
BW		 1117 uint32_t start, uint32_t length)
1118{
ec565b3c 1119 struct i915_page_table *pt;
678d96fb
BW		 1120 uint32_t pde, temp;
1121
1122 gen6_for_each_pde(pt, pd, start, length, temp, pde)
1123 gen6_write_pde(pd, pde, pt);
1124
1125 /* Make sure write is complete before other code can use this page
1126 * table. Also require for WC mapped PTEs */
1127 readl(dev_priv->gtt.gsm);
3e302542
BW		 1128}
1129
b4a74e3a 1130static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
3e302542 1131{
7324cc04 1132 BUG_ON(ppgtt->pd.pd_offset & 0x3f);
b4a74e3a 1133
7324cc04 1134 return (ppgtt->pd.pd_offset / 64) << 16;
b4a74e3a
BW		 1135}
1136
90252e5c 1137static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
6689c167 1138 struct intel_engine_cs *ring)
90252e5c 1139{
90252e5c
BW		 1140 int ret;
1141
90252e5c
BW		 1142 /* NB: TLBs must be flushed and invalidated before a switch */
1143 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1144 if (ret)
1145 return ret;
1146
1147 ret = intel_ring_begin(ring, 6);
1148 if (ret)
1149 return ret;
1150
1151 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1152 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1153 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1154 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1155 intel_ring_emit(ring, get_pd_offset(ppgtt));
1156 intel_ring_emit(ring, MI_NOOP);
1157 intel_ring_advance(ring);
1158
1159 return 0;
1160}
1161
71ba2d64
YZ		 1162static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
1163 struct intel_engine_cs *ring)
1164{
1165 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1166
1167 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1168 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1169 return 0;
1170}
1171
48a10389 1172static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
6689c167 1173 struct intel_engine_cs *ring)
48a10389 1174{
48a10389
BW		 1175 int ret;
1176
48a10389
BW		 1177 /* NB: TLBs must be flushed and invalidated before a switch */
1178 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1179 if (ret)
1180 return ret;
1181
1182 ret = intel_ring_begin(ring, 6);
1183 if (ret)
1184 return ret;
1185
1186 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1187 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1188 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1189 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1190 intel_ring_emit(ring, get_pd_offset(ppgtt));
1191 intel_ring_emit(ring, MI_NOOP);
1192 intel_ring_advance(ring);
1193
90252e5c
BW		 1194 /* XXX: RCS is the only one to auto invalidate the TLBs? */
1195 if (ring->id != RCS) {
1196 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1197 if (ret)
1198 return ret;
1199 }
1200
48a10389
BW		 1201 return 0;
1202}
1203
eeb9488e 1204static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
6689c167 1205 struct intel_engine_cs *ring)
eeb9488e
BW		 1206{
1207 struct drm_device *dev = ppgtt->base.dev;
1208 struct drm_i915_private *dev_priv = dev->dev_private;
1209
48a10389 1210
eeb9488e
BW		 1211 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1212 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1213
1214 POSTING_READ(RING_PP_DIR_DCLV(ring));
1215
1216 return 0;
1217}
1218
82460d97 1219static void gen8_ppgtt_enable(struct drm_device *dev)
eeb9488e 1220{
eeb9488e 1221 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1222 struct intel_engine_cs *ring;
82460d97 1223 int j;
3e302542 1224
eeb9488e
BW		 1225 for_each_ring(ring, dev_priv, j) {
1226 I915_WRITE(RING_MODE_GEN7(ring),
1227 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
eeb9488e 1228 }
eeb9488e 1229}
6197349b 1230
82460d97 1231static void gen7_ppgtt_enable(struct drm_device *dev)
3e302542 1232{
50227e1c 1233 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1234 struct intel_engine_cs *ring;
b4a74e3a 1235 uint32_t ecochk, ecobits;
3e302542 1236 int i;
6197349b 1237
b4a74e3a
BW		 1238 ecobits = I915_READ(GAC_ECO_BITS);
1239 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
a65c2fcd 1240
b4a74e3a
BW		 1241 ecochk = I915_READ(GAM_ECOCHK);
1242 if (IS_HASWELL(dev)) {
1243 ecochk |= ECOCHK_PPGTT_WB_HSW;
1244 } else {
1245 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1246 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1247 }
1248 I915_WRITE(GAM_ECOCHK, ecochk);
a65c2fcd 1249
b4a74e3a 1250 for_each_ring(ring, dev_priv, i) {
6197349b 1251 /* GFX_MODE is per-ring on gen7+ */
b4a74e3a
BW		 1252 I915_WRITE(RING_MODE_GEN7(ring),
1253 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
6197349b 1254 }
b4a74e3a 1255}
6197349b 1256
82460d97 1257static void gen6_ppgtt_enable(struct drm_device *dev)
b4a74e3a 1258{
50227e1c 1259 struct drm_i915_private *dev_priv = dev->dev_private;
b4a74e3a 1260 uint32_t ecochk, gab_ctl, ecobits;
a65c2fcd 1261
b4a74e3a
BW		 1262 ecobits = I915_READ(GAC_ECO_BITS);
1263 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1264 ECOBITS_PPGTT_CACHE64B);
6197349b 1265
b4a74e3a
BW		 1266 gab_ctl = I915_READ(GAB_CTL);
1267 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1268
1269 ecochk = I915_READ(GAM_ECOCHK);
1270 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1271
1272 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
6197349b
BW		 1273}
1274
1d2a314c 1275/* PPGTT support for Sandybdrige/Gen6 and later */
853ba5d2 1276static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 1277 uint64_t start,
1278 uint64_t length,
828c7908 1279 bool use_scratch)
1d2a314c 1280{
853ba5d2
BW		 1281 struct i915_hw_ppgtt *ppgtt =
1282 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 1283 gen6_pte_t *pt_vaddr, scratch_pte;
782f1495
BW		 1284 unsigned first_entry = start >> PAGE_SHIFT;
1285 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 1286 unsigned act_pt = first_entry / GEN6_PTES;
1287 unsigned first_pte = first_entry % GEN6_PTES;
7bddb01f 1288 unsigned last_pte, i;
1d2a314c 1289
24f3a8cf 1290 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
1d2a314c 1291
7bddb01f
DV		 1292 while (num_entries) {
1293 last_pte = first_pte + num_entries;
07749ef3
MT		 1294 if (last_pte > GEN6_PTES)
1295 last_pte = GEN6_PTES;
7bddb01f 1296
06fda602 1297 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
1d2a314c 1298
7bddb01f
DV		 1299 for (i = first_pte; i < last_pte; i++)
1300 pt_vaddr[i] = scratch_pte;
1d2a314c
DV		 1301
1302 kunmap_atomic(pt_vaddr);
1d2a314c 1303
7bddb01f
DV		 1304 num_entries -= last_pte - first_pte;
1305 first_pte = 0;
a15326a5 1306 act_pt++;
7bddb01f 1307 }
1d2a314c
DV		 1308}
1309
853ba5d2 1310static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
def886c3 1311 struct sg_table *pages,
782f1495 1312 uint64_t start,
24f3a8cf 1313 enum i915_cache_level cache_level, u32 flags)
def886c3 1314{
853ba5d2
BW		 1315 struct i915_hw_ppgtt *ppgtt =
1316 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 1317 gen6_pte_t *pt_vaddr;
782f1495 1318 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 1319 unsigned act_pt = first_entry / GEN6_PTES;
1320 unsigned act_pte = first_entry % GEN6_PTES;
6e995e23
ID		 1321 struct sg_page_iter sg_iter;
1322
cc79714f 1323 pt_vaddr = NULL;
6e995e23 1324 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
cc79714f 1325 if (pt_vaddr == NULL)
06fda602 1326 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
6e995e23 1327
cc79714f
CW		 1328 pt_vaddr[act_pte] =
1329 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
24f3a8cf
AG		 1330 cache_level, true, flags);
1331
07749ef3 1332 if (++act_pte == GEN6_PTES) {
6e995e23 1333 kunmap_atomic(pt_vaddr);
cc79714f 1334 pt_vaddr = NULL;
a15326a5 1335 act_pt++;
6e995e23 1336 act_pte = 0;
def886c3 1337 }
def886c3 1338 }
cc79714f
CW		 1339 if (pt_vaddr)
1340 kunmap_atomic(pt_vaddr);
def886c3
DV		 1341}
1342
563222a7
BW		 1343/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we
1344 * are switching between contexts with the same LRCA, we also must do a force
1345 * restore.
1346 */
1347static inline void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
1348{
1349 /* If current vm != vm, */
1350 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
1351}
1352
4933d519 1353static void gen6_initialize_pt(struct i915_address_space *vm,
ec565b3c 1354 struct i915_page_table *pt)
4933d519
MT		 1355{
1356 gen6_pte_t *pt_vaddr, scratch_pte;
1357 int i;
1358
1359 WARN_ON(vm->scratch.addr == 0);
1360
1361 scratch_pte = vm->pte_encode(vm->scratch.addr,
1362 I915_CACHE_LLC, true, 0);
1363
1364 pt_vaddr = kmap_atomic(pt->page);
1365
1366 for (i = 0; i < GEN6_PTES; i++)
1367 pt_vaddr[i] = scratch_pte;
1368
1369 kunmap_atomic(pt_vaddr);
1370}
1371
678d96fb
BW		 1372static int gen6_alloc_va_range(struct i915_address_space *vm,
1373 uint64_t start, uint64_t length)
1374{
4933d519
MT		 1375 DECLARE_BITMAP(new_page_tables, I915_PDES);
1376 struct drm_device *dev = vm->dev;
1377 struct drm_i915_private *dev_priv = dev->dev_private;
678d96fb
BW		 1378 struct i915_hw_ppgtt *ppgtt =
1379 container_of(vm, struct i915_hw_ppgtt, base);
ec565b3c 1380 struct i915_page_table *pt;
4933d519 1381 const uint32_t start_save = start, length_save = length;
678d96fb 1382 uint32_t pde, temp;
4933d519
MT		 1383 int ret;
1384
1385 WARN_ON(upper_32_bits(start));
1386
1387 bitmap_zero(new_page_tables, I915_PDES);
1388
1389 /* The allocation is done in two stages so that we can bail out with
1390 * minimal amount of pain. The first stage finds new page tables that
1391 * need allocation. The second stage marks use ptes within the page
1392 * tables.
1393 */
1394 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1395 if (pt != ppgtt->scratch_pt) {
1396 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1397 continue;
1398 }
1399
1400 /* We've already allocated a page table */
1401 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1402
1403 pt = alloc_pt_single(dev);
1404 if (IS_ERR(pt)) {
1405 ret = PTR_ERR(pt);
1406 goto unwind_out;
1407 }
1408
1409 gen6_initialize_pt(vm, pt);
1410
1411 ppgtt->pd.page_table[pde] = pt;
1412 set_bit(pde, new_page_tables);
72744cb1 1413 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
4933d519
MT		 1414 }
1415
1416 start = start_save;
1417 length = length_save;
678d96fb
BW		 1418
1419 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1420 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1421
1422 bitmap_zero(tmp_bitmap, GEN6_PTES);
1423 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1424 gen6_pte_count(start, length));
1425
4933d519
MT		 1426 if (test_and_clear_bit(pde, new_page_tables))
1427 gen6_write_pde(&ppgtt->pd, pde, pt);
1428
72744cb1
MT		 1429 trace_i915_page_table_entry_map(vm, pde, pt,
1430 gen6_pte_index(start),
1431 gen6_pte_count(start, length),
1432 GEN6_PTES);
4933d519 1433 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
678d96fb
BW		 1434 GEN6_PTES);
1435 }
1436
4933d519
MT		 1437 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1438
1439 /* Make sure write is complete before other code can use this page
1440 * table. Also require for WC mapped PTEs */
1441 readl(dev_priv->gtt.gsm);
1442
563222a7 1443 mark_tlbs_dirty(ppgtt);
678d96fb 1444 return 0;
4933d519
MT		 1445
1446unwind_out:
1447 for_each_set_bit(pde, new_page_tables, I915_PDES) {
ec565b3c 1448 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
4933d519
MT		 1449
1450 ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;
1451 unmap_and_free_pt(pt, vm->dev);
1452 }
1453
1454 mark_tlbs_dirty(ppgtt);
1455 return ret;
678d96fb
BW		 1456}
1457
a00d825d
BW		 1458static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
1459{
09942c65
MT		 1460 struct i915_page_table *pt;
1461 uint32_t pde;
4933d519 1462
09942c65 1463 gen6_for_all_pdes(pt, ppgtt, pde) {
4933d519 1464 if (pt != ppgtt->scratch_pt)
09942c65 1465 unmap_and_free_pt(pt, ppgtt->base.dev);
4933d519 1466 }
06fda602 1467
4933d519 1468 unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
e5815a2e 1469 unmap_and_free_pd(&ppgtt->pd, ppgtt->base.dev);
3440d265
DV		 1470}
1471
a00d825d
BW		 1472static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1473{
1474 struct i915_hw_ppgtt *ppgtt =
1475 container_of(vm, struct i915_hw_ppgtt, base);
1476
a00d825d
BW		 1477 drm_mm_remove_node(&ppgtt->node);
1478
a00d825d
BW		 1479 gen6_ppgtt_free(ppgtt);
1480}
1481
b146520f 1482static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
3440d265 1483{
853ba5d2 1484 struct drm_device *dev = ppgtt->base.dev;
1d2a314c 1485 struct drm_i915_private *dev_priv = dev->dev_private;
e3cc1995 1486 bool retried = false;
b146520f 1487 int ret;
1d2a314c 1488
c8d4c0d6
BW		 1489 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1490 * allocator works in address space sizes, so it's multiplied by page
1491 * size. We allocate at the top of the GTT to avoid fragmentation.
1492 */
1493 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
4933d519
MT		 1494 ppgtt->scratch_pt = alloc_pt_single(ppgtt->base.dev);
1495 if (IS_ERR(ppgtt->scratch_pt))
1496 return PTR_ERR(ppgtt->scratch_pt);
1497
1498 gen6_initialize_pt(&ppgtt->base, ppgtt->scratch_pt);
1499
e3cc1995 1500alloc:
c8d4c0d6
BW		 1501 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1502 &ppgtt->node, GEN6_PD_SIZE,
1503 GEN6_PD_ALIGN, 0,
1504 0, dev_priv->gtt.base.total,
3e8b5ae9 1505 DRM_MM_TOPDOWN);
e3cc1995
BW		 1506 if (ret == -ENOSPC && !retried) {
1507 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1508 GEN6_PD_SIZE, GEN6_PD_ALIGN,
d23db88c
CW		 1509 I915_CACHE_NONE,
1510 0, dev_priv->gtt.base.total,
1511 0);
e3cc1995 1512 if (ret)
678d96fb 1513 goto err_out;
e3cc1995
BW		 1514
1515 retried = true;
1516 goto alloc;
1517 }
c8d4c0d6 1518
c8c26622 1519 if (ret)
678d96fb
BW		 1520 goto err_out;
1521
c8c26622 1522
c8d4c0d6
BW		 1523 if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1524 DRM_DEBUG("Forced to use aperture for PDEs\n");
1d2a314c 1525
c8c26622 1526 return 0;
678d96fb
BW		 1527
1528err_out:
4933d519 1529 unmap_and_free_pt(ppgtt->scratch_pt, ppgtt->base.dev);
678d96fb 1530 return ret;
b146520f
BW		 1531}
1532
b146520f
BW		 1533static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1534{
2f2cf682 1535 return gen6_ppgtt_allocate_page_directories(ppgtt);
4933d519 1536}
06dc68d6 1537
4933d519
MT		 1538static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
1539 uint64_t start, uint64_t length)
1540{
ec565b3c 1541 struct i915_page_table *unused;
4933d519 1542 uint32_t pde, temp;
1d2a314c 1543
4933d519
MT		 1544 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
1545 ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;
b146520f
BW		 1546}
1547
4933d519 1548static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)
b146520f
BW		 1549{
1550 struct drm_device *dev = ppgtt->base.dev;
1551 struct drm_i915_private *dev_priv = dev->dev_private;
1552 int ret;
1553
1554 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1555 if (IS_GEN6(dev)) {
b146520f
BW		 1556 ppgtt->switch_mm = gen6_mm_switch;
1557 } else if (IS_HASWELL(dev)) {
b146520f
BW		 1558 ppgtt->switch_mm = hsw_mm_switch;
1559 } else if (IS_GEN7(dev)) {
b146520f
BW		 1560 ppgtt->switch_mm = gen7_mm_switch;
1561 } else
1562 BUG();
1563
71ba2d64
YZ		 1564 if (intel_vgpu_active(dev))
1565 ppgtt->switch_mm = vgpu_mm_switch;
1566
b146520f
BW		 1567 ret = gen6_ppgtt_alloc(ppgtt);
1568 if (ret)
1569 return ret;
1570
4933d519
MT		 1571 if (aliasing) {
1572 /* preallocate all pts */
09942c65 1573 ret = alloc_pt_range(&ppgtt->pd, 0, I915_PDES,
4933d519
MT		 1574 ppgtt->base.dev);
1575
1576 if (ret) {
1577 gen6_ppgtt_cleanup(&ppgtt->base);
1578 return ret;
1579 }
1580 }
1581
d7b2633d 1582 ppgtt->base.allocate_va_range = aliasing ? NULL : gen6_alloc_va_range;
b146520f
BW		 1583 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1584 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1585 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
b146520f 1586 ppgtt->base.start = 0;
09942c65 1587 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
87d60b63 1588 ppgtt->debug_dump = gen6_dump_ppgtt;
1d2a314c 1589
7324cc04 1590 ppgtt->pd.pd_offset =
07749ef3 1591 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
1d2a314c 1592
678d96fb
BW		 1593 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
1594 ppgtt->pd.pd_offset / sizeof(gen6_pte_t);
1595
4933d519
MT		 1596 if (aliasing)
1597 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1598 else
1599 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
1d2a314c 1600
678d96fb
BW		 1601 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
1602
440fd528 1603 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
b146520f
BW		 1604 ppgtt->node.size >> 20,
1605 ppgtt->node.start / PAGE_SIZE);
3440d265 1606
fa76da34 1607 DRM_DEBUG("Adding PPGTT at offset %x\n",
7324cc04 1608 ppgtt->pd.pd_offset << 10);
fa76da34 1609
b146520f 1610 return 0;
3440d265
DV		 1611}
1612
4933d519
MT		 1613static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,
1614 bool aliasing)
3440d265
DV		 1615{
1616 struct drm_i915_private *dev_priv = dev->dev_private;
3440d265 1617
853ba5d2 1618 ppgtt->base.dev = dev;
8407bb91 1619 ppgtt->base.scratch = dev_priv->gtt.base.scratch;
3440d265 1620
3ed124b2 1621 if (INTEL_INFO(dev)->gen < 8)
4933d519 1622 return gen6_ppgtt_init(ppgtt, aliasing);
d7b2633d
MT		 1623 else if (aliasing)
1624 return gen8_aliasing_ppgtt_init(ppgtt);
3ed124b2 1625 else
d7b2633d 1626 return gen8_ppgtt_init(ppgtt);
fa76da34
DV		 1627}
1628int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1629{
1630 struct drm_i915_private *dev_priv = dev->dev_private;
1631 int ret = 0;
3ed124b2 1632
4933d519 1633 ret = __hw_ppgtt_init(dev, ppgtt, false);
fa76da34 1634 if (ret == 0) {
c7c48dfd 1635 kref_init(&ppgtt->ref);
93bd8649
BW		 1636 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1637 ppgtt->base.total);
7e0d96bc 1638 i915_init_vm(dev_priv, &ppgtt->base);
93bd8649 1639 }
1d2a314c
DV		 1640
1641 return ret;
1642}
1643
82460d97
DV		 1644int i915_ppgtt_init_hw(struct drm_device *dev)
1645{
1646 struct drm_i915_private *dev_priv = dev->dev_private;
1647 struct intel_engine_cs *ring;
1648 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1649 int i, ret = 0;
1650
671b5013
TD		 1651 /* In the case of execlists, PPGTT is enabled by the context descriptor
1652 * and the PDPs are contained within the context itself. We don't
1653 * need to do anything here. */
1654 if (i915.enable_execlists)
1655 return 0;
1656
82460d97
DV		 1657 if (!USES_PPGTT(dev))
1658 return 0;
1659
1660 if (IS_GEN6(dev))
1661 gen6_ppgtt_enable(dev);
1662 else if (IS_GEN7(dev))
1663 gen7_ppgtt_enable(dev);
1664 else if (INTEL_INFO(dev)->gen >= 8)
1665 gen8_ppgtt_enable(dev);
1666 else
5f77eeb0 1667 MISSING_CASE(INTEL_INFO(dev)->gen);
82460d97
DV		 1668
1669 if (ppgtt) {
1670 for_each_ring(ring, dev_priv, i) {
6689c167 1671 ret = ppgtt->switch_mm(ppgtt, ring);
82460d97
DV		 1672 if (ret != 0)
1673 return ret;
7e0d96bc 1674 }
93bd8649 1675 }
1d2a314c
DV		 1676
1677 return ret;
1678}
4d884705
DV		 1679struct i915_hw_ppgtt *
1680i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1681{
1682 struct i915_hw_ppgtt *ppgtt;
1683 int ret;
1684
1685 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1686 if (!ppgtt)
1687 return ERR_PTR(-ENOMEM);
1688
1689 ret = i915_ppgtt_init(dev, ppgtt);
1690 if (ret) {
1691 kfree(ppgtt);
1692 return ERR_PTR(ret);
1693 }
1694
1695 ppgtt->file_priv = fpriv;
1696
198c974d
DCS		 1697 trace_i915_ppgtt_create(&ppgtt->base);
1698
4d884705
DV		 1699 return ppgtt;
1700}
1701
ee960be7
DV		 1702void i915_ppgtt_release(struct kref *kref)
1703{
1704 struct i915_hw_ppgtt *ppgtt =
1705 container_of(kref, struct i915_hw_ppgtt, ref);
1706
198c974d
DCS		 1707 trace_i915_ppgtt_release(&ppgtt->base);
1708
ee960be7
DV		 1709 /* vmas should already be unbound */
1710 WARN_ON(!list_empty(&ppgtt->base.active_list));
1711 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1712
19dd120c
DV		 1713 list_del(&ppgtt->base.global_link);
1714 drm_mm_takedown(&ppgtt->base.mm);
1715
ee960be7
DV		 1716 ppgtt->base.cleanup(&ppgtt->base);
1717 kfree(ppgtt);
1718}
1d2a314c 1719
7e0d96bc 1720static void
6f65e29a
BW		 1721ppgtt_bind_vma(struct i915_vma *vma,
1722 enum i915_cache_level cache_level,
1723 u32 flags)
1d2a314c 1724{
24f3a8cf
AG		 1725 /* Currently applicable only to VLV */
1726 if (vma->obj->gt_ro)
1727 flags |= PTE_READ_ONLY;
1728
782f1495 1729 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
24f3a8cf 1730 cache_level, flags);
1d2a314c
DV		 1731}
1732
7e0d96bc 1733static void ppgtt_unbind_vma(struct i915_vma *vma)
7bddb01f 1734{
6f65e29a 1735 vma->vm->clear_range(vma->vm,
782f1495
BW		 1736 vma->node.start,
1737 vma->obj->base.size,
6f65e29a 1738 true);
7bddb01f
DV		 1739}
1740
a81cc00c
BW		 1741extern int intel_iommu_gfx_mapped;
1742/* Certain Gen5 chipsets require require idling the GPU before
1743 * unmapping anything from the GTT when VT-d is enabled.
1744 */
1745static inline bool needs_idle_maps(struct drm_device *dev)
1746{
1747#ifdef CONFIG_INTEL_IOMMU
1748 /* Query intel_iommu to see if we need the workaround. Presumably that
1749 * was loaded first.
1750 */
1751 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1752 return true;
1753#endif
1754 return false;
1755}
1756
5c042287
BW		 1757static bool do_idling(struct drm_i915_private *dev_priv)
1758{
1759 bool ret = dev_priv->mm.interruptible;
1760
a81cc00c 1761 if (unlikely(dev_priv->gtt.do_idle_maps)) {
5c042287 1762 dev_priv->mm.interruptible = false;
b2da9fe5 1763 if (i915_gpu_idle(dev_priv->dev)) {
5c042287
BW		 1764 DRM_ERROR("Couldn't idle GPU\n");
1765 /* Wait a bit, in hopes it avoids the hang */
1766 udelay(10);
1767 }
1768 }
1769
1770 return ret;
1771}
1772
1773static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1774{
a81cc00c 1775 if (unlikely(dev_priv->gtt.do_idle_maps))
5c042287
BW		 1776 dev_priv->mm.interruptible = interruptible;
1777}
1778
828c7908
BW		 1779void i915_check_and_clear_faults(struct drm_device *dev)
1780{
1781 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1782 struct intel_engine_cs *ring;
828c7908
BW		 1783 int i;
1784
1785 if (INTEL_INFO(dev)->gen < 6)
1786 return;
1787
1788 for_each_ring(ring, dev_priv, i) {
1789 u32 fault_reg;
1790 fault_reg = I915_READ(RING_FAULT_REG(ring));
1791 if (fault_reg & RING_FAULT_VALID) {
1792 DRM_DEBUG_DRIVER("Unexpected fault\n"
59a5d290 1793 "\tAddr: 0x%08lx\n"
828c7908
BW		 1794 "\tAddress space: %s\n"
1795 "\tSource ID: %d\n"
1796 "\tType: %d\n",
1797 fault_reg & PAGE_MASK,
1798 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1799 RING_FAULT_SRCID(fault_reg),
1800 RING_FAULT_FAULT_TYPE(fault_reg));
1801 I915_WRITE(RING_FAULT_REG(ring),
1802 fault_reg & ~RING_FAULT_VALID);
1803 }
1804 }
1805 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1806}
1807
91e56499
CW		 1808static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1809{
1810 if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1811 intel_gtt_chipset_flush();
1812 } else {
1813 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1814 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1815 }
1816}
1817
828c7908
BW		 1818void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1819{
1820 struct drm_i915_private *dev_priv = dev->dev_private;
1821
1822 /* Don't bother messing with faults pre GEN6 as we have little
1823 * documentation supporting that it's a good idea.
1824 */
1825 if (INTEL_INFO(dev)->gen < 6)
1826 return;
1827
1828 i915_check_and_clear_faults(dev);
1829
1830 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
782f1495
BW		 1831 dev_priv->gtt.base.start,
1832 dev_priv->gtt.base.total,
e568af1c 1833 true);
91e56499
CW		 1834
1835 i915_ggtt_flush(dev_priv);
828c7908
BW		 1836}
1837
76aaf220
DV		 1838void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1839{
1840 struct drm_i915_private *dev_priv = dev->dev_private;
05394f39 1841 struct drm_i915_gem_object *obj;
80da2161 1842 struct i915_address_space *vm;
76aaf220 1843
828c7908
BW		 1844 i915_check_and_clear_faults(dev);
1845
bee4a186 1846 /* First fill our portion of the GTT with scratch pages */
853ba5d2 1847 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
782f1495
BW		 1848 dev_priv->gtt.base.start,
1849 dev_priv->gtt.base.total,
828c7908 1850 true);
bee4a186 1851
35c20a60 1852 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
6f65e29a
BW		 1853 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1854 &dev_priv->gtt.base);
1855 if (!vma)
1856 continue;
1857
2c22569b 1858 i915_gem_clflush_object(obj, obj->pin_display);
6f65e29a
BW		 1859 /* The bind_vma code tries to be smart about tracking mappings.
1860 * Unfortunately above, we've just wiped out the mappings
1861 * without telling our object about it. So we need to fake it.
fe14d5f4
TU		 1862 *
1863 * Bind is not expected to fail since this is only called on
1864 * resume and assumption is all requirements exist already.
6f65e29a 1865 */
aff43766 1866 vma->bound &= ~GLOBAL_BIND;
fe14d5f4 1867 WARN_ON(i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND));
76aaf220
DV		 1868 }
1869
80da2161 1870
a2319c08 1871 if (INTEL_INFO(dev)->gen >= 8) {
ee0ce478
VS		 1872 if (IS_CHERRYVIEW(dev))
1873 chv_setup_private_ppat(dev_priv);
1874 else
1875 bdw_setup_private_ppat(dev_priv);
1876
80da2161 1877 return;
a2319c08 1878 }
80da2161 1879
678d96fb
BW		 1880 if (USES_PPGTT(dev)) {
1881 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1882 /* TODO: Perhaps it shouldn't be gen6 specific */
1883
1884 struct i915_hw_ppgtt *ppgtt =
1885 container_of(vm, struct i915_hw_ppgtt,
1886 base);
80da2161 1887
678d96fb
BW		 1888 if (i915_is_ggtt(vm))
1889 ppgtt = dev_priv->mm.aliasing_ppgtt;
1890
1891 gen6_write_page_range(dev_priv, &ppgtt->pd,
1892 0, ppgtt->base.total);
1893 }
76aaf220
DV		 1894 }
1895
91e56499 1896 i915_ggtt_flush(dev_priv);
76aaf220 1897}
7c2e6fdf 1898
74163907 1899int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
7c2e6fdf 1900{
9da3da66 1901 if (obj->has_dma_mapping)
74163907 1902 return 0;
9da3da66
CW		 1903
1904 if (!dma_map_sg(&obj->base.dev->pdev->dev,
1905 obj->pages->sgl, obj->pages->nents,
1906 PCI_DMA_BIDIRECTIONAL))
1907 return -ENOSPC;
1908
1909 return 0;
7c2e6fdf
DV		 1910}
1911
07749ef3 1912static inline void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
94ec8f61
BW		 1913{
1914#ifdef writeq
1915 writeq(pte, addr);
1916#else
1917 iowrite32((u32)pte, addr);
1918 iowrite32(pte >> 32, addr + 4);
1919#endif
1920}
1921
1922static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1923 struct sg_table *st,
782f1495 1924 uint64_t start,
24f3a8cf 1925 enum i915_cache_level level, u32 unused)
94ec8f61
BW		 1926{
1927 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495 1928 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 1929 gen8_pte_t __iomem *gtt_entries =
1930 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
94ec8f61
BW		 1931 int i = 0;
1932 struct sg_page_iter sg_iter;
57007df7 1933 dma_addr_t addr = 0; /* shut up gcc */
94ec8f61
BW		 1934
1935 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1936 addr = sg_dma_address(sg_iter.sg) +
1937 (sg_iter.sg_pgoffset << PAGE_SHIFT);
1938 gen8_set_pte(&gtt_entries[i],
1939 gen8_pte_encode(addr, level, true));
1940 i++;
1941 }
1942
1943 /*
1944 * XXX: This serves as a posting read to make sure that the PTE has
1945 * actually been updated. There is some concern that even though
1946 * registers and PTEs are within the same BAR that they are potentially
1947 * of NUMA access patterns. Therefore, even with the way we assume
1948 * hardware should work, we must keep this posting read for paranoia.
1949 */
1950 if (i != 0)
1951 WARN_ON(readq(&gtt_entries[i-1])
1952 != gen8_pte_encode(addr, level, true));
1953
94ec8f61
BW		 1954 /* This next bit makes the above posting read even more important. We
1955 * want to flush the TLBs only after we're certain all the PTE updates
1956 * have finished.
1957 */
1958 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1959 POSTING_READ(GFX_FLSH_CNTL_GEN6);
94ec8f61
BW		 1960}
1961
e76e9aeb
BW		 1962/*
1963 * Binds an object into the global gtt with the specified cache level. The object
1964 * will be accessible to the GPU via commands whose operands reference offsets
1965 * within the global GTT as well as accessible by the GPU through the GMADR
1966 * mapped BAR (dev_priv->mm.gtt->gtt).
1967 */
853ba5d2 1968static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
7faf1ab2 1969 struct sg_table *st,
782f1495 1970 uint64_t start,
24f3a8cf 1971 enum i915_cache_level level, u32 flags)
e76e9aeb 1972{
853ba5d2 1973 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495 1974 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 1975 gen6_pte_t __iomem *gtt_entries =
1976 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
6e995e23
ID		 1977 int i = 0;
1978 struct sg_page_iter sg_iter;
57007df7 1979 dma_addr_t addr = 0;
e76e9aeb 1980
6e995e23 1981 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2db76d7c 1982 addr = sg_page_iter_dma_address(&sg_iter);
24f3a8cf 1983 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
6e995e23 1984 i++;
e76e9aeb
BW		 1985 }
1986
e76e9aeb
BW		 1987 /* XXX: This serves as a posting read to make sure that the PTE has
1988 * actually been updated. There is some concern that even though
1989 * registers and PTEs are within the same BAR that they are potentially
1990 * of NUMA access patterns. Therefore, even with the way we assume
1991 * hardware should work, we must keep this posting read for paranoia.
1992 */
57007df7
PM		 1993 if (i != 0) {
1994 unsigned long gtt = readl(&gtt_entries[i-1]);
1995 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1996 }
0f9b91c7
BW		 1997
1998 /* This next bit makes the above posting read even more important. We
1999 * want to flush the TLBs only after we're certain all the PTE updates
2000 * have finished.
2001 */
2002 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2003 POSTING_READ(GFX_FLSH_CNTL_GEN6);
e76e9aeb
BW		 2004}
2005
94ec8f61 2006static void gen8_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2007 uint64_t start,
2008 uint64_t length,
94ec8f61
BW		 2009 bool use_scratch)
2010{
2011 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495
BW		 2012 unsigned first_entry = start >> PAGE_SHIFT;
2013 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 2014 gen8_pte_t scratch_pte, __iomem *gtt_base =
2015 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
94ec8f61
BW		 2016 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
2017 int i;
2018
2019 if (WARN(num_entries > max_entries,
2020 "First entry = %d; Num entries = %d (max=%d)\n",
2021 first_entry, num_entries, max_entries))
2022 num_entries = max_entries;
2023
2024 scratch_pte = gen8_pte_encode(vm->scratch.addr,
2025 I915_CACHE_LLC,
2026 use_scratch);
2027 for (i = 0; i < num_entries; i++)
2028 gen8_set_pte(&gtt_base[i], scratch_pte);
2029 readl(gtt_base);
2030}
2031
853ba5d2 2032static void gen6_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2033 uint64_t start,
2034 uint64_t length,
828c7908 2035 bool use_scratch)
7faf1ab2 2036{
853ba5d2 2037 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495
BW		 2038 unsigned first_entry = start >> PAGE_SHIFT;
2039 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 2040 gen6_pte_t scratch_pte, __iomem *gtt_base =
2041 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
a54c0c27 2042 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
7faf1ab2
DV		 2043 int i;
2044
2045 if (WARN(num_entries > max_entries,
2046 "First entry = %d; Num entries = %d (max=%d)\n",
2047 first_entry, num_entries, max_entries))
2048 num_entries = max_entries;
2049
24f3a8cf 2050 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
828c7908 2051
7faf1ab2
DV		 2052 for (i = 0; i < num_entries; i++)
2053 iowrite32(scratch_pte, &gtt_base[i]);
2054 readl(gtt_base);
2055}
2056
6f65e29a
BW		 2057
2058static void i915_ggtt_bind_vma(struct i915_vma *vma,
2059 enum i915_cache_level cache_level,
2060 u32 unused)
7faf1ab2 2061{
6f65e29a 2062 const unsigned long entry = vma->node.start >> PAGE_SHIFT;
7faf1ab2
DV		 2063 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2064 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2065
6f65e29a 2066 BUG_ON(!i915_is_ggtt(vma->vm));
fe14d5f4 2067 intel_gtt_insert_sg_entries(vma->ggtt_view.pages, entry, flags);
aff43766 2068 vma->bound = GLOBAL_BIND;
7faf1ab2
DV		 2069}
2070
853ba5d2 2071static void i915_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2072 uint64_t start,
2073 uint64_t length,
828c7908 2074 bool unused)
7faf1ab2 2075{
782f1495
BW		 2076 unsigned first_entry = start >> PAGE_SHIFT;
2077 unsigned num_entries = length >> PAGE_SHIFT;
7faf1ab2
DV		 2078 intel_gtt_clear_range(first_entry, num_entries);
2079}
2080
6f65e29a
BW		 2081static void i915_ggtt_unbind_vma(struct i915_vma *vma)
2082{
2083 const unsigned int first = vma->node.start >> PAGE_SHIFT;
2084 const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
7faf1ab2 2085
6f65e29a 2086 BUG_ON(!i915_is_ggtt(vma->vm));
aff43766 2087 vma->bound = 0;
6f65e29a
BW		 2088 intel_gtt_clear_range(first, size);
2089}
7faf1ab2 2090
6f65e29a
BW		 2091static void ggtt_bind_vma(struct i915_vma *vma,
2092 enum i915_cache_level cache_level,
2093 u32 flags)
d5bd1449 2094{
6f65e29a 2095 struct drm_device *dev = vma->vm->dev;
7faf1ab2 2096 struct drm_i915_private *dev_priv = dev->dev_private;
6f65e29a 2097 struct drm_i915_gem_object *obj = vma->obj;
ec7adb6e 2098 struct sg_table *pages = obj->pages;
7faf1ab2 2099
24f3a8cf
AG		 2100 /* Currently applicable only to VLV */
2101 if (obj->gt_ro)
2102 flags |= PTE_READ_ONLY;
2103
ec7adb6e
JL		 2104 if (i915_is_ggtt(vma->vm))
2105 pages = vma->ggtt_view.pages;
2106
6f65e29a
BW		 2107 /* If there is no aliasing PPGTT, or the caller needs a global mapping,
2108 * or we have a global mapping already but the cacheability flags have
2109 * changed, set the global PTEs.
2110 *
2111 * If there is an aliasing PPGTT it is anecdotally faster, so use that
2112 * instead if none of the above hold true.
2113 *
2114 * NB: A global mapping should only be needed for special regions like
2115 * "gtt mappable", SNB errata, or if specified via special execbuf
2116 * flags. At all other times, the GPU will use the aliasing PPGTT.
2117 */
2118 if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
aff43766 2119 if (!(vma->bound & GLOBAL_BIND) ||
6f65e29a 2120 (cache_level != obj->cache_level)) {
ec7adb6e 2121 vma->vm->insert_entries(vma->vm, pages,
782f1495 2122 vma->node.start,
24f3a8cf 2123 cache_level, flags);
aff43766 2124 vma->bound |= GLOBAL_BIND;
6f65e29a
BW		 2125 }
2126 }
d5bd1449 2127
6f65e29a 2128 if (dev_priv->mm.aliasing_ppgtt &&
aff43766 2129 (!(vma->bound & LOCAL_BIND) ||
6f65e29a
BW		 2130 (cache_level != obj->cache_level))) {
2131 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
ec7adb6e 2132 appgtt->base.insert_entries(&appgtt->base, pages,
782f1495 2133 vma->node.start,
24f3a8cf 2134 cache_level, flags);
aff43766 2135 vma->bound |= LOCAL_BIND;
6f65e29a 2136 }
d5bd1449
CW		 2137}
2138
6f65e29a 2139static void ggtt_unbind_vma(struct i915_vma *vma)
74163907 2140{
6f65e29a 2141 struct drm_device *dev = vma->vm->dev;
7faf1ab2 2142 struct drm_i915_private *dev_priv = dev->dev_private;
6f65e29a 2143 struct drm_i915_gem_object *obj = vma->obj;
6f65e29a 2144
aff43766 2145 if (vma->bound & GLOBAL_BIND) {
782f1495
BW		 2146 vma->vm->clear_range(vma->vm,
2147 vma->node.start,
2148 obj->base.size,
6f65e29a 2149 true);
aff43766 2150 vma->bound &= ~GLOBAL_BIND;
6f65e29a 2151 }
74898d7e 2152
aff43766 2153 if (vma->bound & LOCAL_BIND) {
6f65e29a
BW		 2154 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2155 appgtt->base.clear_range(&appgtt->base,
782f1495
BW		 2156 vma->node.start,
2157 obj->base.size,
6f65e29a 2158 true);
aff43766 2159 vma->bound &= ~LOCAL_BIND;
6f65e29a 2160 }
74163907
DV		 2161}
2162
2163void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
7c2e6fdf 2164{
5c042287
BW		 2165 struct drm_device *dev = obj->base.dev;
2166 struct drm_i915_private *dev_priv = dev->dev_private;
2167 bool interruptible;
2168
2169 interruptible = do_idling(dev_priv);
2170
9da3da66
CW		 2171 if (!obj->has_dma_mapping)
2172 dma_unmap_sg(&dev->pdev->dev,
2173 obj->pages->sgl, obj->pages->nents,
2174 PCI_DMA_BIDIRECTIONAL);
5c042287
BW		 2175
2176 undo_idling(dev_priv, interruptible);
7c2e6fdf 2177}
644ec02b 2178
42d6ab48
CW		 2179static void i915_gtt_color_adjust(struct drm_mm_node *node,
2180 unsigned long color,
440fd528
TR		 2181 u64 *start,
2182 u64 *end)
42d6ab48
CW		 2183{
2184 if (node->color != color)
2185 *start += 4096;
2186
2187 if (!list_empty(&node->node_list)) {
2188 node = list_entry(node->node_list.next,
2189 struct drm_mm_node,
2190 node_list);
2191 if (node->allocated && node->color != color)
2192 *end -= 4096;
2193 }
2194}
fbe5d36e 2195
f548c0e9
DV		 2196static int i915_gem_setup_global_gtt(struct drm_device *dev,
2197 unsigned long start,
2198 unsigned long mappable_end,
2199 unsigned long end)
644ec02b 2200{
e78891ca
BW		 2201 /* Let GEM Manage all of the aperture.
2202 *
2203 * However, leave one page at the end still bound to the scratch page.
2204 * There are a number of places where the hardware apparently prefetches
2205 * past the end of the object, and we've seen multiple hangs with the
2206 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2207 * aperture. One page should be enough to keep any prefetching inside
2208 * of the aperture.
2209 */
40d74980
BW		 2210 struct drm_i915_private *dev_priv = dev->dev_private;
2211 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
ed2f3452
CW		 2212 struct drm_mm_node *entry;
2213 struct drm_i915_gem_object *obj;
2214 unsigned long hole_start, hole_end;
fa76da34 2215 int ret;
644ec02b 2216
35451cb6
BW		 2217 BUG_ON(mappable_end > end);
2218
ed2f3452 2219 /* Subtract the guard page ... */
40d74980 2220 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
5dda8fa3
YZ		 2221
2222 dev_priv->gtt.base.start = start;
2223 dev_priv->gtt.base.total = end - start;
2224
2225 if (intel_vgpu_active(dev)) {
2226 ret = intel_vgt_balloon(dev);
2227 if (ret)
2228 return ret;
2229 }
2230
42d6ab48 2231 if (!HAS_LLC(dev))
93bd8649 2232 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
644ec02b 2233
ed2f3452 2234 /* Mark any preallocated objects as occupied */
35c20a60 2235 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
40d74980 2236 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
fa76da34 2237
edd41a87 2238 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
c6cfb325
BW		 2239 i915_gem_obj_ggtt_offset(obj), obj->base.size);
2240
2241 WARN_ON(i915_gem_obj_ggtt_bound(obj));
40d74980 2242 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
6c5566a8
DV		 2243 if (ret) {
2244 DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
2245 return ret;
2246 }
aff43766 2247 vma->bound |= GLOBAL_BIND;
ed2f3452
CW		 2248 }
2249
ed2f3452 2250 /* Clear any non-preallocated blocks */
40d74980 2251 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
ed2f3452
CW		 2252 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2253 hole_start, hole_end);
782f1495
BW		 2254 ggtt_vm->clear_range(ggtt_vm, hole_start,
2255 hole_end - hole_start, true);
ed2f3452
CW		 2256 }
2257
2258 /* And finally clear the reserved guard page */
782f1495 2259 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
6c5566a8 2260
fa76da34
DV		 2261 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
2262 struct i915_hw_ppgtt *ppgtt;
2263
2264 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2265 if (!ppgtt)
2266 return -ENOMEM;
2267
4933d519
MT		 2268 ret = __hw_ppgtt_init(dev, ppgtt, true);
2269 if (ret) {
2270 kfree(ppgtt);
fa76da34 2271 return ret;
4933d519 2272 }
fa76da34
DV		 2273
2274 dev_priv->mm.aliasing_ppgtt = ppgtt;
2275 }
2276
6c5566a8 2277 return 0;
e76e9aeb
BW		 2278}
2279
d7e5008f
BW		 2280void i915_gem_init_global_gtt(struct drm_device *dev)
2281{
2282 struct drm_i915_private *dev_priv = dev->dev_private;
2283 unsigned long gtt_size, mappable_size;
d7e5008f 2284
853ba5d2 2285 gtt_size = dev_priv->gtt.base.total;
93d18799 2286 mappable_size = dev_priv->gtt.mappable_end;
d7e5008f 2287
e78891ca 2288 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
e76e9aeb
BW		 2289}
2290
90d0a0e8
DV		 2291void i915_global_gtt_cleanup(struct drm_device *dev)
2292{
2293 struct drm_i915_private *dev_priv = dev->dev_private;
2294 struct i915_address_space *vm = &dev_priv->gtt.base;
2295
70e32544
DV		 2296 if (dev_priv->mm.aliasing_ppgtt) {
2297 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2298
2299 ppgtt->base.cleanup(&ppgtt->base);
2300 }
2301
90d0a0e8 2302 if (drm_mm_initialized(&vm->mm)) {
5dda8fa3
YZ		 2303 if (intel_vgpu_active(dev))
2304 intel_vgt_deballoon();
2305
90d0a0e8
DV		 2306 drm_mm_takedown(&vm->mm);
2307 list_del(&vm->global_link);
2308 }
2309
2310 vm->cleanup(vm);
2311}
70e32544 2312
e76e9aeb
BW		 2313static int setup_scratch_page(struct drm_device *dev)
2314{
2315 struct drm_i915_private *dev_priv = dev->dev_private;
2316 struct page *page;
2317 dma_addr_t dma_addr;
2318
2319 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
2320 if (page == NULL)
2321 return -ENOMEM;
e76e9aeb
BW		 2322 set_pages_uc(page, 1);
2323
2324#ifdef CONFIG_INTEL_IOMMU
2325 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
2326 PCI_DMA_BIDIRECTIONAL);
2327 if (pci_dma_mapping_error(dev->pdev, dma_addr))
2328 return -EINVAL;
2329#else
2330 dma_addr = page_to_phys(page);
2331#endif
853ba5d2
BW		 2332 dev_priv->gtt.base.scratch.page = page;
2333 dev_priv->gtt.base.scratch.addr = dma_addr;
e76e9aeb
BW		 2334
2335 return 0;
2336}
2337
2338static void teardown_scratch_page(struct drm_device *dev)
2339{
2340 struct drm_i915_private *dev_priv = dev->dev_private;
853ba5d2
BW		 2341 struct page *page = dev_priv->gtt.base.scratch.page;
2342
2343 set_pages_wb(page, 1);
2344 pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
e76e9aeb 2345 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
853ba5d2 2346 __free_page(page);
e76e9aeb
BW		 2347}
2348
2349static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2350{
2351 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2352 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2353 return snb_gmch_ctl << 20;
2354}
2355
9459d252
BW		 2356static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2357{
2358 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2359 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2360 if (bdw_gmch_ctl)
2361 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
562d55d9
BW		 2362
2363#ifdef CONFIG_X86_32
2364 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2365 if (bdw_gmch_ctl > 4)
2366 bdw_gmch_ctl = 4;
2367#endif
2368
9459d252
BW		 2369 return bdw_gmch_ctl << 20;
2370}
2371
d7f25f23
DL		 2372static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2373{
2374 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2375 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2376
2377 if (gmch_ctrl)
2378 return 1 << (20 + gmch_ctrl);
2379
2380 return 0;
2381}
2382
baa09f5f 2383static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
e76e9aeb
BW		 2384{
2385 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2386 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2387 return snb_gmch_ctl << 25; /* 32 MB units */
2388}
2389
9459d252
BW		 2390static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2391{
2392 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2393 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2394 return bdw_gmch_ctl << 25; /* 32 MB units */
2395}
2396
d7f25f23
DL		 2397static size_t chv_get_stolen_size(u16 gmch_ctrl)
2398{
2399 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2400 gmch_ctrl &= SNB_GMCH_GMS_MASK;
2401
2402 /*
2403 * 0x0 to 0x10: 32MB increments starting at 0MB
2404 * 0x11 to 0x16: 4MB increments starting at 8MB
2405 * 0x17 to 0x1d: 4MB increments start at 36MB
2406 */
2407 if (gmch_ctrl < 0x11)
2408 return gmch_ctrl << 25;
2409 else if (gmch_ctrl < 0x17)
2410 return (gmch_ctrl - 0x11 + 2) << 22;
2411 else
2412 return (gmch_ctrl - 0x17 + 9) << 22;
2413}
2414
66375014
DL		 2415static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2416{
2417 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2418 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2419
2420 if (gen9_gmch_ctl < 0xf0)
2421 return gen9_gmch_ctl << 25; /* 32 MB units */
2422 else
2423 /* 4MB increments starting at 0xf0 for 4MB */
2424 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2425}
2426
63340133
BW		 2427static int ggtt_probe_common(struct drm_device *dev,
2428 size_t gtt_size)
2429{
2430 struct drm_i915_private *dev_priv = dev->dev_private;
21c34607 2431 phys_addr_t gtt_phys_addr;
63340133
BW		 2432 int ret;
2433
2434 /* For Modern GENs the PTEs and register space are split in the BAR */
21c34607 2435 gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
63340133
BW		 2436 (pci_resource_len(dev->pdev, 0) / 2);
2437
21c34607 2438 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
63340133
BW		 2439 if (!dev_priv->gtt.gsm) {
2440 DRM_ERROR("Failed to map the gtt page table\n");
2441 return -ENOMEM;
2442 }
2443
2444 ret = setup_scratch_page(dev);
2445 if (ret) {
2446 DRM_ERROR("Scratch setup failed\n");
2447 /* iounmap will also get called at remove, but meh */
2448 iounmap(dev_priv->gtt.gsm);
2449 }
2450
2451 return ret;
2452}
2453
fbe5d36e
BW		 2454/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2455 * bits. When using advanced contexts each context stores its own PAT, but
2456 * writing this data shouldn't be harmful even in those cases. */
ee0ce478 2457static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
fbe5d36e 2458{
fbe5d36e
BW		 2459 uint64_t pat;
2460
2461 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2462 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2463 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2464 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2465 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2466 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2467 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2468 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2469
d6a8b72e
RV		 2470 if (!USES_PPGTT(dev_priv->dev))
2471 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2472 * so RTL will always use the value corresponding to
2473 * pat_sel = 000".
2474 * So let's disable cache for GGTT to avoid screen corruptions.
2475 * MOCS still can be used though.
2476 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2477 * before this patch, i.e. the same uncached + snooping access
2478 * like on gen6/7 seems to be in effect.
2479 * - So this just fixes blitter/render access. Again it looks
2480 * like it's not just uncached access, but uncached + snooping.
2481 * So we can still hold onto all our assumptions wrt cpu
2482 * clflushing on LLC machines.
2483 */
2484 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2485
fbe5d36e
BW		 2486 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2487 * write would work. */
2488 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2489 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2490}
2491
ee0ce478
VS		 2492static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2493{
2494 uint64_t pat;
2495
2496 /*
2497 * Map WB on BDW to snooped on CHV.
2498 *
2499 * Only the snoop bit has meaning for CHV, the rest is
2500 * ignored.
2501 *
cf3d262e
VS		 2502 * The hardware will never snoop for certain types of accesses:
2503 * - CPU GTT (GMADR->GGTT->no snoop->memory)
2504 * - PPGTT page tables
2505 * - some other special cycles
2506 *
2507 * As with BDW, we also need to consider the following for GT accesses:
2508 * "For GGTT, there is NO pat_sel[2:0] from the entry,
2509 * so RTL will always use the value corresponding to
2510 * pat_sel = 000".
2511 * Which means we must set the snoop bit in PAT entry 0
2512 * in order to keep the global status page working.
ee0ce478
VS		 2513 */
2514 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2515 GEN8_PPAT(1, 0) |
2516 GEN8_PPAT(2, 0) |
2517 GEN8_PPAT(3, 0) |
2518 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2519 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2520 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2521 GEN8_PPAT(7, CHV_PPAT_SNOOP);
2522
2523 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2524 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2525}
2526
63340133
BW		 2527static int gen8_gmch_probe(struct drm_device *dev,
2528 size_t *gtt_total,
2529 size_t *stolen,
2530 phys_addr_t *mappable_base,
2531 unsigned long *mappable_end)
2532{
2533 struct drm_i915_private *dev_priv = dev->dev_private;
2534 unsigned int gtt_size;
2535 u16 snb_gmch_ctl;
2536 int ret;
2537
2538 /* TODO: We're not aware of mappable constraints on gen8 yet */
2539 *mappable_base = pci_resource_start(dev->pdev, 2);
2540 *mappable_end = pci_resource_len(dev->pdev, 2);
2541
2542 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2543 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2544
2545 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2546
66375014
DL		 2547 if (INTEL_INFO(dev)->gen >= 9) {
2548 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
2549 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2550 } else if (IS_CHERRYVIEW(dev)) {
d7f25f23
DL		 2551 *stolen = chv_get_stolen_size(snb_gmch_ctl);
2552 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
2553 } else {
2554 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
2555 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2556 }
63340133 2557
07749ef3 2558 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
63340133 2559
ee0ce478
VS		 2560 if (IS_CHERRYVIEW(dev))
2561 chv_setup_private_ppat(dev_priv);
2562 else
2563 bdw_setup_private_ppat(dev_priv);
fbe5d36e 2564
63340133
BW		 2565 ret = ggtt_probe_common(dev, gtt_size);
2566
94ec8f61
BW		 2567 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
2568 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
63340133
BW		 2569
2570 return ret;
2571}
2572
baa09f5f
BW		 2573static int gen6_gmch_probe(struct drm_device *dev,
2574 size_t *gtt_total,
41907ddc
BW		 2575 size_t *stolen,
2576 phys_addr_t *mappable_base,
2577 unsigned long *mappable_end)
e76e9aeb
BW		 2578{
2579 struct drm_i915_private *dev_priv = dev->dev_private;
baa09f5f 2580 unsigned int gtt_size;
e76e9aeb 2581 u16 snb_gmch_ctl;
e76e9aeb
BW		 2582 int ret;
2583
41907ddc
BW		 2584 *mappable_base = pci_resource_start(dev->pdev, 2);
2585 *mappable_end = pci_resource_len(dev->pdev, 2);
2586
baa09f5f
BW		 2587 /* 64/512MB is the current min/max we actually know of, but this is just
2588 * a coarse sanity check.
e76e9aeb 2589 */
41907ddc 2590 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
baa09f5f
BW		 2591 DRM_ERROR("Unknown GMADR size (%lx)\n",
2592 dev_priv->gtt.mappable_end);
2593 return -ENXIO;
e76e9aeb
BW		 2594 }
2595
e76e9aeb
BW		 2596 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2597 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
e76e9aeb 2598 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
e76e9aeb 2599
c4ae25ec 2600 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
a93e4161 2601
63340133 2602 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
07749ef3 2603 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
e76e9aeb 2604
63340133 2605 ret = ggtt_probe_common(dev, gtt_size);
e76e9aeb 2606
853ba5d2
BW		 2607 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2608 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
7faf1ab2 2609
e76e9aeb
BW		 2610 return ret;
2611}
2612
853ba5d2 2613static void gen6_gmch_remove(struct i915_address_space *vm)
e76e9aeb 2614{
853ba5d2
BW		 2615
2616 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
5ed16782 2617
853ba5d2
BW		 2618 iounmap(gtt->gsm);
2619 teardown_scratch_page(vm->dev);
644ec02b 2620}
baa09f5f
BW		 2621
2622static int i915_gmch_probe(struct drm_device *dev,
2623 size_t *gtt_total,
41907ddc
BW		 2624 size_t *stolen,
2625 phys_addr_t *mappable_base,
2626 unsigned long *mappable_end)
baa09f5f
BW		 2627{
2628 struct drm_i915_private *dev_priv = dev->dev_private;
2629 int ret;
2630
baa09f5f
BW		 2631 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2632 if (!ret) {
2633 DRM_ERROR("failed to set up gmch\n");
2634 return -EIO;
2635 }
2636
41907ddc 2637 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
baa09f5f
BW		 2638
2639 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
853ba5d2 2640 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
baa09f5f 2641
c0a7f818
CW		 2642 if (unlikely(dev_priv->gtt.do_idle_maps))
2643 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2644
baa09f5f
BW		 2645 return 0;
2646}
2647
853ba5d2 2648static void i915_gmch_remove(struct i915_address_space *vm)
baa09f5f
BW		 2649{
2650 intel_gmch_remove();
2651}
2652
2653int i915_gem_gtt_init(struct drm_device *dev)
2654{
2655 struct drm_i915_private *dev_priv = dev->dev_private;
2656 struct i915_gtt *gtt = &dev_priv->gtt;
baa09f5f
BW		 2657 int ret;
2658
baa09f5f 2659 if (INTEL_INFO(dev)->gen <= 5) {
b2f21b4d 2660 gtt->gtt_probe = i915_gmch_probe;
853ba5d2 2661 gtt->base.cleanup = i915_gmch_remove;
63340133 2662 } else if (INTEL_INFO(dev)->gen < 8) {
b2f21b4d 2663 gtt->gtt_probe = gen6_gmch_probe;
853ba5d2 2664 gtt->base.cleanup = gen6_gmch_remove;
4d15c145 2665 if (IS_HASWELL(dev) && dev_priv->ellc_size)
853ba5d2 2666 gtt->base.pte_encode = iris_pte_encode;
4d15c145 2667 else if (IS_HASWELL(dev))
853ba5d2 2668 gtt->base.pte_encode = hsw_pte_encode;
b2f21b4d 2669 else if (IS_VALLEYVIEW(dev))
853ba5d2 2670 gtt->base.pte_encode = byt_pte_encode;
350ec881
CW		 2671 else if (INTEL_INFO(dev)->gen >= 7)
2672 gtt->base.pte_encode = ivb_pte_encode;
b2f21b4d 2673 else
350ec881 2674 gtt->base.pte_encode = snb_pte_encode;
63340133
BW		 2675 } else {
2676 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2677 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
baa09f5f
BW		 2678 }
2679
853ba5d2 2680 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
b2f21b4d 2681 &gtt->mappable_base, &gtt->mappable_end);
a54c0c27 2682 if (ret)
baa09f5f 2683 return ret;
baa09f5f 2684
853ba5d2
BW		 2685 gtt->base.dev = dev;
2686
baa09f5f 2687 /* GMADR is the PCI mmio aperture into the global GTT. */
853ba5d2
BW		 2688 DRM_INFO("Memory usable by graphics device = %zdM\n",
2689 gtt->base.total >> 20);
b2f21b4d
BW		 2690 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2691 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
5db6c735
DV		 2692#ifdef CONFIG_INTEL_IOMMU
2693 if (intel_iommu_gfx_mapped)
2694 DRM_INFO("VT-d active for gfx access\n");
2695#endif
cfa7c862
DV		 2696 /*
2697 * i915.enable_ppgtt is read-only, so do an early pass to validate the
2698 * user's requested state against the hardware/driver capabilities. We
2699 * do this now so that we can print out any log messages once rather
2700 * than every time we check intel_enable_ppgtt().
2701 */
2702 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2703 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
baa09f5f
BW		 2704
2705 return 0;
2706}
6f65e29a 2707
ec7adb6e
JL		 2708static struct i915_vma *
2709__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2710 struct i915_address_space *vm,
2711 const struct i915_ggtt_view *ggtt_view)
6f65e29a 2712{
dabde5c7 2713 struct i915_vma *vma;
6f65e29a 2714
ec7adb6e
JL		 2715 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
2716 return ERR_PTR(-EINVAL);
dabde5c7
DC		 2717 vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2718 if (vma == NULL)
2719 return ERR_PTR(-ENOMEM);
ec7adb6e 2720
6f65e29a
BW		 2721 INIT_LIST_HEAD(&vma->vma_link);
2722 INIT_LIST_HEAD(&vma->mm_list);
2723 INIT_LIST_HEAD(&vma->exec_list);
2724 vma->vm = vm;
2725 vma->obj = obj;
2726
b1252bcf 2727 if (INTEL_INFO(vm->dev)->gen >= 6) {
7e0d96bc 2728 if (i915_is_ggtt(vm)) {
ec7adb6e
JL		 2729 vma->ggtt_view = *ggtt_view;
2730
7e0d96bc
BW		 2731 vma->unbind_vma = ggtt_unbind_vma;
2732 vma->bind_vma = ggtt_bind_vma;
2733 } else {
2734 vma->unbind_vma = ppgtt_unbind_vma;
2735 vma->bind_vma = ppgtt_bind_vma;
2736 }
b1252bcf 2737 } else {
6f65e29a 2738 BUG_ON(!i915_is_ggtt(vm));
ec7adb6e 2739 vma->ggtt_view = *ggtt_view;
6f65e29a
BW		 2740 vma->unbind_vma = i915_ggtt_unbind_vma;
2741 vma->bind_vma = i915_ggtt_bind_vma;
6f65e29a
BW		 2742 }
2743
f7635669
TU		 2744 list_add_tail(&vma->vma_link, &obj->vma_list);
2745 if (!i915_is_ggtt(vm))
e07f0552 2746 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
6f65e29a
BW		 2747
2748 return vma;
2749}
2750
2751struct i915_vma *
ec7adb6e
JL		 2752i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
2753 struct i915_address_space *vm)
2754{
2755 struct i915_vma *vma;
2756
2757 vma = i915_gem_obj_to_vma(obj, vm);
2758 if (!vma)
2759 vma = __i915_gem_vma_create(obj, vm,
2760 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
2761
2762 return vma;
2763}
2764
2765struct i915_vma *
2766i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
fe14d5f4 2767 const struct i915_ggtt_view *view)
6f65e29a 2768{
ec7adb6e 2769 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
6f65e29a
BW		 2770 struct i915_vma *vma;
2771
ec7adb6e
JL		 2772 if (WARN_ON(!view))
2773 return ERR_PTR(-EINVAL);
2774
2775 vma = i915_gem_obj_to_ggtt_view(obj, view);
2776
2777 if (IS_ERR(vma))
2778 return vma;
2779
6f65e29a 2780 if (!vma)
ec7adb6e 2781 vma = __i915_gem_vma_create(obj, ggtt, view);
6f65e29a
BW		 2782
2783 return vma;
ec7adb6e 2784
6f65e29a 2785}
fe14d5f4 2786
50470bb0
TU		 2787static void
2788rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height,
2789 struct sg_table *st)
2790{
2791 unsigned int column, row;
2792 unsigned int src_idx;
2793 struct scatterlist *sg = st->sgl;
2794
2795 st->nents = 0;
2796
2797 for (column = 0; column < width; column++) {
2798 src_idx = width * (height - 1) + column;
2799 for (row = 0; row < height; row++) {
2800 st->nents++;
2801 /* We don't need the pages, but need to initialize
2802 * the entries so the sg list can be happily traversed.
2803 * The only thing we need are DMA addresses.
2804 */
2805 sg_set_page(sg, NULL, PAGE_SIZE, 0);
2806 sg_dma_address(sg) = in[src_idx];
2807 sg_dma_len(sg) = PAGE_SIZE;
2808 sg = sg_next(sg);
2809 src_idx -= width;
2810 }
2811 }
2812}
2813
2814static struct sg_table *
2815intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
2816 struct drm_i915_gem_object *obj)
2817{
2818 struct drm_device *dev = obj->base.dev;
2819 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
2820 unsigned long size, pages, rot_pages;
2821 struct sg_page_iter sg_iter;
2822 unsigned long i;
2823 dma_addr_t *page_addr_list;
2824 struct sg_table *st;
2825 unsigned int tile_pitch, tile_height;
2826 unsigned int width_pages, height_pages;
1d00dad5 2827 int ret = -ENOMEM;
50470bb0
TU		 2828
2829 pages = obj->base.size / PAGE_SIZE;
2830
2831 /* Calculate tiling geometry. */
2832 tile_height = intel_tile_height(dev, rot_info->pixel_format,
2833 rot_info->fb_modifier);
2834 tile_pitch = PAGE_SIZE / tile_height;
2835 width_pages = DIV_ROUND_UP(rot_info->pitch, tile_pitch);
2836 height_pages = DIV_ROUND_UP(rot_info->height, tile_height);
2837 rot_pages = width_pages * height_pages;
2838 size = rot_pages * PAGE_SIZE;
2839
2840 /* Allocate a temporary list of source pages for random access. */
2841 page_addr_list = drm_malloc_ab(pages, sizeof(dma_addr_t));
2842 if (!page_addr_list)
2843 return ERR_PTR(ret);
2844
2845 /* Allocate target SG list. */
2846 st = kmalloc(sizeof(*st), GFP_KERNEL);
2847 if (!st)
2848 goto err_st_alloc;
2849
2850 ret = sg_alloc_table(st, rot_pages, GFP_KERNEL);
2851 if (ret)
2852 goto err_sg_alloc;
2853
2854 /* Populate source page list from the object. */
2855 i = 0;
2856 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2857 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
2858 i++;
2859 }
2860
2861 /* Rotate the pages. */
2862 rotate_pages(page_addr_list, width_pages, height_pages, st);
2863
2864 DRM_DEBUG_KMS(
2865 "Created rotated page mapping for object size %lu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages).\n",
2866 size, rot_info->pitch, rot_info->height,
2867 rot_info->pixel_format, width_pages, height_pages,
2868 rot_pages);
2869
2870 drm_free_large(page_addr_list);
2871
2872 return st;
2873
2874err_sg_alloc:
2875 kfree(st);
2876err_st_alloc:
2877 drm_free_large(page_addr_list);
2878
2879 DRM_DEBUG_KMS(
2880 "Failed to create rotated mapping for object size %lu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %lu pages)\n",
2881 size, ret, rot_info->pitch, rot_info->height,
2882 rot_info->pixel_format, width_pages, height_pages,
2883 rot_pages);
2884 return ERR_PTR(ret);
2885}
ec7adb6e 2886
50470bb0
TU		 2887static inline int
2888i915_get_ggtt_vma_pages(struct i915_vma *vma)
fe14d5f4 2889{
50470bb0
TU		 2890 int ret = 0;
2891
fe14d5f4
TU		 2892 if (vma->ggtt_view.pages)
2893 return 0;
2894
2895 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
2896 vma->ggtt_view.pages = vma->obj->pages;
50470bb0
TU		 2897 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
2898 vma->ggtt_view.pages =
2899 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
fe14d5f4
TU		 2900 else
2901 WARN_ONCE(1, "GGTT view %u not implemented!\n",
2902 vma->ggtt_view.type);
2903
2904 if (!vma->ggtt_view.pages) {
ec7adb6e 2905 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
fe14d5f4 2906 vma->ggtt_view.type);
50470bb0
TU		 2907 ret = -EINVAL;
2908 } else if (IS_ERR(vma->ggtt_view.pages)) {
2909 ret = PTR_ERR(vma->ggtt_view.pages);
2910 vma->ggtt_view.pages = NULL;
2911 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
2912 vma->ggtt_view.type, ret);
fe14d5f4
TU		 2913 }
2914
50470bb0 2915 return ret;
fe14d5f4
TU		 2916}
2917
2918/**
2919 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
2920 * @vma: VMA to map
2921 * @cache_level: mapping cache level
2922 * @flags: flags like global or local mapping
2923 *
2924 * DMA addresses are taken from the scatter-gather table of this object (or of
2925 * this VMA in case of non-default GGTT views) and PTE entries set up.
2926 * Note that DMA addresses are also the only part of the SG table we care about.
2927 */
2928int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
2929 u32 flags)
2930{
ec7adb6e
JL		 2931 if (i915_is_ggtt(vma->vm)) {
2932 int ret = i915_get_ggtt_vma_pages(vma);
fe14d5f4 2933
ec7adb6e
JL		 2934 if (ret)
2935 return ret;
2936 }
fe14d5f4
TU		 2937
2938 vma->bind_vma(vma, cache_level, flags);
2939
2940 return 0;
2941}
Linux kernel - Deliverables
This page took 0.52427 seconds and 5 git commands to generate.