projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Add functions to emit register offsets to the ring
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
CommitLineData
76aaf220
DV		 1/*
2 * Copyright © 2010 Daniel Vetter
c4ac524c 3 * Copyright © 2011-2014 Intel Corporation
76aaf220
DV		 4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
0e46ce2e 26#include <linux/seq_file.h>
5bab6f60 27#include <linux/stop_machine.h>
760285e7
DH		 28#include <drm/drmP.h>
29#include <drm/i915_drm.h>
76aaf220 30#include "i915_drv.h"
5dda8fa3 31#include "i915_vgpu.h"
76aaf220
DV		 32#include "i915_trace.h"
33#include "intel_drv.h"
34
45f8f69a
TU		 35/**
36 * DOC: Global GTT views
37 *
38 * Background and previous state
39 *
40 * Historically objects could exists (be bound) in global GTT space only as
41 * singular instances with a view representing all of the object's backing pages
42 * in a linear fashion. This view will be called a normal view.
43 *
44 * To support multiple views of the same object, where the number of mapped
45 * pages is not equal to the backing store, or where the layout of the pages
46 * is not linear, concept of a GGTT view was added.
47 *
48 * One example of an alternative view is a stereo display driven by a single
49 * image. In this case we would have a framebuffer looking like this
50 * (2x2 pages):
51 *
52 * 12
53 * 34
54 *
55 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
56 * rendering. In contrast, fed to the display engine would be an alternative
57 * view which could look something like this:
58 *
59 * 1212
60 * 3434
61 *
62 * In this example both the size and layout of pages in the alternative view is
63 * different from the normal view.
64 *
65 * Implementation and usage
66 *
67 * GGTT views are implemented using VMAs and are distinguished via enum
68 * i915_ggtt_view_type and struct i915_ggtt_view.
69 *
70 * A new flavour of core GEM functions which work with GGTT bound objects were
ec7adb6e
JL		 71 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
72 * renaming in large amounts of code. They take the struct i915_ggtt_view
73 * parameter encapsulating all metadata required to implement a view.
45f8f69a
TU		 74 *
75 * As a helper for callers which are only interested in the normal view,
76 * globally const i915_ggtt_view_normal singleton instance exists. All old core
77 * GEM API functions, the ones not taking the view parameter, are operating on,
78 * or with the normal GGTT view.
79 *
80 * Code wanting to add or use a new GGTT view needs to:
81 *
82 * 1. Add a new enum with a suitable name.
83 * 2. Extend the metadata in the i915_ggtt_view structure if required.
84 * 3. Add support to i915_get_vma_pages().
85 *
86 * New views are required to build a scatter-gather table from within the
87 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
88 * exists for the lifetime of an VMA.
89 *
90 * Core API is designed to have copy semantics which means that passed in
91 * struct i915_ggtt_view does not need to be persistent (left around after
92 * calling the core API functions).
93 *
94 */
95
70b9f6f8
DV		 96static int
97i915_get_ggtt_vma_pages(struct i915_vma *vma);
98
fe14d5f4 99const struct i915_ggtt_view i915_ggtt_view_normal;
9abc4648
JL		 100const struct i915_ggtt_view i915_ggtt_view_rotated = {
101 .type = I915_GGTT_VIEW_ROTATED
102};
fe14d5f4 103
cfa7c862
DV		 104static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
105{
1893a71b
CW		 106 bool has_aliasing_ppgtt;
107 bool has_full_ppgtt;
1f9a99e0 108 bool has_full_48bit_ppgtt;
1893a71b
CW		 109
110 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
111 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
1f9a99e0 112 has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9;
1893a71b 113
71ba2d64
YZ		 114 if (intel_vgpu_active(dev))
115 has_full_ppgtt = false; /* emulation is too hard */
116
70ee45e1
DL		 117 /*
118 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
119 * execlists, the sole mechanism available to submit work.
120 */
121 if (INTEL_INFO(dev)->gen < 9 &&
122 (enable_ppgtt == 0 || !has_aliasing_ppgtt))
cfa7c862
DV		 123 return 0;
124
125 if (enable_ppgtt == 1)
126 return 1;
127
1893a71b 128 if (enable_ppgtt == 2 && has_full_ppgtt)
cfa7c862
DV		 129 return 2;
130
1f9a99e0
MT		 131 if (enable_ppgtt == 3 && has_full_48bit_ppgtt)
132 return 3;
133
93a25a9e
DV		 134#ifdef CONFIG_INTEL_IOMMU
135 /* Disable ppgtt on SNB if VT-d is on. */
136 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
137 DRM_INFO("Disabling PPGTT because VT-d is on\n");
cfa7c862 138 return 0;
93a25a9e
DV		 139 }
140#endif
141
62942ed7 142 /* Early VLV doesn't have this */
ca2aed6c
VS		 143 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
144 dev->pdev->revision < 0xb) {
62942ed7
JB		 145 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
146 return 0;
147 }
148
2f82bbdf 149 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
1f9a99e0 150 return has_full_48bit_ppgtt ? 3 : 2;
2f82bbdf
MT		 151 else
152 return has_aliasing_ppgtt ? 1 : 0;
93a25a9e
DV		 153}
154
70b9f6f8
DV		 155static int ppgtt_bind_vma(struct i915_vma *vma,
156 enum i915_cache_level cache_level,
157 u32 unused)
47552659
DV		 158{
159 u32 pte_flags = 0;
160
161 /* Currently applicable only to VLV */
162 if (vma->obj->gt_ro)
163 pte_flags |= PTE_READ_ONLY;
164
165 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
166 cache_level, pte_flags);
70b9f6f8
DV		 167
168 return 0;
47552659
DV		 169}
170
171static void ppgtt_unbind_vma(struct i915_vma *vma)
172{
173 vma->vm->clear_range(vma->vm,
174 vma->node.start,
175 vma->obj->base.size,
176 true);
177}
6f65e29a 178
2c642b07
DV		 179static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
180 enum i915_cache_level level,
181 bool valid)
94ec8f61 182{
07749ef3 183 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
94ec8f61 184 pte |= addr;
63c42e56
BW		 185
186 switch (level) {
187 case I915_CACHE_NONE:
fbe5d36e 188 pte |= PPAT_UNCACHED_INDEX;
63c42e56
BW		 189 break;
190 case I915_CACHE_WT:
191 pte |= PPAT_DISPLAY_ELLC_INDEX;
192 break;
193 default:
194 pte |= PPAT_CACHED_INDEX;
195 break;
196 }
197
94ec8f61
BW		 198 return pte;
199}
200
fe36f55d
MK		 201static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
202 const enum i915_cache_level level)
b1fe6673 203{
07749ef3 204 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
b1fe6673
BW		 205 pde |= addr;
206 if (level != I915_CACHE_NONE)
207 pde |= PPAT_CACHED_PDE_INDEX;
208 else
209 pde |= PPAT_UNCACHED_INDEX;
210 return pde;
211}
212
762d9936
MT		 213#define gen8_pdpe_encode gen8_pde_encode
214#define gen8_pml4e_encode gen8_pde_encode
215
07749ef3
MT		 216static gen6_pte_t snb_pte_encode(dma_addr_t addr,
217 enum i915_cache_level level,
218 bool valid, u32 unused)
54d12527 219{
07749ef3 220 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
54d12527 221 pte |= GEN6_PTE_ADDR_ENCODE(addr);
e7210c3c
BW		 222
223 switch (level) {
350ec881
CW		 224 case I915_CACHE_L3_LLC:
225 case I915_CACHE_LLC:
226 pte |= GEN6_PTE_CACHE_LLC;
227 break;
228 case I915_CACHE_NONE:
229 pte |= GEN6_PTE_UNCACHED;
230 break;
231 default:
5f77eeb0 232 MISSING_CASE(level);
350ec881
CW		 233 }
234
235 return pte;
236}
237
07749ef3
MT		 238static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
239 enum i915_cache_level level,
240 bool valid, u32 unused)
350ec881 241{
07749ef3 242 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
350ec881
CW		 243 pte |= GEN6_PTE_ADDR_ENCODE(addr);
244
245 switch (level) {
246 case I915_CACHE_L3_LLC:
247 pte |= GEN7_PTE_CACHE_L3_LLC;
e7210c3c
BW		 248 break;
249 case I915_CACHE_LLC:
250 pte |= GEN6_PTE_CACHE_LLC;
251 break;
252 case I915_CACHE_NONE:
9119708c 253 pte |= GEN6_PTE_UNCACHED;
e7210c3c
BW		 254 break;
255 default:
5f77eeb0 256 MISSING_CASE(level);
e7210c3c
BW		 257 }
258
54d12527
BW		 259 return pte;
260}
261
07749ef3
MT		 262static gen6_pte_t byt_pte_encode(dma_addr_t addr,
263 enum i915_cache_level level,
264 bool valid, u32 flags)
93c34e70 265{
07749ef3 266 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
93c34e70
KG		 267 pte |= GEN6_PTE_ADDR_ENCODE(addr);
268
24f3a8cf
AG		 269 if (!(flags & PTE_READ_ONLY))
270 pte |= BYT_PTE_WRITEABLE;
93c34e70
KG		 271
272 if (level != I915_CACHE_NONE)
273 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
274
275 return pte;
276}
277
07749ef3
MT		 278static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
279 enum i915_cache_level level,
280 bool valid, u32 unused)
9119708c 281{
07749ef3 282 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
0d8ff15e 283 pte |= HSW_PTE_ADDR_ENCODE(addr);
9119708c
KG		 284
285 if (level != I915_CACHE_NONE)
87a6b688 286 pte |= HSW_WB_LLC_AGE3;
9119708c
KG		 287
288 return pte;
289}
290
07749ef3
MT		 291static gen6_pte_t iris_pte_encode(dma_addr_t addr,
292 enum i915_cache_level level,
293 bool valid, u32 unused)
4d15c145 294{
07749ef3 295 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
4d15c145
BW		 296 pte |= HSW_PTE_ADDR_ENCODE(addr);
297
651d794f
CW		 298 switch (level) {
299 case I915_CACHE_NONE:
300 break;
301 case I915_CACHE_WT:
c51e9701 302 pte |= HSW_WT_ELLC_LLC_AGE3;
651d794f
CW		 303 break;
304 default:
c51e9701 305 pte |= HSW_WB_ELLC_LLC_AGE3;
651d794f
CW		 306 break;
307 }
4d15c145
BW		 308
309 return pte;
310}
311
c114f76a
MK		 312static int __setup_page_dma(struct drm_device *dev,
313 struct i915_page_dma *p, gfp_t flags)
678d96fb
BW		 314{
315 struct device *device = &dev->pdev->dev;
316
c114f76a 317 p->page = alloc_page(flags);
44159ddb
MK		 318 if (!p->page)
319 return -ENOMEM;
678d96fb 320
44159ddb
MK		 321 p->daddr = dma_map_page(device,
322 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
678d96fb 323
44159ddb
MK		 324 if (dma_mapping_error(device, p->daddr)) {
325 __free_page(p->page);
326 return -EINVAL;
327 }
1266cdb1
MT		 328
329 return 0;
678d96fb
BW		 330}
331
c114f76a
MK		 332static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
333{
334 return __setup_page_dma(dev, p, GFP_KERNEL);
335}
336
44159ddb 337static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
06fda602 338{
44159ddb 339 if (WARN_ON(!p->page))
06fda602 340 return;
678d96fb 341
44159ddb
MK		 342 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
343 __free_page(p->page);
344 memset(p, 0, sizeof(*p));
345}
346
d1c54acd 347static void *kmap_page_dma(struct i915_page_dma *p)
73eeea53 348{
d1c54acd
MK		 349 return kmap_atomic(p->page);
350}
73eeea53 351
d1c54acd
MK		 352/* We use the flushing unmap only with ppgtt structures:
353 * page directories, page tables and scratch pages.
354 */
355static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
356{
73eeea53
MK		 357 /* There are only few exceptions for gen >=6. chv and bxt.
358 * And we are not sure about the latter so play safe for now.
359 */
360 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
361 drm_clflush_virt_range(vaddr, PAGE_SIZE);
362
363 kunmap_atomic(vaddr);
364}
365
567047be 366#define kmap_px(px) kmap_page_dma(px_base(px))
d1c54acd
MK		 367#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))
368
567047be
MK		 369#define setup_px(dev, px) setup_page_dma((dev), px_base(px))
370#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
371#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
372#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))
373
d1c54acd
MK		 374static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
375 const uint64_t val)
376{
377 int i;
378 uint64_t * const vaddr = kmap_page_dma(p);
379
380 for (i = 0; i < 512; i++)
381 vaddr[i] = val;
382
383 kunmap_page_dma(dev, vaddr);
384}
385
73eeea53
MK		 386static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
387 const uint32_t val32)
388{
389 uint64_t v = val32;
390
391 v = v << 32 | val32;
392
393 fill_page_dma(dev, p, v);
394}
395
4ad2af1e
MK		 396static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev)
397{
398 struct i915_page_scratch *sp;
399 int ret;
400
401 sp = kzalloc(sizeof(*sp), GFP_KERNEL);
402 if (sp == NULL)
403 return ERR_PTR(-ENOMEM);
404
405 ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
406 if (ret) {
407 kfree(sp);
408 return ERR_PTR(ret);
409 }
410
411 set_pages_uc(px_page(sp), 1);
412
413 return sp;
414}
415
416static void free_scratch_page(struct drm_device *dev,
417 struct i915_page_scratch *sp)
418{
419 set_pages_wb(px_page(sp), 1);
420
421 cleanup_px(dev, sp);
422 kfree(sp);
423}
424
8a1ebd74 425static struct i915_page_table *alloc_pt(struct drm_device *dev)
06fda602 426{
ec565b3c 427 struct i915_page_table *pt;
678d96fb
BW		 428 const size_t count = INTEL_INFO(dev)->gen >= 8 ?
429 GEN8_PTES : GEN6_PTES;
430 int ret = -ENOMEM;
06fda602
BW		 431
432 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
433 if (!pt)
434 return ERR_PTR(-ENOMEM);
435
678d96fb
BW		 436 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
437 GFP_KERNEL);
438
439 if (!pt->used_ptes)
440 goto fail_bitmap;
441
567047be 442 ret = setup_px(dev, pt);
678d96fb 443 if (ret)
44159ddb 444 goto fail_page_m;
06fda602
BW		 445
446 return pt;
678d96fb 447
44159ddb 448fail_page_m:
678d96fb
BW		 449 kfree(pt->used_ptes);
450fail_bitmap:
451 kfree(pt);
452
453 return ERR_PTR(ret);
06fda602
BW		 454}
455
2e906bea 456static void free_pt(struct drm_device *dev, struct i915_page_table *pt)
06fda602 457{
2e906bea
MK		 458 cleanup_px(dev, pt);
459 kfree(pt->used_ptes);
460 kfree(pt);
461}
462
463static void gen8_initialize_pt(struct i915_address_space *vm,
464 struct i915_page_table *pt)
465{
466 gen8_pte_t scratch_pte;
467
468 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
469 I915_CACHE_LLC, true);
470
471 fill_px(vm->dev, pt, scratch_pte);
472}
473
474static void gen6_initialize_pt(struct i915_address_space *vm,
475 struct i915_page_table *pt)
476{
477 gen6_pte_t scratch_pte;
478
479 WARN_ON(px_dma(vm->scratch_page) == 0);
480
481 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
482 I915_CACHE_LLC, true, 0);
483
484 fill32_px(vm->dev, pt, scratch_pte);
06fda602
BW		 485}
486
8a1ebd74 487static struct i915_page_directory *alloc_pd(struct drm_device *dev)
06fda602 488{
ec565b3c 489 struct i915_page_directory *pd;
33c8819f 490 int ret = -ENOMEM;
06fda602
BW		 491
492 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
493 if (!pd)
494 return ERR_PTR(-ENOMEM);
495
33c8819f
MT		 496 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
497 sizeof(*pd->used_pdes), GFP_KERNEL);
498 if (!pd->used_pdes)
a08e111a 499 goto fail_bitmap;
33c8819f 500
567047be 501 ret = setup_px(dev, pd);
33c8819f 502 if (ret)
a08e111a 503 goto fail_page_m;
e5815a2e 504
06fda602 505 return pd;
33c8819f 506
a08e111a 507fail_page_m:
33c8819f 508 kfree(pd->used_pdes);
a08e111a 509fail_bitmap:
33c8819f
MT		 510 kfree(pd);
511
512 return ERR_PTR(ret);
06fda602
BW		 513}
514
2e906bea
MK		 515static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)
516{
517 if (px_page(pd)) {
518 cleanup_px(dev, pd);
519 kfree(pd->used_pdes);
520 kfree(pd);
521 }
522}
523
524static void gen8_initialize_pd(struct i915_address_space *vm,
525 struct i915_page_directory *pd)
526{
527 gen8_pde_t scratch_pde;
528
529 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
530
531 fill_px(vm->dev, pd, scratch_pde);
532}
533
6ac18502
MT		 534static int __pdp_init(struct drm_device *dev,
535 struct i915_page_directory_pointer *pdp)
536{
537 size_t pdpes = I915_PDPES_PER_PDP(dev);
538
539 pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
540 sizeof(unsigned long),
541 GFP_KERNEL);
542 if (!pdp->used_pdpes)
543 return -ENOMEM;
544
545 pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
546 GFP_KERNEL);
547 if (!pdp->page_directory) {
548 kfree(pdp->used_pdpes);
549 /* the PDP might be the statically allocated top level. Keep it
550 * as clean as possible */
551 pdp->used_pdpes = NULL;
552 return -ENOMEM;
553 }
554
555 return 0;
556}
557
558static void __pdp_fini(struct i915_page_directory_pointer *pdp)
559{
560 kfree(pdp->used_pdpes);
561 kfree(pdp->page_directory);
562 pdp->page_directory = NULL;
563}
564
762d9936
MT		 565static struct
566i915_page_directory_pointer *alloc_pdp(struct drm_device *dev)
567{
568 struct i915_page_directory_pointer *pdp;
569 int ret = -ENOMEM;
570
571 WARN_ON(!USES_FULL_48BIT_PPGTT(dev));
572
573 pdp = kzalloc(sizeof(*pdp), GFP_KERNEL);
574 if (!pdp)
575 return ERR_PTR(-ENOMEM);
576
577 ret = __pdp_init(dev, pdp);
578 if (ret)
579 goto fail_bitmap;
580
581 ret = setup_px(dev, pdp);
582 if (ret)
583 goto fail_page_m;
584
585 return pdp;
586
587fail_page_m:
588 __pdp_fini(pdp);
589fail_bitmap:
590 kfree(pdp);
591
592 return ERR_PTR(ret);
593}
594
6ac18502
MT		 595static void free_pdp(struct drm_device *dev,
596 struct i915_page_directory_pointer *pdp)
597{
598 __pdp_fini(pdp);
762d9936
MT		 599 if (USES_FULL_48BIT_PPGTT(dev)) {
600 cleanup_px(dev, pdp);
601 kfree(pdp);
602 }
603}
604
69ab76fd
MT		 605static void gen8_initialize_pdp(struct i915_address_space *vm,
606 struct i915_page_directory_pointer *pdp)
607{
608 gen8_ppgtt_pdpe_t scratch_pdpe;
609
610 scratch_pdpe = gen8_pdpe_encode(px_dma(vm->scratch_pd), I915_CACHE_LLC);
611
612 fill_px(vm->dev, pdp, scratch_pdpe);
613}
614
615static void gen8_initialize_pml4(struct i915_address_space *vm,
616 struct i915_pml4 *pml4)
617{
618 gen8_ppgtt_pml4e_t scratch_pml4e;
619
620 scratch_pml4e = gen8_pml4e_encode(px_dma(vm->scratch_pdp),
621 I915_CACHE_LLC);
622
623 fill_px(vm->dev, pml4, scratch_pml4e);
624}
625
762d9936
MT		 626static void
627gen8_setup_page_directory(struct i915_hw_ppgtt *ppgtt,
628 struct i915_page_directory_pointer *pdp,
629 struct i915_page_directory *pd,
630 int index)
631{
632 gen8_ppgtt_pdpe_t *page_directorypo;
633
634 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
635 return;
636
637 page_directorypo = kmap_px(pdp);
638 page_directorypo[index] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
639 kunmap_px(ppgtt, page_directorypo);
640}
641
642static void
643gen8_setup_page_directory_pointer(struct i915_hw_ppgtt *ppgtt,
644 struct i915_pml4 *pml4,
645 struct i915_page_directory_pointer *pdp,
646 int index)
647{
648 gen8_ppgtt_pml4e_t *pagemap = kmap_px(pml4);
649
650 WARN_ON(!USES_FULL_48BIT_PPGTT(ppgtt->base.dev));
651 pagemap[index] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
652 kunmap_px(ppgtt, pagemap);
6ac18502
MT		 653}
654
94e409c1 655/* Broadwell Page Directory Pointer Descriptors */
e85b26dc 656static int gen8_write_pdp(struct drm_i915_gem_request *req,
7cb6d7ac
MT		 657 unsigned entry,
658 dma_addr_t addr)
94e409c1 659{
e85b26dc 660 struct intel_engine_cs *ring = req->ring;
94e409c1
BW		 661 int ret;
662
663 BUG_ON(entry >= 4);
664
5fb9de1a 665 ret = intel_ring_begin(req, 6);
94e409c1
BW		 666 if (ret)
667 return ret;
668
669 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
f92a9162 670 intel_ring_emit_reg(ring, GEN8_RING_PDP_UDW(ring, entry));
7cb6d7ac 671 intel_ring_emit(ring, upper_32_bits(addr));
94e409c1 672 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
f92a9162 673 intel_ring_emit_reg(ring, GEN8_RING_PDP_LDW(ring, entry));
7cb6d7ac 674 intel_ring_emit(ring, lower_32_bits(addr));
94e409c1
BW		 675 intel_ring_advance(ring);
676
677 return 0;
678}
679
2dba3239
MT		 680static int gen8_legacy_mm_switch(struct i915_hw_ppgtt *ppgtt,
681 struct drm_i915_gem_request *req)
94e409c1 682{
eeb9488e 683 int i, ret;
94e409c1 684
7cb6d7ac 685 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
d852c7bf
MK		 686 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
687
e85b26dc 688 ret = gen8_write_pdp(req, i, pd_daddr);
eeb9488e
BW		 689 if (ret)
690 return ret;
94e409c1 691 }
d595bd4b 692
eeb9488e 693 return 0;
94e409c1
BW		 694}
695
2dba3239
MT		 696static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt,
697 struct drm_i915_gem_request *req)
698{
699 return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4));
700}
701
f9b5b782
MT		 702static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
703 struct i915_page_directory_pointer *pdp,
704 uint64_t start,
705 uint64_t length,
706 gen8_pte_t scratch_pte)
459108b8
BW		 707{
708 struct i915_hw_ppgtt *ppgtt =
709 container_of(vm, struct i915_hw_ppgtt, base);
f9b5b782 710 gen8_pte_t *pt_vaddr;
de5ba8eb
MT		 711 unsigned pdpe = gen8_pdpe_index(start);
712 unsigned pde = gen8_pde_index(start);
713 unsigned pte = gen8_pte_index(start);
782f1495 714 unsigned num_entries = length >> PAGE_SHIFT;
459108b8
BW		 715 unsigned last_pte, i;
716
f9b5b782
MT		 717 if (WARN_ON(!pdp))
718 return;
459108b8
BW		 719
720 while (num_entries) {
ec565b3c
MT		 721 struct i915_page_directory *pd;
722 struct i915_page_table *pt;
06fda602 723
d4ec9da0 724 if (WARN_ON(!pdp->page_directory[pdpe]))
00245266 725 break;
06fda602 726
d4ec9da0 727 pd = pdp->page_directory[pdpe];
06fda602
BW		 728
729 if (WARN_ON(!pd->page_table[pde]))
00245266 730 break;
06fda602
BW		 731
732 pt = pd->page_table[pde];
733
567047be 734 if (WARN_ON(!px_page(pt)))
00245266 735 break;
06fda602 736
7ad47cf2 737 last_pte = pte + num_entries;
07749ef3
MT		 738 if (last_pte > GEN8_PTES)
739 last_pte = GEN8_PTES;
459108b8 740
d1c54acd 741 pt_vaddr = kmap_px(pt);
459108b8 742
7ad47cf2 743 for (i = pte; i < last_pte; i++) {
459108b8 744 pt_vaddr[i] = scratch_pte;
7ad47cf2
BW		 745 num_entries--;
746 }
459108b8 747
d1c54acd 748 kunmap_px(ppgtt, pt);
459108b8 749
7ad47cf2 750 pte = 0;
07749ef3 751 if (++pde == I915_PDES) {
de5ba8eb
MT		 752 if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
753 break;
7ad47cf2
BW		 754 pde = 0;
755 }
459108b8
BW		 756 }
757}
758
f9b5b782
MT		 759static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
760 uint64_t start,
761 uint64_t length,
762 bool use_scratch)
9df15b49
BW		 763{
764 struct i915_hw_ppgtt *ppgtt =
765 container_of(vm, struct i915_hw_ppgtt, base);
f9b5b782
MT		 766 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
767 I915_CACHE_LLC, use_scratch);
768
de5ba8eb
MT		 769 if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
770 gen8_ppgtt_clear_pte_range(vm, &ppgtt->pdp, start, length,
771 scratch_pte);
772 } else {
773 uint64_t templ4, pml4e;
774 struct i915_page_directory_pointer *pdp;
775
776 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
777 gen8_ppgtt_clear_pte_range(vm, pdp, start, length,
778 scratch_pte);
779 }
780 }
f9b5b782
MT		 781}
782
783static void
784gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
785 struct i915_page_directory_pointer *pdp,
3387d433 786 struct sg_page_iter *sg_iter,
f9b5b782
MT		 787 uint64_t start,
788 enum i915_cache_level cache_level)
789{
790 struct i915_hw_ppgtt *ppgtt =
791 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 792 gen8_pte_t *pt_vaddr;
de5ba8eb
MT		 793 unsigned pdpe = gen8_pdpe_index(start);
794 unsigned pde = gen8_pde_index(start);
795 unsigned pte = gen8_pte_index(start);
9df15b49 796
6f1cc993 797 pt_vaddr = NULL;
7ad47cf2 798
3387d433 799 while (__sg_page_iter_next(sg_iter)) {
d7b3de91 800 if (pt_vaddr == NULL) {
d4ec9da0 801 struct i915_page_directory *pd = pdp->page_directory[pdpe];
ec565b3c 802 struct i915_page_table *pt = pd->page_table[pde];
d1c54acd 803 pt_vaddr = kmap_px(pt);
d7b3de91 804 }
9df15b49 805
7ad47cf2 806 pt_vaddr[pte] =
3387d433 807 gen8_pte_encode(sg_page_iter_dma_address(sg_iter),
6f1cc993 808 cache_level, true);
07749ef3 809 if (++pte == GEN8_PTES) {
d1c54acd 810 kunmap_px(ppgtt, pt_vaddr);
6f1cc993 811 pt_vaddr = NULL;
07749ef3 812 if (++pde == I915_PDES) {
de5ba8eb
MT		 813 if (++pdpe == I915_PDPES_PER_PDP(vm->dev))
814 break;
7ad47cf2
BW		 815 pde = 0;
816 }
817 pte = 0;
9df15b49
BW		 818 }
819 }
d1c54acd
MK		 820
821 if (pt_vaddr)
822 kunmap_px(ppgtt, pt_vaddr);
9df15b49
BW		 823}
824
f9b5b782
MT		 825static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
826 struct sg_table *pages,
827 uint64_t start,
828 enum i915_cache_level cache_level,
829 u32 unused)
830{
831 struct i915_hw_ppgtt *ppgtt =
832 container_of(vm, struct i915_hw_ppgtt, base);
3387d433 833 struct sg_page_iter sg_iter;
f9b5b782 834
3387d433 835 __sg_page_iter_start(&sg_iter, pages->sgl, sg_nents(pages->sgl), 0);
de5ba8eb
MT		 836
837 if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
838 gen8_ppgtt_insert_pte_entries(vm, &ppgtt->pdp, &sg_iter, start,
839 cache_level);
840 } else {
841 struct i915_page_directory_pointer *pdp;
842 uint64_t templ4, pml4e;
843 uint64_t length = (uint64_t)pages->orig_nents << PAGE_SHIFT;
844
845 gen8_for_each_pml4e(pdp, &ppgtt->pml4, start, length, templ4, pml4e) {
846 gen8_ppgtt_insert_pte_entries(vm, pdp, &sg_iter,
847 start, cache_level);
848 }
849 }
f9b5b782
MT		 850}
851
f37c0505
MT		 852static void gen8_free_page_tables(struct drm_device *dev,
853 struct i915_page_directory *pd)
7ad47cf2
BW		 854{
855 int i;
856
567047be 857 if (!px_page(pd))
7ad47cf2
BW		 858 return;
859
33c8819f 860 for_each_set_bit(i, pd->used_pdes, I915_PDES) {
06fda602
BW		 861 if (WARN_ON(!pd->page_table[i]))
862 continue;
7ad47cf2 863
a08e111a 864 free_pt(dev, pd->page_table[i]);
06fda602
BW		 865 pd->page_table[i] = NULL;
866 }
d7b3de91
BW		 867}
868
8776f02b
MK		 869static int gen8_init_scratch(struct i915_address_space *vm)
870{
871 struct drm_device *dev = vm->dev;
872
873 vm->scratch_page = alloc_scratch_page(dev);
874 if (IS_ERR(vm->scratch_page))
875 return PTR_ERR(vm->scratch_page);
876
877 vm->scratch_pt = alloc_pt(dev);
878 if (IS_ERR(vm->scratch_pt)) {
879 free_scratch_page(dev, vm->scratch_page);
880 return PTR_ERR(vm->scratch_pt);
881 }
882
883 vm->scratch_pd = alloc_pd(dev);
884 if (IS_ERR(vm->scratch_pd)) {
885 free_pt(dev, vm->scratch_pt);
886 free_scratch_page(dev, vm->scratch_page);
887 return PTR_ERR(vm->scratch_pd);
888 }
889
69ab76fd
MT		 890 if (USES_FULL_48BIT_PPGTT(dev)) {
891 vm->scratch_pdp = alloc_pdp(dev);
892 if (IS_ERR(vm->scratch_pdp)) {
893 free_pd(dev, vm->scratch_pd);
894 free_pt(dev, vm->scratch_pt);
895 free_scratch_page(dev, vm->scratch_page);
896 return PTR_ERR(vm->scratch_pdp);
897 }
898 }
899
8776f02b
MK		 900 gen8_initialize_pt(vm, vm->scratch_pt);
901 gen8_initialize_pd(vm, vm->scratch_pd);
69ab76fd
MT		 902 if (USES_FULL_48BIT_PPGTT(dev))
903 gen8_initialize_pdp(vm, vm->scratch_pdp);
8776f02b
MK		 904
905 return 0;
906}
907
650da34c
ZL		 908static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
909{
910 enum vgt_g2v_type msg;
911 struct drm_device *dev = ppgtt->base.dev;
912 struct drm_i915_private *dev_priv = dev->dev_private;
913 unsigned int offset = vgtif_reg(pdp0_lo);
914 int i;
915
916 if (USES_FULL_48BIT_PPGTT(dev)) {
917 u64 daddr = px_dma(&ppgtt->pml4);
918
919 I915_WRITE(offset, lower_32_bits(daddr));
920 I915_WRITE(offset + 4, upper_32_bits(daddr));
921
922 msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
923 VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
924 } else {
925 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
926 u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
927
928 I915_WRITE(offset, lower_32_bits(daddr));
929 I915_WRITE(offset + 4, upper_32_bits(daddr));
930
931 offset += 8;
932 }
933
934 msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
935 VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
936 }
937
938 I915_WRITE(vgtif_reg(g2v_notify), msg);
939
940 return 0;
941}
942
8776f02b
MK		 943static void gen8_free_scratch(struct i915_address_space *vm)
944{
945 struct drm_device *dev = vm->dev;
946
69ab76fd
MT		 947 if (USES_FULL_48BIT_PPGTT(dev))
948 free_pdp(dev, vm->scratch_pdp);
8776f02b
MK		 949 free_pd(dev, vm->scratch_pd);
950 free_pt(dev, vm->scratch_pt);
951 free_scratch_page(dev, vm->scratch_page);
952}
953
762d9936
MT		 954static void gen8_ppgtt_cleanup_3lvl(struct drm_device *dev,
955 struct i915_page_directory_pointer *pdp)
b45a6715
BW		 956{
957 int i;
958
d4ec9da0
MT		 959 for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {
960 if (WARN_ON(!pdp->page_directory[i]))
06fda602
BW		 961 continue;
962
d4ec9da0
MT		 963 gen8_free_page_tables(dev, pdp->page_directory[i]);
964 free_pd(dev, pdp->page_directory[i]);
7ad47cf2 965 }
69876bed 966
d4ec9da0 967 free_pdp(dev, pdp);
762d9936
MT		 968}
969
970static void gen8_ppgtt_cleanup_4lvl(struct i915_hw_ppgtt *ppgtt)
971{
972 int i;
973
974 for_each_set_bit(i, ppgtt->pml4.used_pml4es, GEN8_PML4ES_PER_PML4) {
975 if (WARN_ON(!ppgtt->pml4.pdps[i]))
976 continue;
977
978 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, ppgtt->pml4.pdps[i]);
979 }
980
981 cleanup_px(ppgtt->base.dev, &ppgtt->pml4);
982}
983
984static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
985{
986 struct i915_hw_ppgtt *ppgtt =
987 container_of(vm, struct i915_hw_ppgtt, base);
988
650da34c
ZL		 989 if (intel_vgpu_active(vm->dev))
990 gen8_ppgtt_notify_vgt(ppgtt, false);
991
762d9936
MT		 992 if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
993 gen8_ppgtt_cleanup_3lvl(ppgtt->base.dev, &ppgtt->pdp);
994 else
995 gen8_ppgtt_cleanup_4lvl(ppgtt);
d4ec9da0 996
8776f02b 997 gen8_free_scratch(vm);
b45a6715
BW		 998}
999
d7b2633d
MT		 1000/**
1001 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
d4ec9da0
MT		 1002 * @vm: Master vm structure.
1003 * @pd: Page directory for this address range.
d7b2633d 1004 * @start: Starting virtual address to begin allocations.
d4ec9da0 1005 * @length: Size of the allocations.
d7b2633d
MT		 1006 * @new_pts: Bitmap set by function with new allocations. Likely used by the
1007 * caller to free on error.
1008 *
1009 * Allocate the required number of page tables. Extremely similar to
1010 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
1011 * the page directory boundary (instead of the page directory pointer). That
1012 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
1013 * possible, and likely that the caller will need to use multiple calls of this
1014 * function to achieve the appropriate allocation.
1015 *
1016 * Return: 0 if success; negative error code otherwise.
1017 */
d4ec9da0 1018static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
e5815a2e 1019 struct i915_page_directory *pd,
5441f0cb 1020 uint64_t start,
d7b2633d
MT		 1021 uint64_t length,
1022 unsigned long *new_pts)
bf2b4ed2 1023{
d4ec9da0 1024 struct drm_device *dev = vm->dev;
d7b2633d 1025 struct i915_page_table *pt;
5441f0cb
MT		 1026 uint64_t temp;
1027 uint32_t pde;
bf2b4ed2 1028
d7b2633d
MT		 1029 gen8_for_each_pde(pt, pd, start, length, temp, pde) {
1030 /* Don't reallocate page tables */
6ac18502 1031 if (test_bit(pde, pd->used_pdes)) {
d7b2633d 1032 /* Scratch is never allocated this way */
d4ec9da0 1033 WARN_ON(pt == vm->scratch_pt);
d7b2633d
MT		 1034 continue;
1035 }
1036
8a1ebd74 1037 pt = alloc_pt(dev);
d7b2633d 1038 if (IS_ERR(pt))
5441f0cb
MT		 1039 goto unwind_out;
1040
d4ec9da0 1041 gen8_initialize_pt(vm, pt);
d7b2633d 1042 pd->page_table[pde] = pt;
966082c9 1043 __set_bit(pde, new_pts);
4c06ec8d 1044 trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
7ad47cf2
BW		 1045 }
1046
bf2b4ed2 1047 return 0;
7ad47cf2
BW		 1048
1049unwind_out:
d7b2633d 1050 for_each_set_bit(pde, new_pts, I915_PDES)
a08e111a 1051 free_pt(dev, pd->page_table[pde]);
7ad47cf2 1052
d7b3de91 1053 return -ENOMEM;
bf2b4ed2
BW		 1054}
1055
d7b2633d
MT		 1056/**
1057 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
d4ec9da0 1058 * @vm: Master vm structure.
d7b2633d
MT		 1059 * @pdp: Page directory pointer for this address range.
1060 * @start: Starting virtual address to begin allocations.
d4ec9da0
MT		 1061 * @length: Size of the allocations.
1062 * @new_pds: Bitmap set by function with new allocations. Likely used by the
d7b2633d
MT		 1063 * caller to free on error.
1064 *
1065 * Allocate the required number of page directories starting at the pde index of
1066 * @start, and ending at the pde index @start + @length. This function will skip
1067 * over already allocated page directories within the range, and only allocate
1068 * new ones, setting the appropriate pointer within the pdp as well as the
1069 * correct position in the bitmap @new_pds.
1070 *
1071 * The function will only allocate the pages within the range for a give page
1072 * directory pointer. In other words, if @start + @length straddles a virtually
1073 * addressed PDP boundary (512GB for 4k pages), there will be more allocations
1074 * required by the caller, This is not currently possible, and the BUG in the
1075 * code will prevent it.
1076 *
1077 * Return: 0 if success; negative error code otherwise.
1078 */
d4ec9da0
MT		 1079static int
1080gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
1081 struct i915_page_directory_pointer *pdp,
1082 uint64_t start,
1083 uint64_t length,
1084 unsigned long *new_pds)
bf2b4ed2 1085{
d4ec9da0 1086 struct drm_device *dev = vm->dev;
d7b2633d 1087 struct i915_page_directory *pd;
69876bed
MT		 1088 uint64_t temp;
1089 uint32_t pdpe;
6ac18502 1090 uint32_t pdpes = I915_PDPES_PER_PDP(dev);
69876bed 1091
6ac18502 1092 WARN_ON(!bitmap_empty(new_pds, pdpes));
d7b2633d 1093
d7b2633d 1094 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
6ac18502 1095 if (test_bit(pdpe, pdp->used_pdpes))
d7b2633d 1096 continue;
33c8819f 1097
8a1ebd74 1098 pd = alloc_pd(dev);
d7b2633d 1099 if (IS_ERR(pd))
d7b3de91 1100 goto unwind_out;
69876bed 1101
d4ec9da0 1102 gen8_initialize_pd(vm, pd);
d7b2633d 1103 pdp->page_directory[pdpe] = pd;
966082c9 1104 __set_bit(pdpe, new_pds);
4c06ec8d 1105 trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
d7b3de91
BW		 1106 }
1107
bf2b4ed2 1108 return 0;
d7b3de91
BW		 1109
1110unwind_out:
6ac18502 1111 for_each_set_bit(pdpe, new_pds, pdpes)
a08e111a 1112 free_pd(dev, pdp->page_directory[pdpe]);
d7b3de91
BW		 1113
1114 return -ENOMEM;
bf2b4ed2
BW		 1115}
1116
762d9936
MT		 1117/**
1118 * gen8_ppgtt_alloc_page_dirpointers() - Allocate pdps for VA range.
1119 * @vm: Master vm structure.
1120 * @pml4: Page map level 4 for this address range.
1121 * @start: Starting virtual address to begin allocations.
1122 * @length: Size of the allocations.
1123 * @new_pdps: Bitmap set by function with new allocations. Likely used by the
1124 * caller to free on error.
1125 *
1126 * Allocate the required number of page directory pointers. Extremely similar to
1127 * gen8_ppgtt_alloc_page_directories() and gen8_ppgtt_alloc_pagetabs().
1128 * The main difference is here we are limited by the pml4 boundary (instead of
1129 * the page directory pointer).
1130 *
1131 * Return: 0 if success; negative error code otherwise.
1132 */
1133static int
1134gen8_ppgtt_alloc_page_dirpointers(struct i915_address_space *vm,
1135 struct i915_pml4 *pml4,
1136 uint64_t start,
1137 uint64_t length,
1138 unsigned long *new_pdps)
1139{
1140 struct drm_device *dev = vm->dev;
1141 struct i915_page_directory_pointer *pdp;
1142 uint64_t temp;
1143 uint32_t pml4e;
1144
1145 WARN_ON(!bitmap_empty(new_pdps, GEN8_PML4ES_PER_PML4));
1146
1147 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
1148 if (!test_bit(pml4e, pml4->used_pml4es)) {
1149 pdp = alloc_pdp(dev);
1150 if (IS_ERR(pdp))
1151 goto unwind_out;
1152
69ab76fd 1153 gen8_initialize_pdp(vm, pdp);
762d9936
MT		 1154 pml4->pdps[pml4e] = pdp;
1155 __set_bit(pml4e, new_pdps);
1156 trace_i915_page_directory_pointer_entry_alloc(vm,
1157 pml4e,
1158 start,
1159 GEN8_PML4E_SHIFT);
1160 }
1161 }
1162
1163 return 0;
1164
1165unwind_out:
1166 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1167 free_pdp(dev, pml4->pdps[pml4e]);
1168
1169 return -ENOMEM;
1170}
1171
d7b2633d 1172static void
3a41a05d 1173free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long *new_pts)
d7b2633d 1174{
d7b2633d
MT		 1175 kfree(new_pts);
1176 kfree(new_pds);
1177}
1178
1179/* Fills in the page directory bitmap, and the array of page tables bitmap. Both
1180 * of these are based on the number of PDPEs in the system.
1181 */
1182static
1183int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
3a41a05d 1184 unsigned long **new_pts,
6ac18502 1185 uint32_t pdpes)
d7b2633d 1186{
d7b2633d 1187 unsigned long *pds;
3a41a05d 1188 unsigned long *pts;
d7b2633d 1189
3a41a05d 1190 pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_TEMPORARY);
d7b2633d
MT		 1191 if (!pds)
1192 return -ENOMEM;
1193
3a41a05d
MW		 1194 pts = kcalloc(pdpes, BITS_TO_LONGS(I915_PDES) * sizeof(unsigned long),
1195 GFP_TEMPORARY);
1196 if (!pts)
1197 goto err_out;
d7b2633d
MT		 1198
1199 *new_pds = pds;
1200 *new_pts = pts;
1201
1202 return 0;
1203
1204err_out:
3a41a05d 1205 free_gen8_temp_bitmaps(pds, pts);
d7b2633d
MT		 1206 return -ENOMEM;
1207}
1208
5b7e4c9c
MK		 1209/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
1210 * the page table structures, we mark them dirty so that
1211 * context switching/execlist queuing code takes extra steps
1212 * to ensure that tlbs are flushed.
1213 */
1214static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
1215{
1216 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
1217}
1218
762d9936
MT		 1219static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm,
1220 struct i915_page_directory_pointer *pdp,
1221 uint64_t start,
1222 uint64_t length)
bf2b4ed2 1223{
e5815a2e
MT		 1224 struct i915_hw_ppgtt *ppgtt =
1225 container_of(vm, struct i915_hw_ppgtt, base);
3a41a05d 1226 unsigned long *new_page_dirs, *new_page_tables;
d4ec9da0 1227 struct drm_device *dev = vm->dev;
5441f0cb 1228 struct i915_page_directory *pd;
33c8819f
MT		 1229 const uint64_t orig_start = start;
1230 const uint64_t orig_length = length;
5441f0cb
MT		 1231 uint64_t temp;
1232 uint32_t pdpe;
d4ec9da0 1233 uint32_t pdpes = I915_PDPES_PER_PDP(dev);
bf2b4ed2
BW		 1234 int ret;
1235
d7b2633d
MT		 1236 /* Wrap is never okay since we can only represent 48b, and we don't
1237 * actually use the other side of the canonical address space.
1238 */
1239 if (WARN_ON(start + length < start))
a05d80ee
MK		 1240 return -ENODEV;
1241
d4ec9da0 1242 if (WARN_ON(start + length > vm->total))
a05d80ee 1243 return -ENODEV;
d7b2633d 1244
6ac18502 1245 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
bf2b4ed2
BW		 1246 if (ret)
1247 return ret;
1248
d7b2633d 1249 /* Do the allocations first so we can easily bail out */
d4ec9da0
MT		 1250 ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
1251 new_page_dirs);
d7b2633d 1252 if (ret) {
3a41a05d 1253 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
d7b2633d
MT		 1254 return ret;
1255 }
1256
1257 /* For every page directory referenced, allocate page tables */
d4ec9da0
MT		 1258 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1259 ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
3a41a05d 1260 new_page_tables + pdpe * BITS_TO_LONGS(I915_PDES));
5441f0cb
MT		 1261 if (ret)
1262 goto err_out;
5441f0cb
MT		 1263 }
1264
33c8819f
MT		 1265 start = orig_start;
1266 length = orig_length;
1267
d7b2633d
MT		 1268 /* Allocations have completed successfully, so set the bitmaps, and do
1269 * the mappings. */
d4ec9da0 1270 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
d1c54acd 1271 gen8_pde_t *const page_directory = kmap_px(pd);
33c8819f 1272 struct i915_page_table *pt;
09120d4e 1273 uint64_t pd_len = length;
33c8819f
MT		 1274 uint64_t pd_start = start;
1275 uint32_t pde;
1276
d7b2633d
MT		 1277 /* Every pd should be allocated, we just did that above. */
1278 WARN_ON(!pd);
1279
1280 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
1281 /* Same reasoning as pd */
1282 WARN_ON(!pt);
1283 WARN_ON(!pd_len);
1284 WARN_ON(!gen8_pte_count(pd_start, pd_len));
1285
1286 /* Set our used ptes within the page table */
1287 bitmap_set(pt->used_ptes,
1288 gen8_pte_index(pd_start),
1289 gen8_pte_count(pd_start, pd_len));
1290
1291 /* Our pde is now pointing to the pagetable, pt */
966082c9 1292 __set_bit(pde, pd->used_pdes);
d7b2633d
MT		 1293
1294 /* Map the PDE to the page table */
fe36f55d
MK		 1295 page_directory[pde] = gen8_pde_encode(px_dma(pt),
1296 I915_CACHE_LLC);
4c06ec8d
MT		 1297 trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
1298 gen8_pte_index(start),
1299 gen8_pte_count(start, length),
1300 GEN8_PTES);
d7b2633d
MT		 1301
1302 /* NB: We haven't yet mapped ptes to pages. At this
1303 * point we're still relying on insert_entries() */
33c8819f 1304 }
d7b2633d 1305
d1c54acd 1306 kunmap_px(ppgtt, page_directory);
d4ec9da0 1307 __set_bit(pdpe, pdp->used_pdpes);
762d9936 1308 gen8_setup_page_directory(ppgtt, pdp, pd, pdpe);
33c8819f
MT		 1309 }
1310
3a41a05d 1311 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
5b7e4c9c 1312 mark_tlbs_dirty(ppgtt);
d7b3de91 1313 return 0;
bf2b4ed2 1314
d7b3de91 1315err_out:
d7b2633d 1316 while (pdpe--) {
3a41a05d
MW		 1317 for_each_set_bit(temp, new_page_tables + pdpe *
1318 BITS_TO_LONGS(I915_PDES), I915_PDES)
d4ec9da0 1319 free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
d7b2633d
MT		 1320 }
1321
6ac18502 1322 for_each_set_bit(pdpe, new_page_dirs, pdpes)
d4ec9da0 1323 free_pd(dev, pdp->page_directory[pdpe]);
d7b2633d 1324
3a41a05d 1325 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
5b7e4c9c 1326 mark_tlbs_dirty(ppgtt);
bf2b4ed2
BW		 1327 return ret;
1328}
1329
762d9936
MT		 1330static int gen8_alloc_va_range_4lvl(struct i915_address_space *vm,
1331 struct i915_pml4 *pml4,
1332 uint64_t start,
1333 uint64_t length)
1334{
1335 DECLARE_BITMAP(new_pdps, GEN8_PML4ES_PER_PML4);
1336 struct i915_hw_ppgtt *ppgtt =
1337 container_of(vm, struct i915_hw_ppgtt, base);
1338 struct i915_page_directory_pointer *pdp;
1339 uint64_t temp, pml4e;
1340 int ret = 0;
1341
1342 /* Do the pml4 allocations first, so we don't need to track the newly
1343 * allocated tables below the pdp */
1344 bitmap_zero(new_pdps, GEN8_PML4ES_PER_PML4);
1345
1346 /* The pagedirectory and pagetable allocations are done in the shared 3
1347 * and 4 level code. Just allocate the pdps.
1348 */
1349 ret = gen8_ppgtt_alloc_page_dirpointers(vm, pml4, start, length,
1350 new_pdps);
1351 if (ret)
1352 return ret;
1353
1354 WARN(bitmap_weight(new_pdps, GEN8_PML4ES_PER_PML4) > 2,
1355 "The allocation has spanned more than 512GB. "
1356 "It is highly likely this is incorrect.");
1357
1358 gen8_for_each_pml4e(pdp, pml4, start, length, temp, pml4e) {
1359 WARN_ON(!pdp);
1360
1361 ret = gen8_alloc_va_range_3lvl(vm, pdp, start, length);
1362 if (ret)
1363 goto err_out;
1364
1365 gen8_setup_page_directory_pointer(ppgtt, pml4, pdp, pml4e);
1366 }
1367
1368 bitmap_or(pml4->used_pml4es, new_pdps, pml4->used_pml4es,
1369 GEN8_PML4ES_PER_PML4);
1370
1371 return 0;
1372
1373err_out:
1374 for_each_set_bit(pml4e, new_pdps, GEN8_PML4ES_PER_PML4)
1375 gen8_ppgtt_cleanup_3lvl(vm->dev, pml4->pdps[pml4e]);
1376
1377 return ret;
1378}
1379
1380static int gen8_alloc_va_range(struct i915_address_space *vm,
1381 uint64_t start, uint64_t length)
1382{
1383 struct i915_hw_ppgtt *ppgtt =
1384 container_of(vm, struct i915_hw_ppgtt, base);
1385
1386 if (USES_FULL_48BIT_PPGTT(vm->dev))
1387 return gen8_alloc_va_range_4lvl(vm, &ppgtt->pml4, start, length);
1388 else
1389 return gen8_alloc_va_range_3lvl(vm, &ppgtt->pdp, start, length);
1390}
1391
ea91e401
MT		 1392static void gen8_dump_pdp(struct i915_page_directory_pointer *pdp,
1393 uint64_t start, uint64_t length,
1394 gen8_pte_t scratch_pte,
1395 struct seq_file *m)
1396{
1397 struct i915_page_directory *pd;
1398 uint64_t temp;
1399 uint32_t pdpe;
1400
1401 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1402 struct i915_page_table *pt;
1403 uint64_t pd_len = length;
1404 uint64_t pd_start = start;
1405 uint32_t pde;
1406
1407 if (!test_bit(pdpe, pdp->used_pdpes))
1408 continue;
1409
1410 seq_printf(m, "\tPDPE #%d\n", pdpe);
1411 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
1412 uint32_t pte;
1413 gen8_pte_t *pt_vaddr;
1414
1415 if (!test_bit(pde, pd->used_pdes))
1416 continue;
1417
1418 pt_vaddr = kmap_px(pt);
1419 for (pte = 0; pte < GEN8_PTES; pte += 4) {
1420 uint64_t va =
1421 (pdpe << GEN8_PDPE_SHIFT) |
1422 (pde << GEN8_PDE_SHIFT) |
1423 (pte << GEN8_PTE_SHIFT);
1424 int i;
1425 bool found = false;
1426
1427 for (i = 0; i < 4; i++)
1428 if (pt_vaddr[pte + i] != scratch_pte)
1429 found = true;
1430 if (!found)
1431 continue;
1432
1433 seq_printf(m, "\t\t0x%llx [%03d,%03d,%04d]: =", va, pdpe, pde, pte);
1434 for (i = 0; i < 4; i++) {
1435 if (pt_vaddr[pte + i] != scratch_pte)
1436 seq_printf(m, " %llx", pt_vaddr[pte + i]);
1437 else
1438 seq_puts(m, " SCRATCH ");
1439 }
1440 seq_puts(m, "\n");
1441 }
1442 /* don't use kunmap_px, it could trigger
1443 * an unnecessary flush.
1444 */
1445 kunmap_atomic(pt_vaddr);
1446 }
1447 }
1448}
1449
1450static void gen8_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1451{
1452 struct i915_address_space *vm = &ppgtt->base;
1453 uint64_t start = ppgtt->base.start;
1454 uint64_t length = ppgtt->base.total;
1455 gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
1456 I915_CACHE_LLC, true);
1457
1458 if (!USES_FULL_48BIT_PPGTT(vm->dev)) {
1459 gen8_dump_pdp(&ppgtt->pdp, start, length, scratch_pte, m);
1460 } else {
1461 uint64_t templ4, pml4e;
1462 struct i915_pml4 *pml4 = &ppgtt->pml4;
1463 struct i915_page_directory_pointer *pdp;
1464
1465 gen8_for_each_pml4e(pdp, pml4, start, length, templ4, pml4e) {
1466 if (!test_bit(pml4e, pml4->used_pml4es))
1467 continue;
1468
1469 seq_printf(m, " PML4E #%llu\n", pml4e);
1470 gen8_dump_pdp(pdp, start, length, scratch_pte, m);
1471 }
1472 }
1473}
1474
331f38e7
ZL		 1475static int gen8_preallocate_top_level_pdps(struct i915_hw_ppgtt *ppgtt)
1476{
3a41a05d 1477 unsigned long *new_page_dirs, *new_page_tables;
331f38e7
ZL		 1478 uint32_t pdpes = I915_PDPES_PER_PDP(dev);
1479 int ret;
1480
1481 /* We allocate temp bitmap for page tables for no gain
1482 * but as this is for init only, lets keep the things simple
1483 */
1484 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1485 if (ret)
1486 return ret;
1487
1488 /* Allocate for all pdps regardless of how the ppgtt
1489 * was defined.
1490 */
1491 ret = gen8_ppgtt_alloc_page_directories(&ppgtt->base, &ppgtt->pdp,
1492 0, 1ULL << 32,
1493 new_page_dirs);
1494 if (!ret)
1495 *ppgtt->pdp.used_pdpes = *new_page_dirs;
1496
3a41a05d 1497 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
331f38e7
ZL		 1498
1499 return ret;
1500}
1501
eb0b44ad 1502/*
f3a964b9
BW		 1503 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
1504 * with a net effect resembling a 2-level page table in normal x86 terms. Each
1505 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
1506 * space.
37aca44a 1507 *
f3a964b9 1508 */
5c5f6457 1509static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
37aca44a 1510{
8776f02b 1511 int ret;
7cb6d7ac 1512
8776f02b
MK		 1513 ret = gen8_init_scratch(&ppgtt->base);
1514 if (ret)
1515 return ret;
69876bed 1516
d7b2633d 1517 ppgtt->base.start = 0;
d7b2633d 1518 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
5c5f6457 1519 ppgtt->base.allocate_va_range = gen8_alloc_va_range;
d7b2633d 1520 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
c7e16f22 1521 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
777dc5bb
DV		 1522 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1523 ppgtt->base.bind_vma = ppgtt_bind_vma;
ea91e401 1524 ppgtt->debug_dump = gen8_dump_ppgtt;
d7b2633d 1525
762d9936
MT		 1526 if (USES_FULL_48BIT_PPGTT(ppgtt->base.dev)) {
1527 ret = setup_px(ppgtt->base.dev, &ppgtt->pml4);
1528 if (ret)
1529 goto free_scratch;
6ac18502 1530
69ab76fd
MT		 1531 gen8_initialize_pml4(&ppgtt->base, &ppgtt->pml4);
1532
762d9936 1533 ppgtt->base.total = 1ULL << 48;
2dba3239 1534 ppgtt->switch_mm = gen8_48b_mm_switch;
762d9936 1535 } else {
25f50337 1536 ret = __pdp_init(ppgtt->base.dev, &ppgtt->pdp);
81ba8aef
MT		 1537 if (ret)
1538 goto free_scratch;
1539
1540 ppgtt->base.total = 1ULL << 32;
2dba3239 1541 ppgtt->switch_mm = gen8_legacy_mm_switch;
762d9936
MT		 1542 trace_i915_page_directory_pointer_entry_alloc(&ppgtt->base,
1543 0, 0,
1544 GEN8_PML4E_SHIFT);
331f38e7
ZL		 1545
1546 if (intel_vgpu_active(ppgtt->base.dev)) {
1547 ret = gen8_preallocate_top_level_pdps(ppgtt);
1548 if (ret)
1549 goto free_scratch;
1550 }
81ba8aef 1551 }
6ac18502 1552
650da34c
ZL		 1553 if (intel_vgpu_active(ppgtt->base.dev))
1554 gen8_ppgtt_notify_vgt(ppgtt, true);
1555
d7b2633d 1556 return 0;
6ac18502
MT		 1557
1558free_scratch:
1559 gen8_free_scratch(&ppgtt->base);
1560 return ret;
d7b2633d
MT		 1561}
1562
87d60b63
BW		 1563static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1564{
87d60b63 1565 struct i915_address_space *vm = &ppgtt->base;
09942c65 1566 struct i915_page_table *unused;
07749ef3 1567 gen6_pte_t scratch_pte;
87d60b63 1568 uint32_t pd_entry;
09942c65
MT		 1569 uint32_t pte, pde, temp;
1570 uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
87d60b63 1571
79ab9370
MK		 1572 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1573 I915_CACHE_LLC, true, 0);
87d60b63 1574
09942c65 1575 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
87d60b63 1576 u32 expected;
07749ef3 1577 gen6_pte_t *pt_vaddr;
567047be 1578 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
09942c65 1579 pd_entry = readl(ppgtt->pd_addr + pde);
87d60b63
BW		 1580 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1581
1582 if (pd_entry != expected)
1583 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1584 pde,
1585 pd_entry,
1586 expected);
1587 seq_printf(m, "\tPDE: %x\n", pd_entry);
1588
d1c54acd
MK		 1589 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1590
07749ef3 1591 for (pte = 0; pte < GEN6_PTES; pte+=4) {
87d60b63 1592 unsigned long va =
07749ef3 1593 (pde * PAGE_SIZE * GEN6_PTES) +
87d60b63
BW		 1594 (pte * PAGE_SIZE);
1595 int i;
1596 bool found = false;
1597 for (i = 0; i < 4; i++)
1598 if (pt_vaddr[pte + i] != scratch_pte)
1599 found = true;
1600 if (!found)
1601 continue;
1602
1603 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1604 for (i = 0; i < 4; i++) {
1605 if (pt_vaddr[pte + i] != scratch_pte)
1606 seq_printf(m, " %08x", pt_vaddr[pte + i]);
1607 else
1608 seq_puts(m, " SCRATCH ");
1609 }
1610 seq_puts(m, "\n");
1611 }
d1c54acd 1612 kunmap_px(ppgtt, pt_vaddr);
87d60b63
BW		 1613 }
1614}
1615
678d96fb 1616/* Write pde (index) from the page directory @pd to the page table @pt */
ec565b3c
MT		 1617static void gen6_write_pde(struct i915_page_directory *pd,
1618 const int pde, struct i915_page_table *pt)
6197349b 1619{
678d96fb
BW		 1620 /* Caller needs to make sure the write completes if necessary */
1621 struct i915_hw_ppgtt *ppgtt =
1622 container_of(pd, struct i915_hw_ppgtt, pd);
1623 u32 pd_entry;
6197349b 1624
567047be 1625 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
678d96fb 1626 pd_entry |= GEN6_PDE_VALID;
6197349b 1627
678d96fb
BW		 1628 writel(pd_entry, ppgtt->pd_addr + pde);
1629}
6197349b 1630
678d96fb
BW		 1631/* Write all the page tables found in the ppgtt structure to incrementing page
1632 * directories. */
1633static void gen6_write_page_range(struct drm_i915_private *dev_priv,
ec565b3c 1634 struct i915_page_directory *pd,
678d96fb
BW		 1635 uint32_t start, uint32_t length)
1636{
ec565b3c 1637 struct i915_page_table *pt;
678d96fb
BW		 1638 uint32_t pde, temp;
1639
1640 gen6_for_each_pde(pt, pd, start, length, temp, pde)
1641 gen6_write_pde(pd, pde, pt);
1642
1643 /* Make sure write is complete before other code can use this page
1644 * table. Also require for WC mapped PTEs */
1645 readl(dev_priv->gtt.gsm);
3e302542
BW		 1646}
1647
b4a74e3a 1648static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
3e302542 1649{
44159ddb 1650 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
b4a74e3a 1651
44159ddb 1652 return (ppgtt->pd.base.ggtt_offset / 64) << 16;
b4a74e3a
BW		 1653}
1654
90252e5c 1655static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
e85b26dc 1656 struct drm_i915_gem_request *req)
90252e5c 1657{
e85b26dc 1658 struct intel_engine_cs *ring = req->ring;
90252e5c
BW		 1659 int ret;
1660
90252e5c 1661 /* NB: TLBs must be flushed and invalidated before a switch */
a84c3ae1 1662 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
90252e5c
BW		 1663 if (ret)
1664 return ret;
1665
5fb9de1a 1666 ret = intel_ring_begin(req, 6);
90252e5c
BW		 1667 if (ret)
1668 return ret;
1669
1670 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
f92a9162 1671 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring));
90252e5c 1672 intel_ring_emit(ring, PP_DIR_DCLV_2G);
f92a9162 1673 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring));
90252e5c
BW		 1674 intel_ring_emit(ring, get_pd_offset(ppgtt));
1675 intel_ring_emit(ring, MI_NOOP);
1676 intel_ring_advance(ring);
1677
1678 return 0;
1679}
1680
71ba2d64 1681static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
e85b26dc 1682 struct drm_i915_gem_request *req)
71ba2d64 1683{
e85b26dc 1684 struct intel_engine_cs *ring = req->ring;
71ba2d64
YZ		 1685 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1686
1687 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1688 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1689 return 0;
1690}
1691
48a10389 1692static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
e85b26dc 1693 struct drm_i915_gem_request *req)
48a10389 1694{
e85b26dc 1695 struct intel_engine_cs *ring = req->ring;
48a10389
BW		 1696 int ret;
1697
48a10389 1698 /* NB: TLBs must be flushed and invalidated before a switch */
a84c3ae1 1699 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
48a10389
BW		 1700 if (ret)
1701 return ret;
1702
5fb9de1a 1703 ret = intel_ring_begin(req, 6);
48a10389
BW		 1704 if (ret)
1705 return ret;
1706
1707 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
f92a9162 1708 intel_ring_emit_reg(ring, RING_PP_DIR_DCLV(ring));
48a10389 1709 intel_ring_emit(ring, PP_DIR_DCLV_2G);
f92a9162 1710 intel_ring_emit_reg(ring, RING_PP_DIR_BASE(ring));
48a10389
BW		 1711 intel_ring_emit(ring, get_pd_offset(ppgtt));
1712 intel_ring_emit(ring, MI_NOOP);
1713 intel_ring_advance(ring);
1714
90252e5c
BW		 1715 /* XXX: RCS is the only one to auto invalidate the TLBs? */
1716 if (ring->id != RCS) {
a84c3ae1 1717 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
90252e5c
BW		 1718 if (ret)
1719 return ret;
1720 }
1721
48a10389
BW		 1722 return 0;
1723}
1724
eeb9488e 1725static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
e85b26dc 1726 struct drm_i915_gem_request *req)
eeb9488e 1727{
e85b26dc 1728 struct intel_engine_cs *ring = req->ring;
eeb9488e
BW		 1729 struct drm_device *dev = ppgtt->base.dev;
1730 struct drm_i915_private *dev_priv = dev->dev_private;
1731
48a10389 1732
eeb9488e
BW		 1733 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1734 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1735
1736 POSTING_READ(RING_PP_DIR_DCLV(ring));
1737
1738 return 0;
1739}
1740
82460d97 1741static void gen8_ppgtt_enable(struct drm_device *dev)
eeb9488e 1742{
eeb9488e 1743 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1744 struct intel_engine_cs *ring;
82460d97 1745 int j;
3e302542 1746
eeb9488e 1747 for_each_ring(ring, dev_priv, j) {
2dba3239 1748 u32 four_level = USES_FULL_48BIT_PPGTT(dev) ? GEN8_GFX_PPGTT_48B : 0;
eeb9488e 1749 I915_WRITE(RING_MODE_GEN7(ring),
2dba3239 1750 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE | four_level));
eeb9488e 1751 }
eeb9488e 1752}
6197349b 1753
82460d97 1754static void gen7_ppgtt_enable(struct drm_device *dev)
3e302542 1755{
50227e1c 1756 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 1757 struct intel_engine_cs *ring;
b4a74e3a 1758 uint32_t ecochk, ecobits;
3e302542 1759 int i;
6197349b 1760
b4a74e3a
BW		 1761 ecobits = I915_READ(GAC_ECO_BITS);
1762 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
a65c2fcd 1763
b4a74e3a
BW		 1764 ecochk = I915_READ(GAM_ECOCHK);
1765 if (IS_HASWELL(dev)) {
1766 ecochk |= ECOCHK_PPGTT_WB_HSW;
1767 } else {
1768 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1769 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1770 }
1771 I915_WRITE(GAM_ECOCHK, ecochk);
a65c2fcd 1772
b4a74e3a 1773 for_each_ring(ring, dev_priv, i) {
6197349b 1774 /* GFX_MODE is per-ring on gen7+ */
b4a74e3a
BW		 1775 I915_WRITE(RING_MODE_GEN7(ring),
1776 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
6197349b 1777 }
b4a74e3a 1778}
6197349b 1779
82460d97 1780static void gen6_ppgtt_enable(struct drm_device *dev)
b4a74e3a 1781{
50227e1c 1782 struct drm_i915_private *dev_priv = dev->dev_private;
b4a74e3a 1783 uint32_t ecochk, gab_ctl, ecobits;
a65c2fcd 1784
b4a74e3a
BW		 1785 ecobits = I915_READ(GAC_ECO_BITS);
1786 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1787 ECOBITS_PPGTT_CACHE64B);
6197349b 1788
b4a74e3a
BW		 1789 gab_ctl = I915_READ(GAB_CTL);
1790 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1791
1792 ecochk = I915_READ(GAM_ECOCHK);
1793 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1794
1795 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
6197349b
BW		 1796}
1797
1d2a314c 1798/* PPGTT support for Sandybdrige/Gen6 and later */
853ba5d2 1799static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 1800 uint64_t start,
1801 uint64_t length,
828c7908 1802 bool use_scratch)
1d2a314c 1803{
853ba5d2
BW		 1804 struct i915_hw_ppgtt *ppgtt =
1805 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 1806 gen6_pte_t *pt_vaddr, scratch_pte;
782f1495
BW		 1807 unsigned first_entry = start >> PAGE_SHIFT;
1808 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 1809 unsigned act_pt = first_entry / GEN6_PTES;
1810 unsigned first_pte = first_entry % GEN6_PTES;
7bddb01f 1811 unsigned last_pte, i;
1d2a314c 1812
c114f76a
MK		 1813 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1814 I915_CACHE_LLC, true, 0);
1d2a314c 1815
7bddb01f
DV		 1816 while (num_entries) {
1817 last_pte = first_pte + num_entries;
07749ef3
MT		 1818 if (last_pte > GEN6_PTES)
1819 last_pte = GEN6_PTES;
7bddb01f 1820
d1c54acd 1821 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1d2a314c 1822
7bddb01f
DV		 1823 for (i = first_pte; i < last_pte; i++)
1824 pt_vaddr[i] = scratch_pte;
1d2a314c 1825
d1c54acd 1826 kunmap_px(ppgtt, pt_vaddr);
1d2a314c 1827
7bddb01f
DV		 1828 num_entries -= last_pte - first_pte;
1829 first_pte = 0;
a15326a5 1830 act_pt++;
7bddb01f 1831 }
1d2a314c
DV		 1832}
1833
853ba5d2 1834static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
def886c3 1835 struct sg_table *pages,
782f1495 1836 uint64_t start,
24f3a8cf 1837 enum i915_cache_level cache_level, u32 flags)
def886c3 1838{
853ba5d2
BW		 1839 struct i915_hw_ppgtt *ppgtt =
1840 container_of(vm, struct i915_hw_ppgtt, base);
07749ef3 1841 gen6_pte_t *pt_vaddr;
782f1495 1842 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 1843 unsigned act_pt = first_entry / GEN6_PTES;
1844 unsigned act_pte = first_entry % GEN6_PTES;
6e995e23
ID		 1845 struct sg_page_iter sg_iter;
1846
cc79714f 1847 pt_vaddr = NULL;
6e995e23 1848 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
cc79714f 1849 if (pt_vaddr == NULL)
d1c54acd 1850 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
6e995e23 1851
cc79714f
CW		 1852 pt_vaddr[act_pte] =
1853 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
24f3a8cf
AG		 1854 cache_level, true, flags);
1855
07749ef3 1856 if (++act_pte == GEN6_PTES) {
d1c54acd 1857 kunmap_px(ppgtt, pt_vaddr);
cc79714f 1858 pt_vaddr = NULL;
a15326a5 1859 act_pt++;
6e995e23 1860 act_pte = 0;
def886c3 1861 }
def886c3 1862 }
cc79714f 1863 if (pt_vaddr)
d1c54acd 1864 kunmap_px(ppgtt, pt_vaddr);
def886c3
DV		 1865}
1866
678d96fb 1867static int gen6_alloc_va_range(struct i915_address_space *vm,
a05d80ee 1868 uint64_t start_in, uint64_t length_in)
678d96fb 1869{
4933d519
MT		 1870 DECLARE_BITMAP(new_page_tables, I915_PDES);
1871 struct drm_device *dev = vm->dev;
1872 struct drm_i915_private *dev_priv = dev->dev_private;
678d96fb
BW		 1873 struct i915_hw_ppgtt *ppgtt =
1874 container_of(vm, struct i915_hw_ppgtt, base);
ec565b3c 1875 struct i915_page_table *pt;
a05d80ee 1876 uint32_t start, length, start_save, length_save;
678d96fb 1877 uint32_t pde, temp;
4933d519
MT		 1878 int ret;
1879
a05d80ee
MK		 1880 if (WARN_ON(start_in + length_in > ppgtt->base.total))
1881 return -ENODEV;
1882
1883 start = start_save = start_in;
1884 length = length_save = length_in;
4933d519
MT		 1885
1886 bitmap_zero(new_page_tables, I915_PDES);
1887
1888 /* The allocation is done in two stages so that we can bail out with
1889 * minimal amount of pain. The first stage finds new page tables that
1890 * need allocation. The second stage marks use ptes within the page
1891 * tables.
1892 */
1893 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
79ab9370 1894 if (pt != vm->scratch_pt) {
4933d519
MT		 1895 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1896 continue;
1897 }
1898
1899 /* We've already allocated a page table */
1900 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1901
8a1ebd74 1902 pt = alloc_pt(dev);
4933d519
MT		 1903 if (IS_ERR(pt)) {
1904 ret = PTR_ERR(pt);
1905 goto unwind_out;
1906 }
1907
1908 gen6_initialize_pt(vm, pt);
1909
1910 ppgtt->pd.page_table[pde] = pt;
966082c9 1911 __set_bit(pde, new_page_tables);
72744cb1 1912 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
4933d519
MT		 1913 }
1914
1915 start = start_save;
1916 length = length_save;
678d96fb
BW		 1917
1918 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1919 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1920
1921 bitmap_zero(tmp_bitmap, GEN6_PTES);
1922 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1923 gen6_pte_count(start, length));
1924
966082c9 1925 if (__test_and_clear_bit(pde, new_page_tables))
4933d519
MT		 1926 gen6_write_pde(&ppgtt->pd, pde, pt);
1927
72744cb1
MT		 1928 trace_i915_page_table_entry_map(vm, pde, pt,
1929 gen6_pte_index(start),
1930 gen6_pte_count(start, length),
1931 GEN6_PTES);
4933d519 1932 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
678d96fb
BW		 1933 GEN6_PTES);
1934 }
1935
4933d519
MT		 1936 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1937
1938 /* Make sure write is complete before other code can use this page
1939 * table. Also require for WC mapped PTEs */
1940 readl(dev_priv->gtt.gsm);
1941
563222a7 1942 mark_tlbs_dirty(ppgtt);
678d96fb 1943 return 0;
4933d519
MT		 1944
1945unwind_out:
1946 for_each_set_bit(pde, new_page_tables, I915_PDES) {
ec565b3c 1947 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
4933d519 1948
79ab9370 1949 ppgtt->pd.page_table[pde] = vm->scratch_pt;
a08e111a 1950 free_pt(vm->dev, pt);
4933d519
MT		 1951 }
1952
1953 mark_tlbs_dirty(ppgtt);
1954 return ret;
678d96fb
BW		 1955}
1956
8776f02b
MK		 1957static int gen6_init_scratch(struct i915_address_space *vm)
1958{
1959 struct drm_device *dev = vm->dev;
1960
1961 vm->scratch_page = alloc_scratch_page(dev);
1962 if (IS_ERR(vm->scratch_page))
1963 return PTR_ERR(vm->scratch_page);
1964
1965 vm->scratch_pt = alloc_pt(dev);
1966 if (IS_ERR(vm->scratch_pt)) {
1967 free_scratch_page(dev, vm->scratch_page);
1968 return PTR_ERR(vm->scratch_pt);
1969 }
1970
1971 gen6_initialize_pt(vm, vm->scratch_pt);
1972
1973 return 0;
1974}
1975
1976static void gen6_free_scratch(struct i915_address_space *vm)
1977{
1978 struct drm_device *dev = vm->dev;
1979
1980 free_pt(dev, vm->scratch_pt);
1981 free_scratch_page(dev, vm->scratch_page);
1982}
1983
061dd493 1984static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
a00d825d 1985{
061dd493
DV		 1986 struct i915_hw_ppgtt *ppgtt =
1987 container_of(vm, struct i915_hw_ppgtt, base);
09942c65
MT		 1988 struct i915_page_table *pt;
1989 uint32_t pde;
4933d519 1990
061dd493
DV		 1991 drm_mm_remove_node(&ppgtt->node);
1992
09942c65 1993 gen6_for_all_pdes(pt, ppgtt, pde) {
79ab9370 1994 if (pt != vm->scratch_pt)
a08e111a 1995 free_pt(ppgtt->base.dev, pt);
4933d519 1996 }
06fda602 1997
8776f02b 1998 gen6_free_scratch(vm);
3440d265
DV		 1999}
2000
b146520f 2001static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
3440d265 2002{
8776f02b 2003 struct i915_address_space *vm = &ppgtt->base;
853ba5d2 2004 struct drm_device *dev = ppgtt->base.dev;
1d2a314c 2005 struct drm_i915_private *dev_priv = dev->dev_private;
e3cc1995 2006 bool retried = false;
b146520f 2007 int ret;
1d2a314c 2008
c8d4c0d6
BW		 2009 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
2010 * allocator works in address space sizes, so it's multiplied by page
2011 * size. We allocate at the top of the GTT to avoid fragmentation.
2012 */
2013 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
4933d519 2014
8776f02b
MK		 2015 ret = gen6_init_scratch(vm);
2016 if (ret)
2017 return ret;
4933d519 2018
e3cc1995 2019alloc:
c8d4c0d6
BW		 2020 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
2021 &ppgtt->node, GEN6_PD_SIZE,
2022 GEN6_PD_ALIGN, 0,
2023 0, dev_priv->gtt.base.total,
3e8b5ae9 2024 DRM_MM_TOPDOWN);
e3cc1995
BW		 2025 if (ret == -ENOSPC && !retried) {
2026 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
2027 GEN6_PD_SIZE, GEN6_PD_ALIGN,
d23db88c
CW		 2028 I915_CACHE_NONE,
2029 0, dev_priv->gtt.base.total,
2030 0);
e3cc1995 2031 if (ret)
678d96fb 2032 goto err_out;
e3cc1995
BW		 2033
2034 retried = true;
2035 goto alloc;
2036 }
c8d4c0d6 2037
c8c26622 2038 if (ret)
678d96fb
BW		 2039 goto err_out;
2040
c8c26622 2041
c8d4c0d6
BW		 2042 if (ppgtt->node.start < dev_priv->gtt.mappable_end)
2043 DRM_DEBUG("Forced to use aperture for PDEs\n");
1d2a314c 2044
c8c26622 2045 return 0;
678d96fb
BW		 2046
2047err_out:
8776f02b 2048 gen6_free_scratch(vm);
678d96fb 2049 return ret;
b146520f
BW		 2050}
2051
b146520f
BW		 2052static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
2053{
2f2cf682 2054 return gen6_ppgtt_allocate_page_directories(ppgtt);
4933d519 2055}
06dc68d6 2056
4933d519
MT		 2057static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
2058 uint64_t start, uint64_t length)
2059{
ec565b3c 2060 struct i915_page_table *unused;
4933d519 2061 uint32_t pde, temp;
1d2a314c 2062
4933d519 2063 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
79ab9370 2064 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
b146520f
BW		 2065}
2066
5c5f6457 2067static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
b146520f
BW		 2068{
2069 struct drm_device *dev = ppgtt->base.dev;
2070 struct drm_i915_private *dev_priv = dev->dev_private;
2071 int ret;
2072
2073 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
2074 if (IS_GEN6(dev)) {
b146520f
BW		 2075 ppgtt->switch_mm = gen6_mm_switch;
2076 } else if (IS_HASWELL(dev)) {
b146520f
BW		 2077 ppgtt->switch_mm = hsw_mm_switch;
2078 } else if (IS_GEN7(dev)) {
b146520f
BW		 2079 ppgtt->switch_mm = gen7_mm_switch;
2080 } else
2081 BUG();
2082
71ba2d64
YZ		 2083 if (intel_vgpu_active(dev))
2084 ppgtt->switch_mm = vgpu_mm_switch;
2085
b146520f
BW		 2086 ret = gen6_ppgtt_alloc(ppgtt);
2087 if (ret)
2088 return ret;
2089
5c5f6457 2090 ppgtt->base.allocate_va_range = gen6_alloc_va_range;
b146520f
BW		 2091 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
2092 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
777dc5bb
DV		 2093 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
2094 ppgtt->base.bind_vma = ppgtt_bind_vma;
b146520f 2095 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
b146520f 2096 ppgtt->base.start = 0;
09942c65 2097 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
87d60b63 2098 ppgtt->debug_dump = gen6_dump_ppgtt;
1d2a314c 2099
44159ddb 2100 ppgtt->pd.base.ggtt_offset =
07749ef3 2101 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
1d2a314c 2102
678d96fb 2103 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
44159ddb 2104 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
678d96fb 2105
5c5f6457 2106 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
1d2a314c 2107
678d96fb
BW		 2108 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
2109
440fd528 2110 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
b146520f
BW		 2111 ppgtt->node.size >> 20,
2112 ppgtt->node.start / PAGE_SIZE);
3440d265 2113
fa76da34 2114 DRM_DEBUG("Adding PPGTT at offset %x\n",
44159ddb 2115 ppgtt->pd.base.ggtt_offset << 10);
fa76da34 2116
b146520f 2117 return 0;
3440d265
DV		 2118}
2119
5c5f6457 2120static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
3440d265 2121{
853ba5d2 2122 ppgtt->base.dev = dev;
3440d265 2123
3ed124b2 2124 if (INTEL_INFO(dev)->gen < 8)
5c5f6457 2125 return gen6_ppgtt_init(ppgtt);
3ed124b2 2126 else
d7b2633d 2127 return gen8_ppgtt_init(ppgtt);
fa76da34 2128}
c114f76a 2129
a2cad9df
MW		 2130static void i915_address_space_init(struct i915_address_space *vm,
2131 struct drm_i915_private *dev_priv)
2132{
2133 drm_mm_init(&vm->mm, vm->start, vm->total);
2134 vm->dev = dev_priv->dev;
2135 INIT_LIST_HEAD(&vm->active_list);
2136 INIT_LIST_HEAD(&vm->inactive_list);
2137 list_add_tail(&vm->global_link, &dev_priv->vm_list);
2138}
2139
fa76da34
DV		 2140int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
2141{
2142 struct drm_i915_private *dev_priv = dev->dev_private;
2143 int ret = 0;
3ed124b2 2144
5c5f6457 2145 ret = __hw_ppgtt_init(dev, ppgtt);
fa76da34 2146 if (ret == 0) {
c7c48dfd 2147 kref_init(&ppgtt->ref);
a2cad9df 2148 i915_address_space_init(&ppgtt->base, dev_priv);
93bd8649 2149 }
1d2a314c
DV		 2150
2151 return ret;
2152}
2153
82460d97
DV		 2154int i915_ppgtt_init_hw(struct drm_device *dev)
2155{
671b5013
TD		 2156 /* In the case of execlists, PPGTT is enabled by the context descriptor
2157 * and the PDPs are contained within the context itself. We don't
2158 * need to do anything here. */
2159 if (i915.enable_execlists)
2160 return 0;
2161
82460d97
DV		 2162 if (!USES_PPGTT(dev))
2163 return 0;
2164
2165 if (IS_GEN6(dev))
2166 gen6_ppgtt_enable(dev);
2167 else if (IS_GEN7(dev))
2168 gen7_ppgtt_enable(dev);
2169 else if (INTEL_INFO(dev)->gen >= 8)
2170 gen8_ppgtt_enable(dev);
2171 else
5f77eeb0 2172 MISSING_CASE(INTEL_INFO(dev)->gen);
82460d97 2173
4ad2fd88
JH		 2174 return 0;
2175}
1d2a314c 2176
b3dd6b96 2177int i915_ppgtt_init_ring(struct drm_i915_gem_request *req)
4ad2fd88 2178{
b3dd6b96 2179 struct drm_i915_private *dev_priv = req->ring->dev->dev_private;
4ad2fd88
JH		 2180 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2181
2182 if (i915.enable_execlists)
2183 return 0;
2184
2185 if (!ppgtt)
2186 return 0;
2187
e85b26dc 2188 return ppgtt->switch_mm(ppgtt, req);
1d2a314c 2189}
4ad2fd88 2190
4d884705
DV		 2191struct i915_hw_ppgtt *
2192i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
2193{
2194 struct i915_hw_ppgtt *ppgtt;
2195 int ret;
2196
2197 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2198 if (!ppgtt)
2199 return ERR_PTR(-ENOMEM);
2200
2201 ret = i915_ppgtt_init(dev, ppgtt);
2202 if (ret) {
2203 kfree(ppgtt);
2204 return ERR_PTR(ret);
2205 }
2206
2207 ppgtt->file_priv = fpriv;
2208
198c974d
DCS		 2209 trace_i915_ppgtt_create(&ppgtt->base);
2210
4d884705
DV		 2211 return ppgtt;
2212}
2213
ee960be7
DV		 2214void i915_ppgtt_release(struct kref *kref)
2215{
2216 struct i915_hw_ppgtt *ppgtt =
2217 container_of(kref, struct i915_hw_ppgtt, ref);
2218
198c974d
DCS		 2219 trace_i915_ppgtt_release(&ppgtt->base);
2220
ee960be7
DV		 2221 /* vmas should already be unbound */
2222 WARN_ON(!list_empty(&ppgtt->base.active_list));
2223 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
2224
19dd120c
DV		 2225 list_del(&ppgtt->base.global_link);
2226 drm_mm_takedown(&ppgtt->base.mm);
2227
ee960be7
DV		 2228 ppgtt->base.cleanup(&ppgtt->base);
2229 kfree(ppgtt);
2230}
1d2a314c 2231
a81cc00c
BW		 2232extern int intel_iommu_gfx_mapped;
2233/* Certain Gen5 chipsets require require idling the GPU before
2234 * unmapping anything from the GTT when VT-d is enabled.
2235 */
2c642b07 2236static bool needs_idle_maps(struct drm_device *dev)
a81cc00c
BW		 2237{
2238#ifdef CONFIG_INTEL_IOMMU
2239 /* Query intel_iommu to see if we need the workaround. Presumably that
2240 * was loaded first.
2241 */
2242 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
2243 return true;
2244#endif
2245 return false;
2246}
2247
5c042287
BW		 2248static bool do_idling(struct drm_i915_private *dev_priv)
2249{
2250 bool ret = dev_priv->mm.interruptible;
2251
a81cc00c 2252 if (unlikely(dev_priv->gtt.do_idle_maps)) {
5c042287 2253 dev_priv->mm.interruptible = false;
b2da9fe5 2254 if (i915_gpu_idle(dev_priv->dev)) {
5c042287
BW		 2255 DRM_ERROR("Couldn't idle GPU\n");
2256 /* Wait a bit, in hopes it avoids the hang */
2257 udelay(10);
2258 }
2259 }
2260
2261 return ret;
2262}
2263
2264static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
2265{
a81cc00c 2266 if (unlikely(dev_priv->gtt.do_idle_maps))
5c042287
BW		 2267 dev_priv->mm.interruptible = interruptible;
2268}
2269
828c7908
BW		 2270void i915_check_and_clear_faults(struct drm_device *dev)
2271{
2272 struct drm_i915_private *dev_priv = dev->dev_private;
a4872ba6 2273 struct intel_engine_cs *ring;
828c7908
BW		 2274 int i;
2275
2276 if (INTEL_INFO(dev)->gen < 6)
2277 return;
2278
2279 for_each_ring(ring, dev_priv, i) {
2280 u32 fault_reg;
2281 fault_reg = I915_READ(RING_FAULT_REG(ring));
2282 if (fault_reg & RING_FAULT_VALID) {
2283 DRM_DEBUG_DRIVER("Unexpected fault\n"
59a5d290 2284 "\tAddr: 0x%08lx\n"
828c7908
BW		 2285 "\tAddress space: %s\n"
2286 "\tSource ID: %d\n"
2287 "\tType: %d\n",
2288 fault_reg & PAGE_MASK,
2289 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
2290 RING_FAULT_SRCID(fault_reg),
2291 RING_FAULT_FAULT_TYPE(fault_reg));
2292 I915_WRITE(RING_FAULT_REG(ring),
2293 fault_reg & ~RING_FAULT_VALID);
2294 }
2295 }
2296 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
2297}
2298
91e56499
CW		 2299static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
2300{
2301 if (INTEL_INFO(dev_priv->dev)->gen < 6) {
2302 intel_gtt_chipset_flush();
2303 } else {
2304 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2305 POSTING_READ(GFX_FLSH_CNTL_GEN6);
2306 }
2307}
2308
828c7908
BW		 2309void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
2310{
2311 struct drm_i915_private *dev_priv = dev->dev_private;
2312
2313 /* Don't bother messing with faults pre GEN6 as we have little
2314 * documentation supporting that it's a good idea.
2315 */
2316 if (INTEL_INFO(dev)->gen < 6)
2317 return;
2318
2319 i915_check_and_clear_faults(dev);
2320
2321 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
782f1495
BW		 2322 dev_priv->gtt.base.start,
2323 dev_priv->gtt.base.total,
e568af1c 2324 true);
91e56499
CW		 2325
2326 i915_ggtt_flush(dev_priv);
828c7908
BW		 2327}
2328
74163907 2329int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
7c2e6fdf 2330{
9da3da66
CW		 2331 if (!dma_map_sg(&obj->base.dev->pdev->dev,
2332 obj->pages->sgl, obj->pages->nents,
2333 PCI_DMA_BIDIRECTIONAL))
2334 return -ENOSPC;
2335
2336 return 0;
7c2e6fdf
DV		 2337}
2338
2c642b07 2339static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
94ec8f61
BW		 2340{
2341#ifdef writeq
2342 writeq(pte, addr);
2343#else
2344 iowrite32((u32)pte, addr);
2345 iowrite32(pte >> 32, addr + 4);
2346#endif
2347}
2348
2349static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2350 struct sg_table *st,
782f1495 2351 uint64_t start,
24f3a8cf 2352 enum i915_cache_level level, u32 unused)
94ec8f61
BW		 2353{
2354 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495 2355 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 2356 gen8_pte_t __iomem *gtt_entries =
2357 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
94ec8f61
BW		 2358 int i = 0;
2359 struct sg_page_iter sg_iter;
57007df7 2360 dma_addr_t addr = 0; /* shut up gcc */
94ec8f61
BW		 2361
2362 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2363 addr = sg_dma_address(sg_iter.sg) +
2364 (sg_iter.sg_pgoffset << PAGE_SHIFT);
2365 gen8_set_pte(&gtt_entries[i],
2366 gen8_pte_encode(addr, level, true));
2367 i++;
2368 }
2369
2370 /*
2371 * XXX: This serves as a posting read to make sure that the PTE has
2372 * actually been updated. There is some concern that even though
2373 * registers and PTEs are within the same BAR that they are potentially
2374 * of NUMA access patterns. Therefore, even with the way we assume
2375 * hardware should work, we must keep this posting read for paranoia.
2376 */
2377 if (i != 0)
2378 WARN_ON(readq(&gtt_entries[i-1])
2379 != gen8_pte_encode(addr, level, true));
2380
94ec8f61
BW		 2381 /* This next bit makes the above posting read even more important. We
2382 * want to flush the TLBs only after we're certain all the PTE updates
2383 * have finished.
2384 */
2385 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2386 POSTING_READ(GFX_FLSH_CNTL_GEN6);
94ec8f61
BW		 2387}
2388
e76e9aeb
BW		 2389/*
2390 * Binds an object into the global gtt with the specified cache level. The object
2391 * will be accessible to the GPU via commands whose operands reference offsets
2392 * within the global GTT as well as accessible by the GPU through the GMADR
2393 * mapped BAR (dev_priv->mm.gtt->gtt).
2394 */
853ba5d2 2395static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
7faf1ab2 2396 struct sg_table *st,
782f1495 2397 uint64_t start,
24f3a8cf 2398 enum i915_cache_level level, u32 flags)
e76e9aeb 2399{
853ba5d2 2400 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495 2401 unsigned first_entry = start >> PAGE_SHIFT;
07749ef3
MT		 2402 gen6_pte_t __iomem *gtt_entries =
2403 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
6e995e23
ID		 2404 int i = 0;
2405 struct sg_page_iter sg_iter;
57007df7 2406 dma_addr_t addr = 0;
e76e9aeb 2407
6e995e23 2408 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
2db76d7c 2409 addr = sg_page_iter_dma_address(&sg_iter);
24f3a8cf 2410 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
6e995e23 2411 i++;
e76e9aeb
BW		 2412 }
2413
e76e9aeb
BW		 2414 /* XXX: This serves as a posting read to make sure that the PTE has
2415 * actually been updated. There is some concern that even though
2416 * registers and PTEs are within the same BAR that they are potentially
2417 * of NUMA access patterns. Therefore, even with the way we assume
2418 * hardware should work, we must keep this posting read for paranoia.
2419 */
57007df7
PM		 2420 if (i != 0) {
2421 unsigned long gtt = readl(&gtt_entries[i-1]);
2422 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
2423 }
0f9b91c7
BW		 2424
2425 /* This next bit makes the above posting read even more important. We
2426 * want to flush the TLBs only after we're certain all the PTE updates
2427 * have finished.
2428 */
2429 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
2430 POSTING_READ(GFX_FLSH_CNTL_GEN6);
e76e9aeb
BW		 2431}
2432
94ec8f61 2433static void gen8_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2434 uint64_t start,
2435 uint64_t length,
94ec8f61
BW		 2436 bool use_scratch)
2437{
2438 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495
BW		 2439 unsigned first_entry = start >> PAGE_SHIFT;
2440 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 2441 gen8_pte_t scratch_pte, __iomem *gtt_base =
2442 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
94ec8f61
BW		 2443 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
2444 int i;
2445
2446 if (WARN(num_entries > max_entries,
2447 "First entry = %d; Num entries = %d (max=%d)\n",
2448 first_entry, num_entries, max_entries))
2449 num_entries = max_entries;
2450
c114f76a 2451 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
94ec8f61
BW		 2452 I915_CACHE_LLC,
2453 use_scratch);
2454 for (i = 0; i < num_entries; i++)
2455 gen8_set_pte(&gtt_base[i], scratch_pte);
2456 readl(gtt_base);
2457}
2458
853ba5d2 2459static void gen6_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2460 uint64_t start,
2461 uint64_t length,
828c7908 2462 bool use_scratch)
7faf1ab2 2463{
853ba5d2 2464 struct drm_i915_private *dev_priv = vm->dev->dev_private;
782f1495
BW		 2465 unsigned first_entry = start >> PAGE_SHIFT;
2466 unsigned num_entries = length >> PAGE_SHIFT;
07749ef3
MT		 2467 gen6_pte_t scratch_pte, __iomem *gtt_base =
2468 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
a54c0c27 2469 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
7faf1ab2
DV		 2470 int i;
2471
2472 if (WARN(num_entries > max_entries,
2473 "First entry = %d; Num entries = %d (max=%d)\n",
2474 first_entry, num_entries, max_entries))
2475 num_entries = max_entries;
2476
c114f76a
MK		 2477 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
2478 I915_CACHE_LLC, use_scratch, 0);
828c7908 2479
7faf1ab2
DV		 2480 for (i = 0; i < num_entries; i++)
2481 iowrite32(scratch_pte, &gtt_base[i]);
2482 readl(gtt_base);
2483}
2484
d369d2d9
DV		 2485static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2486 struct sg_table *pages,
2487 uint64_t start,
2488 enum i915_cache_level cache_level, u32 unused)
7faf1ab2
DV		 2489{
2490 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
2491 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
2492
d369d2d9 2493 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
0875546c 2494
7faf1ab2
DV		 2495}
2496
853ba5d2 2497static void i915_ggtt_clear_range(struct i915_address_space *vm,
782f1495
BW		 2498 uint64_t start,
2499 uint64_t length,
828c7908 2500 bool unused)
7faf1ab2 2501{
782f1495
BW		 2502 unsigned first_entry = start >> PAGE_SHIFT;
2503 unsigned num_entries = length >> PAGE_SHIFT;
7faf1ab2
DV		 2504 intel_gtt_clear_range(first_entry, num_entries);
2505}
2506
70b9f6f8
DV		 2507static int ggtt_bind_vma(struct i915_vma *vma,
2508 enum i915_cache_level cache_level,
2509 u32 flags)
0a878716
DV		 2510{
2511 struct drm_i915_gem_object *obj = vma->obj;
2512 u32 pte_flags = 0;
2513 int ret;
2514
2515 ret = i915_get_ggtt_vma_pages(vma);
2516 if (ret)
2517 return ret;
2518
2519 /* Currently applicable only to VLV */
2520 if (obj->gt_ro)
2521 pte_flags |= PTE_READ_ONLY;
2522
2523 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
2524 vma->node.start,
2525 cache_level, pte_flags);
2526
2527 /*
2528 * Without aliasing PPGTT there's no difference between
2529 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
2530 * upgrade to both bound if we bind either to avoid double-binding.
2531 */
2532 vma->bound |= GLOBAL_BIND | LOCAL_BIND;
2533
2534 return 0;
2535}
2536
5bab6f60
CW		 2537struct ggtt_bind_vma__cb {
2538 struct i915_vma *vma;
2539 enum i915_cache_level cache_level;
2540 u32 flags;
2541};
2542
2543static int ggtt_bind_vma__cb(void *_arg)
2544{
2545 struct ggtt_bind_vma__cb *arg = _arg;
2546 return ggtt_bind_vma(arg->vma, arg->cache_level, arg->flags);
2547}
2548
2549static int ggtt_bind_vma__BKL(struct i915_vma *vma,
2550 enum i915_cache_level cache_level,
2551 u32 flags)
2552{
2553 struct ggtt_bind_vma__cb arg = { vma, cache_level, flags };
2554 return stop_machine(ggtt_bind_vma__cb, &arg, NULL);
2555}
2556
0a878716
DV		 2557static int aliasing_gtt_bind_vma(struct i915_vma *vma,
2558 enum i915_cache_level cache_level,
2559 u32 flags)
d5bd1449 2560{
6f65e29a 2561 struct drm_device *dev = vma->vm->dev;
7faf1ab2 2562 struct drm_i915_private *dev_priv = dev->dev_private;
6f65e29a 2563 struct drm_i915_gem_object *obj = vma->obj;
ec7adb6e 2564 struct sg_table *pages = obj->pages;
f329f5f6 2565 u32 pte_flags = 0;
70b9f6f8
DV		 2566 int ret;
2567
2568 ret = i915_get_ggtt_vma_pages(vma);
2569 if (ret)
2570 return ret;
2571 pages = vma->ggtt_view.pages;
7faf1ab2 2572
24f3a8cf
AG		 2573 /* Currently applicable only to VLV */
2574 if (obj->gt_ro)
f329f5f6 2575 pte_flags |= PTE_READ_ONLY;
24f3a8cf 2576
ec7adb6e 2577
0a878716 2578 if (flags & GLOBAL_BIND) {
0875546c
DV		 2579 vma->vm->insert_entries(vma->vm, pages,
2580 vma->node.start,
2581 cache_level, pte_flags);
6f65e29a 2582 }
d5bd1449 2583
0a878716 2584 if (flags & LOCAL_BIND) {
6f65e29a 2585 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
ec7adb6e 2586 appgtt->base.insert_entries(&appgtt->base, pages,
782f1495 2587 vma->node.start,
f329f5f6 2588 cache_level, pte_flags);
6f65e29a 2589 }
70b9f6f8
DV		 2590
2591 return 0;
d5bd1449
CW		 2592}
2593
6f65e29a 2594static void ggtt_unbind_vma(struct i915_vma *vma)
74163907 2595{
6f65e29a 2596 struct drm_device *dev = vma->vm->dev;
7faf1ab2 2597 struct drm_i915_private *dev_priv = dev->dev_private;
6f65e29a 2598 struct drm_i915_gem_object *obj = vma->obj;
06615ee5
JL		 2599 const uint64_t size = min_t(uint64_t,
2600 obj->base.size,
2601 vma->node.size);
6f65e29a 2602
aff43766 2603 if (vma->bound & GLOBAL_BIND) {
782f1495
BW		 2604 vma->vm->clear_range(vma->vm,
2605 vma->node.start,
06615ee5 2606 size,
6f65e29a 2607 true);
6f65e29a 2608 }
74898d7e 2609
0875546c 2610 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {
6f65e29a 2611 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
06615ee5 2612
6f65e29a 2613 appgtt->base.clear_range(&appgtt->base,
782f1495 2614 vma->node.start,
06615ee5 2615 size,
6f65e29a 2616 true);
6f65e29a 2617 }
74163907
DV		 2618}
2619
2620void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
7c2e6fdf 2621{
5c042287
BW		 2622 struct drm_device *dev = obj->base.dev;
2623 struct drm_i915_private *dev_priv = dev->dev_private;
2624 bool interruptible;
2625
2626 interruptible = do_idling(dev_priv);
2627
5ec5b516
ID		 2628 dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents,
2629 PCI_DMA_BIDIRECTIONAL);
5c042287
BW		 2630
2631 undo_idling(dev_priv, interruptible);
7c2e6fdf 2632}
644ec02b 2633
42d6ab48
CW		 2634static void i915_gtt_color_adjust(struct drm_mm_node *node,
2635 unsigned long color,
440fd528
TR		 2636 u64 *start,
2637 u64 *end)
42d6ab48
CW		 2638{
2639 if (node->color != color)
2640 *start += 4096;
2641
2642 if (!list_empty(&node->node_list)) {
2643 node = list_entry(node->node_list.next,
2644 struct drm_mm_node,
2645 node_list);
2646 if (node->allocated && node->color != color)
2647 *end -= 4096;
2648 }
2649}
fbe5d36e 2650
f548c0e9 2651static int i915_gem_setup_global_gtt(struct drm_device *dev,
088e0df4
MT		 2652 u64 start,
2653 u64 mappable_end,
2654 u64 end)
644ec02b 2655{
e78891ca
BW		 2656 /* Let GEM Manage all of the aperture.
2657 *
2658 * However, leave one page at the end still bound to the scratch page.
2659 * There are a number of places where the hardware apparently prefetches
2660 * past the end of the object, and we've seen multiple hangs with the
2661 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2662 * aperture. One page should be enough to keep any prefetching inside
2663 * of the aperture.
2664 */
40d74980
BW		 2665 struct drm_i915_private *dev_priv = dev->dev_private;
2666 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
ed2f3452
CW		 2667 struct drm_mm_node *entry;
2668 struct drm_i915_gem_object *obj;
2669 unsigned long hole_start, hole_end;
fa76da34 2670 int ret;
644ec02b 2671
35451cb6
BW		 2672 BUG_ON(mappable_end > end);
2673
a2cad9df 2674 ggtt_vm->start = start;
5dda8fa3 2675
a2cad9df
MW		 2676 /* Subtract the guard page before address space initialization to
2677 * shrink the range used by drm_mm */
2678 ggtt_vm->total = end - start - PAGE_SIZE;
2679 i915_address_space_init(ggtt_vm, dev_priv);
2680 ggtt_vm->total += PAGE_SIZE;
5dda8fa3
YZ		 2681
2682 if (intel_vgpu_active(dev)) {
2683 ret = intel_vgt_balloon(dev);
2684 if (ret)
2685 return ret;
2686 }
2687
42d6ab48 2688 if (!HAS_LLC(dev))
a2cad9df 2689 ggtt_vm->mm.color_adjust = i915_gtt_color_adjust;
644ec02b 2690
ed2f3452 2691 /* Mark any preallocated objects as occupied */
35c20a60 2692 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
40d74980 2693 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
fa76da34 2694
088e0df4 2695 DRM_DEBUG_KMS("reserving preallocated space: %llx + %zx\n",
c6cfb325
BW		 2696 i915_gem_obj_ggtt_offset(obj), obj->base.size);
2697
2698 WARN_ON(i915_gem_obj_ggtt_bound(obj));
40d74980 2699 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
6c5566a8
DV		 2700 if (ret) {
2701 DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
2702 return ret;
2703 }
aff43766 2704 vma->bound |= GLOBAL_BIND;
7c4a7d60 2705 list_add_tail(&vma->mm_list, &ggtt_vm->inactive_list);
ed2f3452
CW		 2706 }
2707
ed2f3452 2708 /* Clear any non-preallocated blocks */
40d74980 2709 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
ed2f3452
CW		 2710 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2711 hole_start, hole_end);
782f1495
BW		 2712 ggtt_vm->clear_range(ggtt_vm, hole_start,
2713 hole_end - hole_start, true);
ed2f3452
CW		 2714 }
2715
2716 /* And finally clear the reserved guard page */
782f1495 2717 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
6c5566a8 2718
fa76da34
DV		 2719 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
2720 struct i915_hw_ppgtt *ppgtt;
2721
2722 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2723 if (!ppgtt)
2724 return -ENOMEM;
2725
5c5f6457
DV		 2726 ret = __hw_ppgtt_init(dev, ppgtt);
2727 if (ret) {
2728 ppgtt->base.cleanup(&ppgtt->base);
2729 kfree(ppgtt);
2730 return ret;
2731 }
2732
2733 if (ppgtt->base.allocate_va_range)
2734 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2735 ppgtt->base.total);
4933d519 2736 if (ret) {
061dd493 2737 ppgtt->base.cleanup(&ppgtt->base);
4933d519 2738 kfree(ppgtt);
fa76da34 2739 return ret;
4933d519 2740 }
fa76da34 2741
5c5f6457
DV		 2742 ppgtt->base.clear_range(&ppgtt->base,
2743 ppgtt->base.start,
2744 ppgtt->base.total,
2745 true);
2746
fa76da34 2747 dev_priv->mm.aliasing_ppgtt = ppgtt;
0a878716
DV		 2748 WARN_ON(dev_priv->gtt.base.bind_vma != ggtt_bind_vma);
2749 dev_priv->gtt.base.bind_vma = aliasing_gtt_bind_vma;
fa76da34
DV		 2750 }
2751
6c5566a8 2752 return 0;
e76e9aeb
BW		 2753}
2754
d7e5008f
BW		 2755void i915_gem_init_global_gtt(struct drm_device *dev)
2756{
2757 struct drm_i915_private *dev_priv = dev->dev_private;
c44ef60e 2758 u64 gtt_size, mappable_size;
d7e5008f 2759
853ba5d2 2760 gtt_size = dev_priv->gtt.base.total;
93d18799 2761 mappable_size = dev_priv->gtt.mappable_end;
d7e5008f 2762
e78891ca 2763 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
e76e9aeb
BW		 2764}
2765
90d0a0e8
DV		 2766void i915_global_gtt_cleanup(struct drm_device *dev)
2767{
2768 struct drm_i915_private *dev_priv = dev->dev_private;
2769 struct i915_address_space *vm = &dev_priv->gtt.base;
2770
70e32544
DV		 2771 if (dev_priv->mm.aliasing_ppgtt) {
2772 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2773
2774 ppgtt->base.cleanup(&ppgtt->base);
2775 }
2776
90d0a0e8 2777 if (drm_mm_initialized(&vm->mm)) {
5dda8fa3
YZ		 2778 if (intel_vgpu_active(dev))
2779 intel_vgt_deballoon();
2780
90d0a0e8
DV		 2781 drm_mm_takedown(&vm->mm);
2782 list_del(&vm->global_link);
2783 }
2784
2785 vm->cleanup(vm);
2786}
70e32544 2787
2c642b07 2788static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
e76e9aeb
BW		 2789{
2790 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2791 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2792 return snb_gmch_ctl << 20;
2793}
2794
2c642b07 2795static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
9459d252
BW		 2796{
2797 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2798 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2799 if (bdw_gmch_ctl)
2800 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
562d55d9
BW		 2801
2802#ifdef CONFIG_X86_32
2803 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2804 if (bdw_gmch_ctl > 4)
2805 bdw_gmch_ctl = 4;
2806#endif
2807
9459d252
BW		 2808 return bdw_gmch_ctl << 20;
2809}
2810
2c642b07 2811static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
d7f25f23
DL		 2812{
2813 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2814 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2815
2816 if (gmch_ctrl)
2817 return 1 << (20 + gmch_ctrl);
2818
2819 return 0;
2820}
2821
2c642b07 2822static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
e76e9aeb
BW		 2823{
2824 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2825 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2826 return snb_gmch_ctl << 25; /* 32 MB units */
2827}
2828
2c642b07 2829static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
9459d252
BW		 2830{
2831 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2832 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2833 return bdw_gmch_ctl << 25; /* 32 MB units */
2834}
2835
d7f25f23
DL		 2836static size_t chv_get_stolen_size(u16 gmch_ctrl)
2837{
2838 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2839 gmch_ctrl &= SNB_GMCH_GMS_MASK;
2840
2841 /*
2842 * 0x0 to 0x10: 32MB increments starting at 0MB
2843 * 0x11 to 0x16: 4MB increments starting at 8MB
2844 * 0x17 to 0x1d: 4MB increments start at 36MB
2845 */
2846 if (gmch_ctrl < 0x11)
2847 return gmch_ctrl << 25;
2848 else if (gmch_ctrl < 0x17)
2849 return (gmch_ctrl - 0x11 + 2) << 22;
2850 else
2851 return (gmch_ctrl - 0x17 + 9) << 22;
2852}
2853
66375014
DL		 2854static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2855{
2856 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2857 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2858
2859 if (gen9_gmch_ctl < 0xf0)
2860 return gen9_gmch_ctl << 25; /* 32 MB units */
2861 else
2862 /* 4MB increments starting at 0xf0 for 4MB */
2863 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2864}
2865
63340133
BW		 2866static int ggtt_probe_common(struct drm_device *dev,
2867 size_t gtt_size)
2868{
2869 struct drm_i915_private *dev_priv = dev->dev_private;
4ad2af1e 2870 struct i915_page_scratch *scratch_page;
21c34607 2871 phys_addr_t gtt_phys_addr;
63340133
BW		 2872
2873 /* For Modern GENs the PTEs and register space are split in the BAR */
21c34607 2874 gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
63340133
BW		 2875 (pci_resource_len(dev->pdev, 0) / 2);
2876
2a073f89
ID		 2877 /*
2878 * On BXT writes larger than 64 bit to the GTT pagetable range will be
2879 * dropped. For WC mappings in general we have 64 byte burst writes
2880 * when the WC buffer is flushed, so we can't use it, but have to
2881 * resort to an uncached mapping. The WC issue is easily caught by the
2882 * readback check when writing GTT PTE entries.
2883 */
2884 if (IS_BROXTON(dev))
2885 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size);
2886 else
2887 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
63340133
BW		 2888 if (!dev_priv->gtt.gsm) {
2889 DRM_ERROR("Failed to map the gtt page table\n");
2890 return -ENOMEM;
2891 }
2892
4ad2af1e
MK		 2893 scratch_page = alloc_scratch_page(dev);
2894 if (IS_ERR(scratch_page)) {
63340133
BW		 2895 DRM_ERROR("Scratch setup failed\n");
2896 /* iounmap will also get called at remove, but meh */
2897 iounmap(dev_priv->gtt.gsm);
4ad2af1e 2898 return PTR_ERR(scratch_page);
63340133
BW		 2899 }
2900
4ad2af1e
MK		 2901 dev_priv->gtt.base.scratch_page = scratch_page;
2902
2903 return 0;
63340133
BW		 2904}
2905
fbe5d36e
BW		 2906/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2907 * bits. When using advanced contexts each context stores its own PAT, but
2908 * writing this data shouldn't be harmful even in those cases. */
ee0ce478 2909static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
fbe5d36e 2910{
fbe5d36e
BW		 2911 uint64_t pat;
2912
2913 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2914 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2915 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2916 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2917 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2918 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2919 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2920 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2921
d6a8b72e
RV		 2922 if (!USES_PPGTT(dev_priv->dev))
2923 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2924 * so RTL will always use the value corresponding to
2925 * pat_sel = 000".
2926 * So let's disable cache for GGTT to avoid screen corruptions.
2927 * MOCS still can be used though.
2928 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2929 * before this patch, i.e. the same uncached + snooping access
2930 * like on gen6/7 seems to be in effect.
2931 * - So this just fixes blitter/render access. Again it looks
2932 * like it's not just uncached access, but uncached + snooping.
2933 * So we can still hold onto all our assumptions wrt cpu
2934 * clflushing on LLC machines.
2935 */
2936 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2937
fbe5d36e
BW		 2938 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2939 * write would work. */
7e435ad2
VS		 2940 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2941 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
fbe5d36e
BW		 2942}
2943
ee0ce478
VS		 2944static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2945{
2946 uint64_t pat;
2947
2948 /*
2949 * Map WB on BDW to snooped on CHV.
2950 *
2951 * Only the snoop bit has meaning for CHV, the rest is
2952 * ignored.
2953 *
cf3d262e
VS		 2954 * The hardware will never snoop for certain types of accesses:
2955 * - CPU GTT (GMADR->GGTT->no snoop->memory)
2956 * - PPGTT page tables
2957 * - some other special cycles
2958 *
2959 * As with BDW, we also need to consider the following for GT accesses:
2960 * "For GGTT, there is NO pat_sel[2:0] from the entry,
2961 * so RTL will always use the value corresponding to
2962 * pat_sel = 000".
2963 * Which means we must set the snoop bit in PAT entry 0
2964 * in order to keep the global status page working.
ee0ce478
VS		 2965 */
2966 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2967 GEN8_PPAT(1, 0) |
2968 GEN8_PPAT(2, 0) |
2969 GEN8_PPAT(3, 0) |
2970 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2971 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2972 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2973 GEN8_PPAT(7, CHV_PPAT_SNOOP);
2974
7e435ad2
VS		 2975 I915_WRITE(GEN8_PRIVATE_PAT_LO, pat);
2976 I915_WRITE(GEN8_PRIVATE_PAT_HI, pat >> 32);
ee0ce478
VS		 2977}
2978
63340133 2979static int gen8_gmch_probe(struct drm_device *dev,
c44ef60e 2980 u64 *gtt_total,
63340133
BW		 2981 size_t *stolen,
2982 phys_addr_t *mappable_base,
c44ef60e 2983 u64 *mappable_end)
63340133
BW		 2984{
2985 struct drm_i915_private *dev_priv = dev->dev_private;
c44ef60e 2986 u64 gtt_size;
63340133
BW		 2987 u16 snb_gmch_ctl;
2988 int ret;
2989
2990 /* TODO: We're not aware of mappable constraints on gen8 yet */
2991 *mappable_base = pci_resource_start(dev->pdev, 2);
2992 *mappable_end = pci_resource_len(dev->pdev, 2);
2993
2994 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2995 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2996
2997 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2998
66375014
DL		 2999 if (INTEL_INFO(dev)->gen >= 9) {
3000 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
3001 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
3002 } else if (IS_CHERRYVIEW(dev)) {
d7f25f23
DL		 3003 *stolen = chv_get_stolen_size(snb_gmch_ctl);
3004 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
3005 } else {
3006 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
3007 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
3008 }
63340133 3009
07749ef3 3010 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
63340133 3011
5a4e33a3 3012 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
ee0ce478
VS		 3013 chv_setup_private_ppat(dev_priv);
3014 else
3015 bdw_setup_private_ppat(dev_priv);
fbe5d36e 3016
63340133
BW		 3017 ret = ggtt_probe_common(dev, gtt_size);
3018
94ec8f61
BW		 3019 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
3020 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
777dc5bb
DV		 3021 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3022 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
63340133 3023
5bab6f60
CW		 3024 if (IS_CHERRYVIEW(dev))
3025 dev_priv->gtt.base.bind_vma = ggtt_bind_vma__BKL;
3026
63340133
BW		 3027 return ret;
3028}
3029
baa09f5f 3030static int gen6_gmch_probe(struct drm_device *dev,
c44ef60e 3031 u64 *gtt_total,
41907ddc
BW		 3032 size_t *stolen,
3033 phys_addr_t *mappable_base,
c44ef60e 3034 u64 *mappable_end)
e76e9aeb
BW		 3035{
3036 struct drm_i915_private *dev_priv = dev->dev_private;
baa09f5f 3037 unsigned int gtt_size;
e76e9aeb 3038 u16 snb_gmch_ctl;
e76e9aeb
BW		 3039 int ret;
3040
41907ddc
BW		 3041 *mappable_base = pci_resource_start(dev->pdev, 2);
3042 *mappable_end = pci_resource_len(dev->pdev, 2);
3043
baa09f5f
BW		 3044 /* 64/512MB is the current min/max we actually know of, but this is just
3045 * a coarse sanity check.
e76e9aeb 3046 */
41907ddc 3047 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
c44ef60e 3048 DRM_ERROR("Unknown GMADR size (%llx)\n",
baa09f5f
BW		 3049 dev_priv->gtt.mappable_end);
3050 return -ENXIO;
e76e9aeb
BW		 3051 }
3052
e76e9aeb
BW		 3053 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
3054 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
e76e9aeb 3055 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
e76e9aeb 3056
c4ae25ec 3057 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
a93e4161 3058
63340133 3059 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
07749ef3 3060 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
e76e9aeb 3061
63340133 3062 ret = ggtt_probe_common(dev, gtt_size);
e76e9aeb 3063
853ba5d2
BW		 3064 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
3065 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
777dc5bb
DV		 3066 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3067 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
7faf1ab2 3068
e76e9aeb
BW		 3069 return ret;
3070}
3071
853ba5d2 3072static void gen6_gmch_remove(struct i915_address_space *vm)
e76e9aeb 3073{
853ba5d2
BW		 3074
3075 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
5ed16782 3076
853ba5d2 3077 iounmap(gtt->gsm);
4ad2af1e 3078 free_scratch_page(vm->dev, vm->scratch_page);
644ec02b 3079}
baa09f5f
BW		 3080
3081static int i915_gmch_probe(struct drm_device *dev,
c44ef60e 3082 u64 *gtt_total,
41907ddc
BW		 3083 size_t *stolen,
3084 phys_addr_t *mappable_base,
c44ef60e 3085 u64 *mappable_end)
baa09f5f
BW		 3086{
3087 struct drm_i915_private *dev_priv = dev->dev_private;
3088 int ret;
3089
baa09f5f
BW		 3090 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
3091 if (!ret) {
3092 DRM_ERROR("failed to set up gmch\n");
3093 return -EIO;
3094 }
3095
41907ddc 3096 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
baa09f5f
BW		 3097
3098 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
d369d2d9 3099 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
853ba5d2 3100 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
d369d2d9
DV		 3101 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
3102 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
baa09f5f 3103
c0a7f818
CW		 3104 if (unlikely(dev_priv->gtt.do_idle_maps))
3105 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
3106
baa09f5f
BW		 3107 return 0;
3108}
3109
853ba5d2 3110static void i915_gmch_remove(struct i915_address_space *vm)
baa09f5f
BW		 3111{
3112 intel_gmch_remove();
3113}
3114
3115int i915_gem_gtt_init(struct drm_device *dev)
3116{
3117 struct drm_i915_private *dev_priv = dev->dev_private;
3118 struct i915_gtt *gtt = &dev_priv->gtt;
baa09f5f
BW		 3119 int ret;
3120
baa09f5f 3121 if (INTEL_INFO(dev)->gen <= 5) {
b2f21b4d 3122 gtt->gtt_probe = i915_gmch_probe;
853ba5d2 3123 gtt->base.cleanup = i915_gmch_remove;
63340133 3124 } else if (INTEL_INFO(dev)->gen < 8) {
b2f21b4d 3125 gtt->gtt_probe = gen6_gmch_probe;
853ba5d2 3126 gtt->base.cleanup = gen6_gmch_remove;
4d15c145 3127 if (IS_HASWELL(dev) && dev_priv->ellc_size)
853ba5d2 3128 gtt->base.pte_encode = iris_pte_encode;
4d15c145 3129 else if (IS_HASWELL(dev))
853ba5d2 3130 gtt->base.pte_encode = hsw_pte_encode;
b2f21b4d 3131 else if (IS_VALLEYVIEW(dev))
853ba5d2 3132 gtt->base.pte_encode = byt_pte_encode;
350ec881
CW		 3133 else if (INTEL_INFO(dev)->gen >= 7)
3134 gtt->base.pte_encode = ivb_pte_encode;
b2f21b4d 3135 else
350ec881 3136 gtt->base.pte_encode = snb_pte_encode;
63340133
BW		 3137 } else {
3138 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
3139 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
baa09f5f
BW		 3140 }
3141
c114f76a
MK		 3142 gtt->base.dev = dev;
3143
853ba5d2 3144 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
b2f21b4d 3145 &gtt->mappable_base, &gtt->mappable_end);
a54c0c27 3146 if (ret)
baa09f5f 3147 return ret;
baa09f5f 3148
baa09f5f 3149 /* GMADR is the PCI mmio aperture into the global GTT. */
c44ef60e 3150 DRM_INFO("Memory usable by graphics device = %lluM\n",
853ba5d2 3151 gtt->base.total >> 20);
c44ef60e 3152 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20);
b2f21b4d 3153 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
5db6c735
DV		 3154#ifdef CONFIG_INTEL_IOMMU
3155 if (intel_iommu_gfx_mapped)
3156 DRM_INFO("VT-d active for gfx access\n");
3157#endif
cfa7c862
DV		 3158 /*
3159 * i915.enable_ppgtt is read-only, so do an early pass to validate the
3160 * user's requested state against the hardware/driver capabilities. We
3161 * do this now so that we can print out any log messages once rather
3162 * than every time we check intel_enable_ppgtt().
3163 */
3164 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
3165 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
baa09f5f
BW		 3166
3167 return 0;
3168}
6f65e29a 3169
fa42331b
DV		 3170void i915_gem_restore_gtt_mappings(struct drm_device *dev)
3171{
3172 struct drm_i915_private *dev_priv = dev->dev_private;
3173 struct drm_i915_gem_object *obj;
3174 struct i915_address_space *vm;
2c3d9984
TU		 3175 struct i915_vma *vma;
3176 bool flush;
fa42331b
DV		 3177
3178 i915_check_and_clear_faults(dev);
3179
3180 /* First fill our portion of the GTT with scratch pages */
3181 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
3182 dev_priv->gtt.base.start,
3183 dev_priv->gtt.base.total,
3184 true);
3185
2c3d9984
TU		 3186 /* Cache flush objects bound into GGTT and rebind them. */
3187 vm = &dev_priv->gtt.base;
fa42331b 3188 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2c3d9984
TU		 3189 flush = false;
3190 list_for_each_entry(vma, &obj->vma_list, vma_link) {
3191 if (vma->vm != vm)
3192 continue;
fa42331b 3193
2c3d9984
TU		 3194 WARN_ON(i915_vma_bind(vma, obj->cache_level,
3195 PIN_UPDATE));
fa42331b 3196
2c3d9984
TU		 3197 flush = true;
3198 }
3199
3200 if (flush)
3201 i915_gem_clflush_object(obj, obj->pin_display);
3202 }
fa42331b
DV		 3203
3204 if (INTEL_INFO(dev)->gen >= 8) {
3205 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
3206 chv_setup_private_ppat(dev_priv);
3207 else
3208 bdw_setup_private_ppat(dev_priv);
3209
3210 return;
3211 }
3212
3213 if (USES_PPGTT(dev)) {
3214 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
3215 /* TODO: Perhaps it shouldn't be gen6 specific */
3216
3217 struct i915_hw_ppgtt *ppgtt =
3218 container_of(vm, struct i915_hw_ppgtt,
3219 base);
3220
3221 if (i915_is_ggtt(vm))
3222 ppgtt = dev_priv->mm.aliasing_ppgtt;
3223
3224 gen6_write_page_range(dev_priv, &ppgtt->pd,
3225 0, ppgtt->base.total);
3226 }
3227 }
3228
3229 i915_ggtt_flush(dev_priv);
3230}
3231
ec7adb6e
JL		 3232static struct i915_vma *
3233__i915_gem_vma_create(struct drm_i915_gem_object *obj,
3234 struct i915_address_space *vm,
3235 const struct i915_ggtt_view *ggtt_view)
6f65e29a 3236{
dabde5c7 3237 struct i915_vma *vma;
6f65e29a 3238
ec7adb6e
JL		 3239 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
3240 return ERR_PTR(-EINVAL);
e20d2ab7
CW		 3241
3242 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
dabde5c7
DC		 3243 if (vma == NULL)
3244 return ERR_PTR(-ENOMEM);
ec7adb6e 3245
6f65e29a
BW		 3246 INIT_LIST_HEAD(&vma->vma_link);
3247 INIT_LIST_HEAD(&vma->mm_list);
3248 INIT_LIST_HEAD(&vma->exec_list);
3249 vma->vm = vm;
3250 vma->obj = obj;
3251
777dc5bb 3252 if (i915_is_ggtt(vm))
ec7adb6e 3253 vma->ggtt_view = *ggtt_view;
6f65e29a 3254
f7635669
TU		 3255 list_add_tail(&vma->vma_link, &obj->vma_list);
3256 if (!i915_is_ggtt(vm))
e07f0552 3257 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
6f65e29a
BW		 3258
3259 return vma;
3260}
3261
3262struct i915_vma *
ec7adb6e
JL		 3263i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
3264 struct i915_address_space *vm)
3265{
3266 struct i915_vma *vma;
3267
3268 vma = i915_gem_obj_to_vma(obj, vm);
3269 if (!vma)
3270 vma = __i915_gem_vma_create(obj, vm,
3271 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
3272
3273 return vma;
3274}
3275
3276struct i915_vma *
3277i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
fe14d5f4 3278 const struct i915_ggtt_view *view)
6f65e29a 3279{
ec7adb6e 3280 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
6f65e29a
BW		 3281 struct i915_vma *vma;
3282
ec7adb6e
JL		 3283 if (WARN_ON(!view))
3284 return ERR_PTR(-EINVAL);
3285
3286 vma = i915_gem_obj_to_ggtt_view(obj, view);
3287
3288 if (IS_ERR(vma))
3289 return vma;
3290
6f65e29a 3291 if (!vma)
ec7adb6e 3292 vma = __i915_gem_vma_create(obj, ggtt, view);
6f65e29a
BW		 3293
3294 return vma;
ec7adb6e 3295
6f65e29a 3296}
fe14d5f4 3297
804beb4b
TU		 3298static struct scatterlist *
3299rotate_pages(dma_addr_t *in, unsigned int offset,
3300 unsigned int width, unsigned int height,
3301 struct sg_table *st, struct scatterlist *sg)
50470bb0
TU		 3302{
3303 unsigned int column, row;
3304 unsigned int src_idx;
50470bb0 3305
804beb4b
TU		 3306 if (!sg) {
3307 st->nents = 0;
3308 sg = st->sgl;
3309 }
50470bb0
TU		 3310
3311 for (column = 0; column < width; column++) {
3312 src_idx = width * (height - 1) + column;
3313 for (row = 0; row < height; row++) {
3314 st->nents++;
3315 /* We don't need the pages, but need to initialize
3316 * the entries so the sg list can be happily traversed.
3317 * The only thing we need are DMA addresses.
3318 */
3319 sg_set_page(sg, NULL, PAGE_SIZE, 0);
804beb4b 3320 sg_dma_address(sg) = in[offset + src_idx];
50470bb0
TU		 3321 sg_dma_len(sg) = PAGE_SIZE;
3322 sg = sg_next(sg);
3323 src_idx -= width;
3324 }
3325 }
804beb4b
TU		 3326
3327 return sg;
50470bb0
TU		 3328}
3329
3330static struct sg_table *
3331intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
3332 struct drm_i915_gem_object *obj)
3333{
50470bb0 3334 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
84fe03f7 3335 unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
89e3e142 3336 unsigned int size_pages_uv;
50470bb0
TU		 3337 struct sg_page_iter sg_iter;
3338 unsigned long i;
3339 dma_addr_t *page_addr_list;
3340 struct sg_table *st;
89e3e142
TU		 3341 unsigned int uv_start_page;
3342 struct scatterlist *sg;
1d00dad5 3343 int ret = -ENOMEM;
50470bb0 3344
50470bb0 3345 /* Allocate a temporary list of source pages for random access. */
84fe03f7
TU		 3346 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE,
3347 sizeof(dma_addr_t));
50470bb0
TU		 3348 if (!page_addr_list)
3349 return ERR_PTR(ret);
3350
89e3e142
TU		 3351 /* Account for UV plane with NV12. */
3352 if (rot_info->pixel_format == DRM_FORMAT_NV12)
3353 size_pages_uv = rot_info->size_uv >> PAGE_SHIFT;
3354 else
3355 size_pages_uv = 0;
3356
50470bb0
TU		 3357 /* Allocate target SG list. */
3358 st = kmalloc(sizeof(*st), GFP_KERNEL);
3359 if (!st)
3360 goto err_st_alloc;
3361
89e3e142 3362 ret = sg_alloc_table(st, size_pages + size_pages_uv, GFP_KERNEL);
50470bb0
TU		 3363 if (ret)
3364 goto err_sg_alloc;
3365
3366 /* Populate source page list from the object. */
3367 i = 0;
3368 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
3369 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
3370 i++;
3371 }
3372
3373 /* Rotate the pages. */
89e3e142 3374 sg = rotate_pages(page_addr_list, 0,
84fe03f7 3375 rot_info->width_pages, rot_info->height_pages,
804beb4b 3376 st, NULL);
50470bb0 3377
89e3e142
TU		 3378 /* Append the UV plane if NV12. */
3379 if (rot_info->pixel_format == DRM_FORMAT_NV12) {
3380 uv_start_page = size_pages;
3381
3382 /* Check for tile-row un-alignment. */
3383 if (offset_in_page(rot_info->uv_offset))
3384 uv_start_page--;
3385
dedf278c
TU		 3386 rot_info->uv_start_page = uv_start_page;
3387
89e3e142
TU		 3388 rotate_pages(page_addr_list, uv_start_page,
3389 rot_info->width_pages_uv,
3390 rot_info->height_pages_uv,
3391 st, sg);
3392 }
3393
50470bb0 3394 DRM_DEBUG_KMS(
89e3e142 3395 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0)).\n",
c9f8fd2d 3396 obj->base.size, rot_info->pitch, rot_info->height,
84fe03f7 3397 rot_info->pixel_format, rot_info->width_pages,
89e3e142
TU		 3398 rot_info->height_pages, size_pages + size_pages_uv,
3399 size_pages);
50470bb0
TU		 3400
3401 drm_free_large(page_addr_list);
3402
3403 return st;
3404
3405err_sg_alloc:
3406 kfree(st);
3407err_st_alloc:
3408 drm_free_large(page_addr_list);
3409
3410 DRM_DEBUG_KMS(
89e3e142 3411 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages (%u plane 0))\n",
c9f8fd2d 3412 obj->base.size, ret, rot_info->pitch, rot_info->height,
84fe03f7 3413 rot_info->pixel_format, rot_info->width_pages,
89e3e142
TU		 3414 rot_info->height_pages, size_pages + size_pages_uv,
3415 size_pages);
50470bb0
TU		 3416 return ERR_PTR(ret);
3417}
ec7adb6e 3418
8bd7ef16
JL		 3419static struct sg_table *
3420intel_partial_pages(const struct i915_ggtt_view *view,
3421 struct drm_i915_gem_object *obj)
3422{
3423 struct sg_table *st;
3424 struct scatterlist *sg;
3425 struct sg_page_iter obj_sg_iter;
3426 int ret = -ENOMEM;
3427
3428 st = kmalloc(sizeof(*st), GFP_KERNEL);
3429 if (!st)
3430 goto err_st_alloc;
3431
3432 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
3433 if (ret)
3434 goto err_sg_alloc;
3435
3436 sg = st->sgl;
3437 st->nents = 0;
3438 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
3439 view->params.partial.offset)
3440 {
3441 if (st->nents >= view->params.partial.size)
3442 break;
3443
3444 sg_set_page(sg, NULL, PAGE_SIZE, 0);
3445 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
3446 sg_dma_len(sg) = PAGE_SIZE;
3447
3448 sg = sg_next(sg);
3449 st->nents++;
3450 }
3451
3452 return st;
3453
3454err_sg_alloc:
3455 kfree(st);
3456err_st_alloc:
3457 return ERR_PTR(ret);
3458}
3459
70b9f6f8 3460static int
50470bb0 3461i915_get_ggtt_vma_pages(struct i915_vma *vma)
fe14d5f4 3462{
50470bb0
TU		 3463 int ret = 0;
3464
fe14d5f4
TU		 3465 if (vma->ggtt_view.pages)
3466 return 0;
3467
3468 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
3469 vma->ggtt_view.pages = vma->obj->pages;
50470bb0
TU		 3470 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
3471 vma->ggtt_view.pages =
3472 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
8bd7ef16
JL		 3473 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
3474 vma->ggtt_view.pages =
3475 intel_partial_pages(&vma->ggtt_view, vma->obj);
fe14d5f4
TU		 3476 else
3477 WARN_ONCE(1, "GGTT view %u not implemented!\n",
3478 vma->ggtt_view.type);
3479
3480 if (!vma->ggtt_view.pages) {
ec7adb6e 3481 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
fe14d5f4 3482 vma->ggtt_view.type);
50470bb0
TU		 3483 ret = -EINVAL;
3484 } else if (IS_ERR(vma->ggtt_view.pages)) {
3485 ret = PTR_ERR(vma->ggtt_view.pages);
3486 vma->ggtt_view.pages = NULL;
3487 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
3488 vma->ggtt_view.type, ret);
fe14d5f4
TU		 3489 }
3490
50470bb0 3491 return ret;
fe14d5f4
TU		 3492}
3493
3494/**
3495 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
3496 * @vma: VMA to map
3497 * @cache_level: mapping cache level
3498 * @flags: flags like global or local mapping
3499 *
3500 * DMA addresses are taken from the scatter-gather table of this object (or of
3501 * this VMA in case of non-default GGTT views) and PTE entries set up.
3502 * Note that DMA addresses are also the only part of the SG table we care about.
3503 */
3504int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
3505 u32 flags)
3506{
75d04a37
MK		 3507 int ret;
3508 u32 bind_flags;
1d335d1b 3509
75d04a37
MK		 3510 if (WARN_ON(flags == 0))
3511 return -EINVAL;
1d335d1b 3512
75d04a37 3513 bind_flags = 0;
0875546c
DV		 3514 if (flags & PIN_GLOBAL)
3515 bind_flags |= GLOBAL_BIND;
3516 if (flags & PIN_USER)
3517 bind_flags |= LOCAL_BIND;
3518
3519 if (flags & PIN_UPDATE)
3520 bind_flags |= vma->bound;
3521 else
3522 bind_flags &= ~vma->bound;
3523
75d04a37
MK		 3524 if (bind_flags == 0)
3525 return 0;
3526
3527 if (vma->bound == 0 && vma->vm->allocate_va_range) {
3528 trace_i915_va_alloc(vma->vm,
3529 vma->node.start,
3530 vma->node.size,
3531 VM_TO_TRACE_NAME(vma->vm));
3532
b2dd4511
MK		 3533 /* XXX: i915_vma_pin() will fix this +- hack */
3534 vma->pin_count++;
75d04a37
MK		 3535 ret = vma->vm->allocate_va_range(vma->vm,
3536 vma->node.start,
3537 vma->node.size);
b2dd4511 3538 vma->pin_count--;
75d04a37
MK		 3539 if (ret)
3540 return ret;
3541 }
3542
3543 ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
70b9f6f8
DV		 3544 if (ret)
3545 return ret;
0875546c
DV		 3546
3547 vma->bound |= bind_flags;
fe14d5f4
TU		 3548
3549 return 0;
3550}
91e6711e
JL		 3551
3552/**
3553 * i915_ggtt_view_size - Get the size of a GGTT view.
3554 * @obj: Object the view is of.
3555 * @view: The view in question.
3556 *
3557 * @return The size of the GGTT view in bytes.
3558 */
3559size_t
3560i915_ggtt_view_size(struct drm_i915_gem_object *obj,
3561 const struct i915_ggtt_view *view)
3562{
9e759ff1 3563 if (view->type == I915_GGTT_VIEW_NORMAL) {
91e6711e 3564 return obj->base.size;
9e759ff1
TU		 3565 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
3566 return view->rotation_info.size;
8bd7ef16
JL		 3567 } else if (view->type == I915_GGTT_VIEW_PARTIAL) {
3568 return view->params.partial.size << PAGE_SHIFT;
91e6711e
JL		 3569 } else {
3570 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type);
3571 return obj->base.size;
3572 }
3573}
Linux kernel - Deliverables
This page took 0.582913 seconds and 5 git commands to generate.