projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
drm/i915: Create page table allocators
[deliverable/linux.git] / drivers / gpu / drm / i915 / i915_gem_gtt.c
1 /*
2 * Copyright © 2010 Daniel Vetter
3 * Copyright © 2011-2014 Intel Corporation
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
26 #include <linux/seq_file.h>
27 #include <drm/drmP.h>
28 #include <drm/i915_drm.h>
29 #include "i915_drv.h"
30 #include "i915_vgpu.h"
31 #include "i915_trace.h"
32 #include "intel_drv.h"
33
34 /**
35 * DOC: Global GTT views
36 *
37 * Background and previous state
38 *
39 * Historically objects could exists (be bound) in global GTT space only as
40 * singular instances with a view representing all of the object's backing pages
41 * in a linear fashion. This view will be called a normal view.
42 *
43 * To support multiple views of the same object, where the number of mapped
44 * pages is not equal to the backing store, or where the layout of the pages
45 * is not linear, concept of a GGTT view was added.
46 *
47 * One example of an alternative view is a stereo display driven by a single
48 * image. In this case we would have a framebuffer looking like this
49 * (2x2 pages):
50 *
51 * 12
52 * 34
53 *
54 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
55 * rendering. In contrast, fed to the display engine would be an alternative
56 * view which could look something like this:
57 *
58 * 1212
59 * 3434
60 *
61 * In this example both the size and layout of pages in the alternative view is
62 * different from the normal view.
63 *
64 * Implementation and usage
65 *
66 * GGTT views are implemented using VMAs and are distinguished via enum
67 * i915_ggtt_view_type and struct i915_ggtt_view.
68 *
69 * A new flavour of core GEM functions which work with GGTT bound objects were
70 * added with the _view suffix. They take the struct i915_ggtt_view parameter
71 * encapsulating all metadata required to implement a view.
72 *
73 * As a helper for callers which are only interested in the normal view,
74 * globally const i915_ggtt_view_normal singleton instance exists. All old core
75 * GEM API functions, the ones not taking the view parameter, are operating on,
76 * or with the normal GGTT view.
77 *
78 * Code wanting to add or use a new GGTT view needs to:
79 *
80 * 1. Add a new enum with a suitable name.
81 * 2. Extend the metadata in the i915_ggtt_view structure if required.
82 * 3. Add support to i915_get_vma_pages().
83 *
84 * New views are required to build a scatter-gather table from within the
85 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
86 * exists for the lifetime of an VMA.
87 *
88 * Core API is designed to have copy semantics which means that passed in
89 * struct i915_ggtt_view does not need to be persistent (left around after
90 * calling the core API functions).
91 *
92 */
93
94 const struct i915_ggtt_view i915_ggtt_view_normal;
95
96 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
97 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
98
99 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
100 {
101 bool has_aliasing_ppgtt;
102 bool has_full_ppgtt;
103
104 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
105 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
106
107 if (intel_vgpu_active(dev))
108 has_full_ppgtt = false; /* emulation is too hard */
109
110 /*
111 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
112 * execlists, the sole mechanism available to submit work.
113 */
114 if (INTEL_INFO(dev)->gen < 9 &&
115 (enable_ppgtt == 0 || !has_aliasing_ppgtt))
116 return 0;
117
118 if (enable_ppgtt == 1)
119 return 1;
120
121 if (enable_ppgtt == 2 && has_full_ppgtt)
122 return 2;
123
124 #ifdef CONFIG_INTEL_IOMMU
125 /* Disable ppgtt on SNB if VT-d is on. */
126 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
127 DRM_INFO("Disabling PPGTT because VT-d is on\n");
128 return 0;
129 }
130 #endif
131
132 /* Early VLV doesn't have this */
133 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
134 dev->pdev->revision < 0xb) {
135 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
136 return 0;
137 }
138
139 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
140 return 2;
141 else
142 return has_aliasing_ppgtt ? 1 : 0;
143 }
144
145
146 static void ppgtt_bind_vma(struct i915_vma *vma,
147 enum i915_cache_level cache_level,
148 u32 flags);
149 static void ppgtt_unbind_vma(struct i915_vma *vma);
150
151 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
152 enum i915_cache_level level,
153 bool valid)
154 {
155 gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
156 pte |= addr;
157
158 switch (level) {
159 case I915_CACHE_NONE:
160 pte |= PPAT_UNCACHED_INDEX;
161 break;
162 case I915_CACHE_WT:
163 pte |= PPAT_DISPLAY_ELLC_INDEX;
164 break;
165 default:
166 pte |= PPAT_CACHED_INDEX;
167 break;
168 }
169
170 return pte;
171 }
172
173 static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
174 dma_addr_t addr,
175 enum i915_cache_level level)
176 {
177 gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
178 pde |= addr;
179 if (level != I915_CACHE_NONE)
180 pde |= PPAT_CACHED_PDE_INDEX;
181 else
182 pde |= PPAT_UNCACHED_INDEX;
183 return pde;
184 }
185
186 static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
187 enum i915_cache_level level,
188 bool valid, u32 unused)
189 {
190 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
191 pte |= GEN6_PTE_ADDR_ENCODE(addr);
192
193 switch (level) {
194 case I915_CACHE_L3_LLC:
195 case I915_CACHE_LLC:
196 pte |= GEN6_PTE_CACHE_LLC;
197 break;
198 case I915_CACHE_NONE:
199 pte |= GEN6_PTE_UNCACHED;
200 break;
201 default:
202 MISSING_CASE(level);
203 }
204
205 return pte;
206 }
207
208 static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
209 enum i915_cache_level level,
210 bool valid, u32 unused)
211 {
212 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
213 pte |= GEN6_PTE_ADDR_ENCODE(addr);
214
215 switch (level) {
216 case I915_CACHE_L3_LLC:
217 pte |= GEN7_PTE_CACHE_L3_LLC;
218 break;
219 case I915_CACHE_LLC:
220 pte |= GEN6_PTE_CACHE_LLC;
221 break;
222 case I915_CACHE_NONE:
223 pte |= GEN6_PTE_UNCACHED;
224 break;
225 default:
226 MISSING_CASE(level);
227 }
228
229 return pte;
230 }
231
232 static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
233 enum i915_cache_level level,
234 bool valid, u32 flags)
235 {
236 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
237 pte |= GEN6_PTE_ADDR_ENCODE(addr);
238
239 if (!(flags & PTE_READ_ONLY))
240 pte |= BYT_PTE_WRITEABLE;
241
242 if (level != I915_CACHE_NONE)
243 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
244
245 return pte;
246 }
247
248 static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
249 enum i915_cache_level level,
250 bool valid, u32 unused)
251 {
252 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
253 pte |= HSW_PTE_ADDR_ENCODE(addr);
254
255 if (level != I915_CACHE_NONE)
256 pte |= HSW_WB_LLC_AGE3;
257
258 return pte;
259 }
260
261 static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
262 enum i915_cache_level level,
263 bool valid, u32 unused)
264 {
265 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
266 pte |= HSW_PTE_ADDR_ENCODE(addr);
267
268 switch (level) {
269 case I915_CACHE_NONE:
270 break;
271 case I915_CACHE_WT:
272 pte |= HSW_WT_ELLC_LLC_AGE3;
273 break;
274 default:
275 pte |= HSW_WB_ELLC_LLC_AGE3;
276 break;
277 }
278
279 return pte;
280 }
281
282 static void unmap_and_free_pt(struct i915_page_table_entry *pt)
283 {
284 if (WARN_ON(!pt->page))
285 return;
286 __free_page(pt->page);
287 kfree(pt);
288 }
289
290 static struct i915_page_table_entry *alloc_pt_single(void)
291 {
292 struct i915_page_table_entry *pt;
293
294 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
295 if (!pt)
296 return ERR_PTR(-ENOMEM);
297
298 pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
299 if (!pt->page) {
300 kfree(pt);
301 return ERR_PTR(-ENOMEM);
302 }
303
304 return pt;
305 }
306
307 /**
308 * alloc_pt_range() - Allocate a multiple page tables
309 * @pd: The page directory which will have at least @count entries
310 * available to point to the allocated page tables.
311 * @pde: First page directory entry for which we are allocating.
312 * @count: Number of pages to allocate.
313 *
314 * Allocates multiple page table pages and sets the appropriate entries in the
315 * page table structure within the page directory. Function cleans up after
316 * itself on any failures.
317 *
318 * Return: 0 if allocation succeeded.
319 */
320 static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, size_t count)
321 {
322 int i, ret;
323
324 /* 512 is the max page tables per page_directory on any platform. */
325 if (WARN_ON(pde + count > GEN6_PPGTT_PD_ENTRIES))
326 return -EINVAL;
327
328 for (i = pde; i < pde + count; i++) {
329 struct i915_page_table_entry *pt = alloc_pt_single();
330
331 if (IS_ERR(pt)) {
332 ret = PTR_ERR(pt);
333 goto err_out;
334 }
335 WARN(pd->page_table[i],
336 "Leaking page directory entry %d (%pa)\n",
337 i, pd->page_table[i]);
338 pd->page_table[i] = pt;
339 }
340
341 return 0;
342
343 err_out:
344 while (i-- > pde)
345 unmap_and_free_pt(pd->page_table[i]);
346 return ret;
347 }
348
349 static void unmap_and_free_pd(struct i915_page_directory_entry *pd)
350 {
351 if (pd->page) {
352 __free_page(pd->page);
353 kfree(pd);
354 }
355 }
356
357 static struct i915_page_directory_entry *alloc_pd_single(void)
358 {
359 struct i915_page_directory_entry *pd;
360
361 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
362 if (!pd)
363 return ERR_PTR(-ENOMEM);
364
365 pd->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
366 if (!pd->page) {
367 kfree(pd);
368 return ERR_PTR(-ENOMEM);
369 }
370
371 return pd;
372 }
373
374 /* Broadwell Page Directory Pointer Descriptors */
375 static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
376 uint64_t val)
377 {
378 int ret;
379
380 BUG_ON(entry >= 4);
381
382 ret = intel_ring_begin(ring, 6);
383 if (ret)
384 return ret;
385
386 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
387 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
388 intel_ring_emit(ring, (u32)(val >> 32));
389 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
390 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
391 intel_ring_emit(ring, (u32)(val));
392 intel_ring_advance(ring);
393
394 return 0;
395 }
396
397 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
398 struct intel_engine_cs *ring)
399 {
400 int i, ret;
401
402 /* bit of a hack to find the actual last used pd */
403 int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
404
405 for (i = used_pd - 1; i >= 0; i--) {
406 dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr;
407 ret = gen8_write_pdp(ring, i, addr);
408 if (ret)
409 return ret;
410 }
411
412 return 0;
413 }
414
415 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
416 uint64_t start,
417 uint64_t length,
418 bool use_scratch)
419 {
420 struct i915_hw_ppgtt *ppgtt =
421 container_of(vm, struct i915_hw_ppgtt, base);
422 gen8_gtt_pte_t *pt_vaddr, scratch_pte;
423 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
424 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
425 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
426 unsigned num_entries = length >> PAGE_SHIFT;
427 unsigned last_pte, i;
428
429 scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
430 I915_CACHE_LLC, use_scratch);
431
432 while (num_entries) {
433 struct i915_page_directory_entry *pd;
434 struct i915_page_table_entry *pt;
435 struct page *page_table;
436
437 if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))
438 continue;
439
440 pd = ppgtt->pdp.page_directory[pdpe];
441
442 if (WARN_ON(!pd->page_table[pde]))
443 continue;
444
445 pt = pd->page_table[pde];
446
447 if (WARN_ON(!pt->page))
448 continue;
449
450 page_table = pt->page;
451
452 last_pte = pte + num_entries;
453 if (last_pte > GEN8_PTES_PER_PAGE)
454 last_pte = GEN8_PTES_PER_PAGE;
455
456 pt_vaddr = kmap_atomic(page_table);
457
458 for (i = pte; i < last_pte; i++) {
459 pt_vaddr[i] = scratch_pte;
460 num_entries--;
461 }
462
463 if (!HAS_LLC(ppgtt->base.dev))
464 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
465 kunmap_atomic(pt_vaddr);
466
467 pte = 0;
468 if (++pde == GEN8_PDES_PER_PAGE) {
469 pdpe++;
470 pde = 0;
471 }
472 }
473 }
474
475 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
476 struct sg_table *pages,
477 uint64_t start,
478 enum i915_cache_level cache_level, u32 unused)
479 {
480 struct i915_hw_ppgtt *ppgtt =
481 container_of(vm, struct i915_hw_ppgtt, base);
482 gen8_gtt_pte_t *pt_vaddr;
483 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
484 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
485 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
486 struct sg_page_iter sg_iter;
487
488 pt_vaddr = NULL;
489
490 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
491 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
492 break;
493
494 if (pt_vaddr == NULL) {
495 struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe];
496 struct i915_page_table_entry *pt = pd->page_table[pde];
497 struct page *page_table = pt->page;
498
499 pt_vaddr = kmap_atomic(page_table);
500 }
501
502 pt_vaddr[pte] =
503 gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
504 cache_level, true);
505 if (++pte == GEN8_PTES_PER_PAGE) {
506 if (!HAS_LLC(ppgtt->base.dev))
507 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
508 kunmap_atomic(pt_vaddr);
509 pt_vaddr = NULL;
510 if (++pde == GEN8_PDES_PER_PAGE) {
511 pdpe++;
512 pde = 0;
513 }
514 pte = 0;
515 }
516 }
517 if (pt_vaddr) {
518 if (!HAS_LLC(ppgtt->base.dev))
519 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
520 kunmap_atomic(pt_vaddr);
521 }
522 }
523
524 static void gen8_free_page_tables(struct i915_page_directory_entry *pd)
525 {
526 int i;
527
528 if (!pd->page)
529 return;
530
531 for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
532 if (WARN_ON(!pd->page_table[i]))
533 continue;
534
535 unmap_and_free_pt(pd->page_table[i]);
536 pd->page_table[i] = NULL;
537 }
538 }
539
540 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
541 {
542 int i;
543
544 for (i = 0; i < ppgtt->num_pd_pages; i++) {
545 if (WARN_ON(!ppgtt->pdp.page_directory[i]))
546 continue;
547
548 gen8_free_page_tables(ppgtt->pdp.page_directory[i]);
549 unmap_and_free_pd(ppgtt->pdp.page_directory[i]);
550 }
551 }
552
553 static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
554 {
555 struct pci_dev *hwdev = ppgtt->base.dev->pdev;
556 int i, j;
557
558 for (i = 0; i < ppgtt->num_pd_pages; i++) {
559 /* TODO: In the future we'll support sparse mappings, so this
560 * will have to change. */
561 if (!ppgtt->pdp.page_directory[i]->daddr)
562 continue;
563
564 pci_unmap_page(hwdev, ppgtt->pdp.page_directory[i]->daddr, PAGE_SIZE,
565 PCI_DMA_BIDIRECTIONAL);
566
567 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
568 struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];
569 struct i915_page_table_entry *pt;
570 dma_addr_t addr;
571
572 if (WARN_ON(!pd->page_table[j]))
573 continue;
574
575 pt = pd->page_table[j];
576 addr = pt->daddr;
577
578 if (addr)
579 pci_unmap_page(hwdev, addr, PAGE_SIZE,
580 PCI_DMA_BIDIRECTIONAL);
581 }
582 }
583 }
584
585 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
586 {
587 struct i915_hw_ppgtt *ppgtt =
588 container_of(vm, struct i915_hw_ppgtt, base);
589
590 gen8_ppgtt_unmap_pages(ppgtt);
591 gen8_ppgtt_free(ppgtt);
592 }
593
594 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
595 {
596 int i, ret;
597
598 for (i = 0; i < ppgtt->num_pd_pages; i++) {
599 ret = alloc_pt_range(ppgtt->pdp.page_directory[i],
600 0, GEN8_PDES_PER_PAGE);
601 if (ret)
602 goto unwind_out;
603 }
604
605 return 0;
606
607 unwind_out:
608 while (i--)
609 gen8_free_page_tables(ppgtt->pdp.page_directory[i]);
610
611 return -ENOMEM;
612 }
613
614 static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
615 const int max_pdp)
616 {
617 int i;
618
619 for (i = 0; i < max_pdp; i++) {
620 ppgtt->pdp.page_directory[i] = alloc_pd_single();
621 if (IS_ERR(ppgtt->pdp.page_directory[i]))
622 goto unwind_out;
623 }
624
625 ppgtt->num_pd_pages = max_pdp;
626 BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPES);
627
628 return 0;
629
630 unwind_out:
631 while (i--)
632 unmap_and_free_pd(ppgtt->pdp.page_directory[i]);
633
634 return -ENOMEM;
635 }
636
637 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
638 const int max_pdp)
639 {
640 int ret;
641
642 ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
643 if (ret)
644 return ret;
645
646 ret = gen8_ppgtt_allocate_page_tables(ppgtt);
647 if (ret)
648 goto err_out;
649
650 ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
651
652 return 0;
653
654 err_out:
655 gen8_ppgtt_free(ppgtt);
656 return ret;
657 }
658
659 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
660 const int pd)
661 {
662 dma_addr_t pd_addr;
663 int ret;
664
665 pd_addr = pci_map_page(ppgtt->base.dev->pdev,
666 ppgtt->pdp.page_directory[pd]->page, 0,
667 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
668
669 ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
670 if (ret)
671 return ret;
672
673 ppgtt->pdp.page_directory[pd]->daddr = pd_addr;
674
675 return 0;
676 }
677
678 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
679 const int pd,
680 const int pt)
681 {
682 dma_addr_t pt_addr;
683 struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd];
684 struct i915_page_table_entry *ptab = pdir->page_table[pt];
685 struct page *p = ptab->page;
686 int ret;
687
688 pt_addr = pci_map_page(ppgtt->base.dev->pdev,
689 p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
690 ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
691 if (ret)
692 return ret;
693
694 ptab->daddr = pt_addr;
695
696 return 0;
697 }
698
699 /**
700 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
701 * with a net effect resembling a 2-level page table in normal x86 terms. Each
702 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
703 * space.
704 *
705 * FIXME: split allocation into smaller pieces. For now we only ever do this
706 * once, but with full PPGTT, the multiple contiguous allocations will be bad.
707 * TODO: Do something with the size parameter
708 */
709 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
710 {
711 const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
712 const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
713 int i, j, ret;
714
715 if (size % (1<<30))
716 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
717
718 /* 1. Do all our allocations for page directories and page tables. */
719 ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
720 if (ret)
721 return ret;
722
723 /*
724 * 2. Create DMA mappings for the page directories and page tables.
725 */
726 for (i = 0; i < max_pdp; i++) {
727 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
728 if (ret)
729 goto bail;
730
731 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
732 ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
733 if (ret)
734 goto bail;
735 }
736 }
737
738 /*
739 * 3. Map all the page directory entires to point to the page tables
740 * we've allocated.
741 *
742 * For now, the PPGTT helper functions all require that the PDEs are
743 * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
744 * will never need to touch the PDEs again.
745 */
746 for (i = 0; i < max_pdp; i++) {
747 struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];
748 gen8_ppgtt_pde_t *pd_vaddr;
749 pd_vaddr = kmap_atomic(ppgtt->pdp.page_directory[i]->page);
750 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
751 struct i915_page_table_entry *pt = pd->page_table[j];
752 dma_addr_t addr = pt->daddr;
753 pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
754 I915_CACHE_LLC);
755 }
756 if (!HAS_LLC(ppgtt->base.dev))
757 drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
758 kunmap_atomic(pd_vaddr);
759 }
760
761 ppgtt->switch_mm = gen8_mm_switch;
762 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
763 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
764 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
765 ppgtt->base.start = 0;
766 ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
767
768 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
769
770 DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
771 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
772 DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
773 ppgtt->num_pd_entries,
774 (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
775 return 0;
776
777 bail:
778 gen8_ppgtt_unmap_pages(ppgtt);
779 gen8_ppgtt_free(ppgtt);
780 return ret;
781 }
782
783 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
784 {
785 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
786 struct i915_address_space *vm = &ppgtt->base;
787 gen6_gtt_pte_t __iomem *pd_addr;
788 gen6_gtt_pte_t scratch_pte;
789 uint32_t pd_entry;
790 int pte, pde;
791
792 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
793
794 pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
795 ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
796
797 seq_printf(m, " VM %p (pd_offset %x-%x):\n", vm,
798 ppgtt->pd.pd_offset,
799 ppgtt->pd.pd_offset + ppgtt->num_pd_entries);
800 for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
801 u32 expected;
802 gen6_gtt_pte_t *pt_vaddr;
803 dma_addr_t pt_addr = ppgtt->pd.page_table[pde]->daddr;
804 pd_entry = readl(pd_addr + pde);
805 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
806
807 if (pd_entry != expected)
808 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
809 pde,
810 pd_entry,
811 expected);
812 seq_printf(m, "\tPDE: %x\n", pd_entry);
813
814 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[pde]->page);
815 for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
816 unsigned long va =
817 (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
818 (pte * PAGE_SIZE);
819 int i;
820 bool found = false;
821 for (i = 0; i < 4; i++)
822 if (pt_vaddr[pte + i] != scratch_pte)
823 found = true;
824 if (!found)
825 continue;
826
827 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
828 for (i = 0; i < 4; i++) {
829 if (pt_vaddr[pte + i] != scratch_pte)
830 seq_printf(m, " %08x", pt_vaddr[pte + i]);
831 else
832 seq_puts(m, " SCRATCH ");
833 }
834 seq_puts(m, "\n");
835 }
836 kunmap_atomic(pt_vaddr);
837 }
838 }
839
840 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
841 {
842 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
843 gen6_gtt_pte_t __iomem *pd_addr;
844 uint32_t pd_entry;
845 int i;
846
847 WARN_ON(ppgtt->pd.pd_offset & 0x3f);
848 pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
849 ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
850 for (i = 0; i < ppgtt->num_pd_entries; i++) {
851 dma_addr_t pt_addr;
852
853 pt_addr = ppgtt->pd.page_table[i]->daddr;
854 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
855 pd_entry |= GEN6_PDE_VALID;
856
857 writel(pd_entry, pd_addr + i);
858 }
859 readl(pd_addr);
860 }
861
862 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
863 {
864 BUG_ON(ppgtt->pd.pd_offset & 0x3f);
865
866 return (ppgtt->pd.pd_offset / 64) << 16;
867 }
868
869 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
870 struct intel_engine_cs *ring)
871 {
872 int ret;
873
874 /* NB: TLBs must be flushed and invalidated before a switch */
875 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
876 if (ret)
877 return ret;
878
879 ret = intel_ring_begin(ring, 6);
880 if (ret)
881 return ret;
882
883 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
884 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
885 intel_ring_emit(ring, PP_DIR_DCLV_2G);
886 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
887 intel_ring_emit(ring, get_pd_offset(ppgtt));
888 intel_ring_emit(ring, MI_NOOP);
889 intel_ring_advance(ring);
890
891 return 0;
892 }
893
894 static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
895 struct intel_engine_cs *ring)
896 {
897 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
898
899 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
900 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
901 return 0;
902 }
903
904 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
905 struct intel_engine_cs *ring)
906 {
907 int ret;
908
909 /* NB: TLBs must be flushed and invalidated before a switch */
910 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
911 if (ret)
912 return ret;
913
914 ret = intel_ring_begin(ring, 6);
915 if (ret)
916 return ret;
917
918 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
919 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
920 intel_ring_emit(ring, PP_DIR_DCLV_2G);
921 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
922 intel_ring_emit(ring, get_pd_offset(ppgtt));
923 intel_ring_emit(ring, MI_NOOP);
924 intel_ring_advance(ring);
925
926 /* XXX: RCS is the only one to auto invalidate the TLBs? */
927 if (ring->id != RCS) {
928 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
929 if (ret)
930 return ret;
931 }
932
933 return 0;
934 }
935
936 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
937 struct intel_engine_cs *ring)
938 {
939 struct drm_device *dev = ppgtt->base.dev;
940 struct drm_i915_private *dev_priv = dev->dev_private;
941
942
943 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
944 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
945
946 POSTING_READ(RING_PP_DIR_DCLV(ring));
947
948 return 0;
949 }
950
951 static void gen8_ppgtt_enable(struct drm_device *dev)
952 {
953 struct drm_i915_private *dev_priv = dev->dev_private;
954 struct intel_engine_cs *ring;
955 int j;
956
957 for_each_ring(ring, dev_priv, j) {
958 I915_WRITE(RING_MODE_GEN7(ring),
959 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
960 }
961 }
962
963 static void gen7_ppgtt_enable(struct drm_device *dev)
964 {
965 struct drm_i915_private *dev_priv = dev->dev_private;
966 struct intel_engine_cs *ring;
967 uint32_t ecochk, ecobits;
968 int i;
969
970 ecobits = I915_READ(GAC_ECO_BITS);
971 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
972
973 ecochk = I915_READ(GAM_ECOCHK);
974 if (IS_HASWELL(dev)) {
975 ecochk |= ECOCHK_PPGTT_WB_HSW;
976 } else {
977 ecochk |= ECOCHK_PPGTT_LLC_IVB;
978 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
979 }
980 I915_WRITE(GAM_ECOCHK, ecochk);
981
982 for_each_ring(ring, dev_priv, i) {
983 /* GFX_MODE is per-ring on gen7+ */
984 I915_WRITE(RING_MODE_GEN7(ring),
985 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
986 }
987 }
988
989 static void gen6_ppgtt_enable(struct drm_device *dev)
990 {
991 struct drm_i915_private *dev_priv = dev->dev_private;
992 uint32_t ecochk, gab_ctl, ecobits;
993
994 ecobits = I915_READ(GAC_ECO_BITS);
995 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
996 ECOBITS_PPGTT_CACHE64B);
997
998 gab_ctl = I915_READ(GAB_CTL);
999 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
1000
1001 ecochk = I915_READ(GAM_ECOCHK);
1002 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
1003
1004 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
1005 }
1006
1007 /* PPGTT support for Sandybdrige/Gen6 and later */
1008 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1009 uint64_t start,
1010 uint64_t length,
1011 bool use_scratch)
1012 {
1013 struct i915_hw_ppgtt *ppgtt =
1014 container_of(vm, struct i915_hw_ppgtt, base);
1015 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
1016 unsigned first_entry = start >> PAGE_SHIFT;
1017 unsigned num_entries = length >> PAGE_SHIFT;
1018 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
1019 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
1020 unsigned last_pte, i;
1021
1022 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true, 0);
1023
1024 while (num_entries) {
1025 last_pte = first_pte + num_entries;
1026 if (last_pte > I915_PPGTT_PT_ENTRIES)
1027 last_pte = I915_PPGTT_PT_ENTRIES;
1028
1029 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
1030
1031 for (i = first_pte; i < last_pte; i++)
1032 pt_vaddr[i] = scratch_pte;
1033
1034 kunmap_atomic(pt_vaddr);
1035
1036 num_entries -= last_pte - first_pte;
1037 first_pte = 0;
1038 act_pt++;
1039 }
1040 }
1041
1042 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1043 struct sg_table *pages,
1044 uint64_t start,
1045 enum i915_cache_level cache_level, u32 flags)
1046 {
1047 struct i915_hw_ppgtt *ppgtt =
1048 container_of(vm, struct i915_hw_ppgtt, base);
1049 gen6_gtt_pte_t *pt_vaddr;
1050 unsigned first_entry = start >> PAGE_SHIFT;
1051 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
1052 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
1053 struct sg_page_iter sg_iter;
1054
1055 pt_vaddr = NULL;
1056 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
1057 if (pt_vaddr == NULL)
1058 pt_vaddr = kmap_atomic(ppgtt->pd.page_table[act_pt]->page);
1059
1060 pt_vaddr[act_pte] =
1061 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
1062 cache_level, true, flags);
1063
1064 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
1065 kunmap_atomic(pt_vaddr);
1066 pt_vaddr = NULL;
1067 act_pt++;
1068 act_pte = 0;
1069 }
1070 }
1071 if (pt_vaddr)
1072 kunmap_atomic(pt_vaddr);
1073 }
1074
1075 static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
1076 {
1077 int i;
1078
1079 for (i = 0; i < ppgtt->num_pd_entries; i++)
1080 pci_unmap_page(ppgtt->base.dev->pdev,
1081 ppgtt->pd.page_table[i]->daddr,
1082 4096, PCI_DMA_BIDIRECTIONAL);
1083 }
1084
1085 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
1086 {
1087 int i;
1088
1089 for (i = 0; i < ppgtt->num_pd_entries; i++)
1090 unmap_and_free_pt(ppgtt->pd.page_table[i]);
1091
1092 unmap_and_free_pd(&ppgtt->pd);
1093 }
1094
1095 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1096 {
1097 struct i915_hw_ppgtt *ppgtt =
1098 container_of(vm, struct i915_hw_ppgtt, base);
1099
1100 drm_mm_remove_node(&ppgtt->node);
1101
1102 gen6_ppgtt_unmap_pages(ppgtt);
1103 gen6_ppgtt_free(ppgtt);
1104 }
1105
1106 static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1107 {
1108 struct drm_device *dev = ppgtt->base.dev;
1109 struct drm_i915_private *dev_priv = dev->dev_private;
1110 bool retried = false;
1111 int ret;
1112
1113 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1114 * allocator works in address space sizes, so it's multiplied by page
1115 * size. We allocate at the top of the GTT to avoid fragmentation.
1116 */
1117 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
1118 alloc:
1119 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1120 &ppgtt->node, GEN6_PD_SIZE,
1121 GEN6_PD_ALIGN, 0,
1122 0, dev_priv->gtt.base.total,
1123 DRM_MM_TOPDOWN);
1124 if (ret == -ENOSPC && !retried) {
1125 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1126 GEN6_PD_SIZE, GEN6_PD_ALIGN,
1127 I915_CACHE_NONE,
1128 0, dev_priv->gtt.base.total,
1129 0);
1130 if (ret)
1131 return ret;
1132
1133 retried = true;
1134 goto alloc;
1135 }
1136
1137 if (ret)
1138 return ret;
1139
1140 if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1141 DRM_DEBUG("Forced to use aperture for PDEs\n");
1142
1143 ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
1144 return 0;
1145 }
1146
1147 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1148 {
1149 int ret;
1150
1151 ret = gen6_ppgtt_allocate_page_directories(ppgtt);
1152 if (ret)
1153 return ret;
1154
1155 ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries);
1156 if (ret) {
1157 drm_mm_remove_node(&ppgtt->node);
1158 return ret;
1159 }
1160
1161 return 0;
1162 }
1163
1164 static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
1165 {
1166 struct drm_device *dev = ppgtt->base.dev;
1167 int i;
1168
1169 for (i = 0; i < ppgtt->num_pd_entries; i++) {
1170 struct page *page;
1171 dma_addr_t pt_addr;
1172
1173 page = ppgtt->pd.page_table[i]->page;
1174 pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
1175 PCI_DMA_BIDIRECTIONAL);
1176
1177 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
1178 gen6_ppgtt_unmap_pages(ppgtt);
1179 return -EIO;
1180 }
1181
1182 ppgtt->pd.page_table[i]->daddr = pt_addr;
1183 }
1184
1185 return 0;
1186 }
1187
1188 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1189 {
1190 struct drm_device *dev = ppgtt->base.dev;
1191 struct drm_i915_private *dev_priv = dev->dev_private;
1192 int ret;
1193
1194 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
1195 if (IS_GEN6(dev)) {
1196 ppgtt->switch_mm = gen6_mm_switch;
1197 } else if (IS_HASWELL(dev)) {
1198 ppgtt->switch_mm = hsw_mm_switch;
1199 } else if (IS_GEN7(dev)) {
1200 ppgtt->switch_mm = gen7_mm_switch;
1201 } else
1202 BUG();
1203
1204 if (intel_vgpu_active(dev))
1205 ppgtt->switch_mm = vgpu_mm_switch;
1206
1207 ret = gen6_ppgtt_alloc(ppgtt);
1208 if (ret)
1209 return ret;
1210
1211 ret = gen6_ppgtt_setup_page_tables(ppgtt);
1212 if (ret) {
1213 gen6_ppgtt_free(ppgtt);
1214 return ret;
1215 }
1216
1217 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1218 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1219 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
1220 ppgtt->base.start = 0;
1221 ppgtt->base.total = ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
1222 ppgtt->debug_dump = gen6_dump_ppgtt;
1223
1224 ppgtt->pd.pd_offset =
1225 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
1226
1227 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
1228
1229 DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
1230 ppgtt->node.size >> 20,
1231 ppgtt->node.start / PAGE_SIZE);
1232
1233 gen6_write_pdes(ppgtt);
1234 DRM_DEBUG("Adding PPGTT at offset %x\n",
1235 ppgtt->pd.pd_offset << 10);
1236
1237 return 0;
1238 }
1239
1240 static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1241 {
1242 struct drm_i915_private *dev_priv = dev->dev_private;
1243
1244 ppgtt->base.dev = dev;
1245 ppgtt->base.scratch = dev_priv->gtt.base.scratch;
1246
1247 if (INTEL_INFO(dev)->gen < 8)
1248 return gen6_ppgtt_init(ppgtt);
1249 else
1250 return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
1251 }
1252 int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1253 {
1254 struct drm_i915_private *dev_priv = dev->dev_private;
1255 int ret = 0;
1256
1257 ret = __hw_ppgtt_init(dev, ppgtt);
1258 if (ret == 0) {
1259 kref_init(&ppgtt->ref);
1260 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1261 ppgtt->base.total);
1262 i915_init_vm(dev_priv, &ppgtt->base);
1263 }
1264
1265 return ret;
1266 }
1267
1268 int i915_ppgtt_init_hw(struct drm_device *dev)
1269 {
1270 struct drm_i915_private *dev_priv = dev->dev_private;
1271 struct intel_engine_cs *ring;
1272 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1273 int i, ret = 0;
1274
1275 /* In the case of execlists, PPGTT is enabled by the context descriptor
1276 * and the PDPs are contained within the context itself. We don't
1277 * need to do anything here. */
1278 if (i915.enable_execlists)
1279 return 0;
1280
1281 if (!USES_PPGTT(dev))
1282 return 0;
1283
1284 if (IS_GEN6(dev))
1285 gen6_ppgtt_enable(dev);
1286 else if (IS_GEN7(dev))
1287 gen7_ppgtt_enable(dev);
1288 else if (INTEL_INFO(dev)->gen >= 8)
1289 gen8_ppgtt_enable(dev);
1290 else
1291 MISSING_CASE(INTEL_INFO(dev)->gen);
1292
1293 if (ppgtt) {
1294 for_each_ring(ring, dev_priv, i) {
1295 ret = ppgtt->switch_mm(ppgtt, ring);
1296 if (ret != 0)
1297 return ret;
1298 }
1299 }
1300
1301 return ret;
1302 }
1303 struct i915_hw_ppgtt *
1304 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1305 {
1306 struct i915_hw_ppgtt *ppgtt;
1307 int ret;
1308
1309 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1310 if (!ppgtt)
1311 return ERR_PTR(-ENOMEM);
1312
1313 ret = i915_ppgtt_init(dev, ppgtt);
1314 if (ret) {
1315 kfree(ppgtt);
1316 return ERR_PTR(ret);
1317 }
1318
1319 ppgtt->file_priv = fpriv;
1320
1321 trace_i915_ppgtt_create(&ppgtt->base);
1322
1323 return ppgtt;
1324 }
1325
1326 void i915_ppgtt_release(struct kref *kref)
1327 {
1328 struct i915_hw_ppgtt *ppgtt =
1329 container_of(kref, struct i915_hw_ppgtt, ref);
1330
1331 trace_i915_ppgtt_release(&ppgtt->base);
1332
1333 /* vmas should already be unbound */
1334 WARN_ON(!list_empty(&ppgtt->base.active_list));
1335 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1336
1337 list_del(&ppgtt->base.global_link);
1338 drm_mm_takedown(&ppgtt->base.mm);
1339
1340 ppgtt->base.cleanup(&ppgtt->base);
1341 kfree(ppgtt);
1342 }
1343
1344 static void
1345 ppgtt_bind_vma(struct i915_vma *vma,
1346 enum i915_cache_level cache_level,
1347 u32 flags)
1348 {
1349 /* Currently applicable only to VLV */
1350 if (vma->obj->gt_ro)
1351 flags |= PTE_READ_ONLY;
1352
1353 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1354 cache_level, flags);
1355 }
1356
1357 static void ppgtt_unbind_vma(struct i915_vma *vma)
1358 {
1359 vma->vm->clear_range(vma->vm,
1360 vma->node.start,
1361 vma->obj->base.size,
1362 true);
1363 }
1364
1365 extern int intel_iommu_gfx_mapped;
1366 /* Certain Gen5 chipsets require require idling the GPU before
1367 * unmapping anything from the GTT when VT-d is enabled.
1368 */
1369 static inline bool needs_idle_maps(struct drm_device *dev)
1370 {
1371 #ifdef CONFIG_INTEL_IOMMU
1372 /* Query intel_iommu to see if we need the workaround. Presumably that
1373 * was loaded first.
1374 */
1375 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1376 return true;
1377 #endif
1378 return false;
1379 }
1380
1381 static bool do_idling(struct drm_i915_private *dev_priv)
1382 {
1383 bool ret = dev_priv->mm.interruptible;
1384
1385 if (unlikely(dev_priv->gtt.do_idle_maps)) {
1386 dev_priv->mm.interruptible = false;
1387 if (i915_gpu_idle(dev_priv->dev)) {
1388 DRM_ERROR("Couldn't idle GPU\n");
1389 /* Wait a bit, in hopes it avoids the hang */
1390 udelay(10);
1391 }
1392 }
1393
1394 return ret;
1395 }
1396
1397 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1398 {
1399 if (unlikely(dev_priv->gtt.do_idle_maps))
1400 dev_priv->mm.interruptible = interruptible;
1401 }
1402
1403 void i915_check_and_clear_faults(struct drm_device *dev)
1404 {
1405 struct drm_i915_private *dev_priv = dev->dev_private;
1406 struct intel_engine_cs *ring;
1407 int i;
1408
1409 if (INTEL_INFO(dev)->gen < 6)
1410 return;
1411
1412 for_each_ring(ring, dev_priv, i) {
1413 u32 fault_reg;
1414 fault_reg = I915_READ(RING_FAULT_REG(ring));
1415 if (fault_reg & RING_FAULT_VALID) {
1416 DRM_DEBUG_DRIVER("Unexpected fault\n"
1417 "\tAddr: 0x%08lx\n"
1418 "\tAddress space: %s\n"
1419 "\tSource ID: %d\n"
1420 "\tType: %d\n",
1421 fault_reg & PAGE_MASK,
1422 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1423 RING_FAULT_SRCID(fault_reg),
1424 RING_FAULT_FAULT_TYPE(fault_reg));
1425 I915_WRITE(RING_FAULT_REG(ring),
1426 fault_reg & ~RING_FAULT_VALID);
1427 }
1428 }
1429 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1430 }
1431
1432 static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1433 {
1434 if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1435 intel_gtt_chipset_flush();
1436 } else {
1437 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1438 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1439 }
1440 }
1441
1442 void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1443 {
1444 struct drm_i915_private *dev_priv = dev->dev_private;
1445
1446 /* Don't bother messing with faults pre GEN6 as we have little
1447 * documentation supporting that it's a good idea.
1448 */
1449 if (INTEL_INFO(dev)->gen < 6)
1450 return;
1451
1452 i915_check_and_clear_faults(dev);
1453
1454 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1455 dev_priv->gtt.base.start,
1456 dev_priv->gtt.base.total,
1457 true);
1458
1459 i915_ggtt_flush(dev_priv);
1460 }
1461
1462 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1463 {
1464 struct drm_i915_private *dev_priv = dev->dev_private;
1465 struct drm_i915_gem_object *obj;
1466 struct i915_address_space *vm;
1467
1468 i915_check_and_clear_faults(dev);
1469
1470 /* First fill our portion of the GTT with scratch pages */
1471 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1472 dev_priv->gtt.base.start,
1473 dev_priv->gtt.base.total,
1474 true);
1475
1476 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1477 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1478 &dev_priv->gtt.base);
1479 if (!vma)
1480 continue;
1481
1482 i915_gem_clflush_object(obj, obj->pin_display);
1483 /* The bind_vma code tries to be smart about tracking mappings.
1484 * Unfortunately above, we've just wiped out the mappings
1485 * without telling our object about it. So we need to fake it.
1486 *
1487 * Bind is not expected to fail since this is only called on
1488 * resume and assumption is all requirements exist already.
1489 */
1490 vma->bound &= ~GLOBAL_BIND;
1491 WARN_ON(i915_vma_bind(vma, obj->cache_level, GLOBAL_BIND));
1492 }
1493
1494
1495 if (INTEL_INFO(dev)->gen >= 8) {
1496 if (IS_CHERRYVIEW(dev))
1497 chv_setup_private_ppat(dev_priv);
1498 else
1499 bdw_setup_private_ppat(dev_priv);
1500
1501 return;
1502 }
1503
1504 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1505 /* TODO: Perhaps it shouldn't be gen6 specific */
1506 if (i915_is_ggtt(vm)) {
1507 if (dev_priv->mm.aliasing_ppgtt)
1508 gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
1509 continue;
1510 }
1511
1512 gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
1513 }
1514
1515 i915_ggtt_flush(dev_priv);
1516 }
1517
1518 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1519 {
1520 if (obj->has_dma_mapping)
1521 return 0;
1522
1523 if (!dma_map_sg(&obj->base.dev->pdev->dev,
1524 obj->pages->sgl, obj->pages->nents,
1525 PCI_DMA_BIDIRECTIONAL))
1526 return -ENOSPC;
1527
1528 return 0;
1529 }
1530
1531 static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
1532 {
1533 #ifdef writeq
1534 writeq(pte, addr);
1535 #else
1536 iowrite32((u32)pte, addr);
1537 iowrite32(pte >> 32, addr + 4);
1538 #endif
1539 }
1540
1541 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1542 struct sg_table *st,
1543 uint64_t start,
1544 enum i915_cache_level level, u32 unused)
1545 {
1546 struct drm_i915_private *dev_priv = vm->dev->dev_private;
1547 unsigned first_entry = start >> PAGE_SHIFT;
1548 gen8_gtt_pte_t __iomem *gtt_entries =
1549 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1550 int i = 0;
1551 struct sg_page_iter sg_iter;
1552 dma_addr_t addr = 0; /* shut up gcc */
1553
1554 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1555 addr = sg_dma_address(sg_iter.sg) +
1556 (sg_iter.sg_pgoffset << PAGE_SHIFT);
1557 gen8_set_pte(&gtt_entries[i],
1558 gen8_pte_encode(addr, level, true));
1559 i++;
1560 }
1561
1562 /*
1563 * XXX: This serves as a posting read to make sure that the PTE has
1564 * actually been updated. There is some concern that even though
1565 * registers and PTEs are within the same BAR that they are potentially
1566 * of NUMA access patterns. Therefore, even with the way we assume
1567 * hardware should work, we must keep this posting read for paranoia.
1568 */
1569 if (i != 0)
1570 WARN_ON(readq(&gtt_entries[i-1])
1571 != gen8_pte_encode(addr, level, true));
1572
1573 /* This next bit makes the above posting read even more important. We
1574 * want to flush the TLBs only after we're certain all the PTE updates
1575 * have finished.
1576 */
1577 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1578 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1579 }
1580
1581 /*
1582 * Binds an object into the global gtt with the specified cache level. The object
1583 * will be accessible to the GPU via commands whose operands reference offsets
1584 * within the global GTT as well as accessible by the GPU through the GMADR
1585 * mapped BAR (dev_priv->mm.gtt->gtt).
1586 */
1587 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1588 struct sg_table *st,
1589 uint64_t start,
1590 enum i915_cache_level level, u32 flags)
1591 {
1592 struct drm_i915_private *dev_priv = vm->dev->dev_private;
1593 unsigned first_entry = start >> PAGE_SHIFT;
1594 gen6_gtt_pte_t __iomem *gtt_entries =
1595 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1596 int i = 0;
1597 struct sg_page_iter sg_iter;
1598 dma_addr_t addr = 0;
1599
1600 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1601 addr = sg_page_iter_dma_address(&sg_iter);
1602 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
1603 i++;
1604 }
1605
1606 /* XXX: This serves as a posting read to make sure that the PTE has
1607 * actually been updated. There is some concern that even though
1608 * registers and PTEs are within the same BAR that they are potentially
1609 * of NUMA access patterns. Therefore, even with the way we assume
1610 * hardware should work, we must keep this posting read for paranoia.
1611 */
1612 if (i != 0) {
1613 unsigned long gtt = readl(&gtt_entries[i-1]);
1614 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1615 }
1616
1617 /* This next bit makes the above posting read even more important. We
1618 * want to flush the TLBs only after we're certain all the PTE updates
1619 * have finished.
1620 */
1621 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1622 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1623 }
1624
1625 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1626 uint64_t start,
1627 uint64_t length,
1628 bool use_scratch)
1629 {
1630 struct drm_i915_private *dev_priv = vm->dev->dev_private;
1631 unsigned first_entry = start >> PAGE_SHIFT;
1632 unsigned num_entries = length >> PAGE_SHIFT;
1633 gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
1634 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1635 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1636 int i;
1637
1638 if (WARN(num_entries > max_entries,
1639 "First entry = %d; Num entries = %d (max=%d)\n",
1640 first_entry, num_entries, max_entries))
1641 num_entries = max_entries;
1642
1643 scratch_pte = gen8_pte_encode(vm->scratch.addr,
1644 I915_CACHE_LLC,
1645 use_scratch);
1646 for (i = 0; i < num_entries; i++)
1647 gen8_set_pte(&gtt_base[i], scratch_pte);
1648 readl(gtt_base);
1649 }
1650
1651 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
1652 uint64_t start,
1653 uint64_t length,
1654 bool use_scratch)
1655 {
1656 struct drm_i915_private *dev_priv = vm->dev->dev_private;
1657 unsigned first_entry = start >> PAGE_SHIFT;
1658 unsigned num_entries = length >> PAGE_SHIFT;
1659 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1660 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1661 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1662 int i;
1663
1664 if (WARN(num_entries > max_entries,
1665 "First entry = %d; Num entries = %d (max=%d)\n",
1666 first_entry, num_entries, max_entries))
1667 num_entries = max_entries;
1668
1669 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch, 0);
1670
1671 for (i = 0; i < num_entries; i++)
1672 iowrite32(scratch_pte, &gtt_base[i]);
1673 readl(gtt_base);
1674 }
1675
1676
1677 static void i915_ggtt_bind_vma(struct i915_vma *vma,
1678 enum i915_cache_level cache_level,
1679 u32 unused)
1680 {
1681 const unsigned long entry = vma->node.start >> PAGE_SHIFT;
1682 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1683 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1684
1685 BUG_ON(!i915_is_ggtt(vma->vm));
1686 intel_gtt_insert_sg_entries(vma->ggtt_view.pages, entry, flags);
1687 vma->bound = GLOBAL_BIND;
1688 }
1689
1690 static void i915_ggtt_clear_range(struct i915_address_space *vm,
1691 uint64_t start,
1692 uint64_t length,
1693 bool unused)
1694 {
1695 unsigned first_entry = start >> PAGE_SHIFT;
1696 unsigned num_entries = length >> PAGE_SHIFT;
1697 intel_gtt_clear_range(first_entry, num_entries);
1698 }
1699
1700 static void i915_ggtt_unbind_vma(struct i915_vma *vma)
1701 {
1702 const unsigned int first = vma->node.start >> PAGE_SHIFT;
1703 const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
1704
1705 BUG_ON(!i915_is_ggtt(vma->vm));
1706 vma->bound = 0;
1707 intel_gtt_clear_range(first, size);
1708 }
1709
1710 static void ggtt_bind_vma(struct i915_vma *vma,
1711 enum i915_cache_level cache_level,
1712 u32 flags)
1713 {
1714 struct drm_device *dev = vma->vm->dev;
1715 struct drm_i915_private *dev_priv = dev->dev_private;
1716 struct drm_i915_gem_object *obj = vma->obj;
1717
1718 /* Currently applicable only to VLV */
1719 if (obj->gt_ro)
1720 flags |= PTE_READ_ONLY;
1721
1722 /* If there is no aliasing PPGTT, or the caller needs a global mapping,
1723 * or we have a global mapping already but the cacheability flags have
1724 * changed, set the global PTEs.
1725 *
1726 * If there is an aliasing PPGTT it is anecdotally faster, so use that
1727 * instead if none of the above hold true.
1728 *
1729 * NB: A global mapping should only be needed for special regions like
1730 * "gtt mappable", SNB errata, or if specified via special execbuf
1731 * flags. At all other times, the GPU will use the aliasing PPGTT.
1732 */
1733 if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
1734 if (!(vma->bound & GLOBAL_BIND) ||
1735 (cache_level != obj->cache_level)) {
1736 vma->vm->insert_entries(vma->vm, vma->ggtt_view.pages,
1737 vma->node.start,
1738 cache_level, flags);
1739 vma->bound |= GLOBAL_BIND;
1740 }
1741 }
1742
1743 if (dev_priv->mm.aliasing_ppgtt &&
1744 (!(vma->bound & LOCAL_BIND) ||
1745 (cache_level != obj->cache_level))) {
1746 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1747 appgtt->base.insert_entries(&appgtt->base,
1748 vma->ggtt_view.pages,
1749 vma->node.start,
1750 cache_level, flags);
1751 vma->bound |= LOCAL_BIND;
1752 }
1753 }
1754
1755 static void ggtt_unbind_vma(struct i915_vma *vma)
1756 {
1757 struct drm_device *dev = vma->vm->dev;
1758 struct drm_i915_private *dev_priv = dev->dev_private;
1759 struct drm_i915_gem_object *obj = vma->obj;
1760
1761 if (vma->bound & GLOBAL_BIND) {
1762 vma->vm->clear_range(vma->vm,
1763 vma->node.start,
1764 obj->base.size,
1765 true);
1766 vma->bound &= ~GLOBAL_BIND;
1767 }
1768
1769 if (vma->bound & LOCAL_BIND) {
1770 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1771 appgtt->base.clear_range(&appgtt->base,
1772 vma->node.start,
1773 obj->base.size,
1774 true);
1775 vma->bound &= ~LOCAL_BIND;
1776 }
1777 }
1778
1779 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1780 {
1781 struct drm_device *dev = obj->base.dev;
1782 struct drm_i915_private *dev_priv = dev->dev_private;
1783 bool interruptible;
1784
1785 interruptible = do_idling(dev_priv);
1786
1787 if (!obj->has_dma_mapping)
1788 dma_unmap_sg(&dev->pdev->dev,
1789 obj->pages->sgl, obj->pages->nents,
1790 PCI_DMA_BIDIRECTIONAL);
1791
1792 undo_idling(dev_priv, interruptible);
1793 }
1794
1795 static void i915_gtt_color_adjust(struct drm_mm_node *node,
1796 unsigned long color,
1797 unsigned long *start,
1798 unsigned long *end)
1799 {
1800 if (node->color != color)
1801 *start += 4096;
1802
1803 if (!list_empty(&node->node_list)) {
1804 node = list_entry(node->node_list.next,
1805 struct drm_mm_node,
1806 node_list);
1807 if (node->allocated && node->color != color)
1808 *end -= 4096;
1809 }
1810 }
1811
1812 static int i915_gem_setup_global_gtt(struct drm_device *dev,
1813 unsigned long start,
1814 unsigned long mappable_end,
1815 unsigned long end)
1816 {
1817 /* Let GEM Manage all of the aperture.
1818 *
1819 * However, leave one page at the end still bound to the scratch page.
1820 * There are a number of places where the hardware apparently prefetches
1821 * past the end of the object, and we've seen multiple hangs with the
1822 * GPU head pointer stuck in a batchbuffer bound at the last page of the
1823 * aperture. One page should be enough to keep any prefetching inside
1824 * of the aperture.
1825 */
1826 struct drm_i915_private *dev_priv = dev->dev_private;
1827 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
1828 struct drm_mm_node *entry;
1829 struct drm_i915_gem_object *obj;
1830 unsigned long hole_start, hole_end;
1831 int ret;
1832
1833 BUG_ON(mappable_end > end);
1834
1835 /* Subtract the guard page ... */
1836 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
1837
1838 dev_priv->gtt.base.start = start;
1839 dev_priv->gtt.base.total = end - start;
1840
1841 if (intel_vgpu_active(dev)) {
1842 ret = intel_vgt_balloon(dev);
1843 if (ret)
1844 return ret;
1845 }
1846
1847 if (!HAS_LLC(dev))
1848 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
1849
1850 /* Mark any preallocated objects as occupied */
1851 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
1852 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
1853
1854 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
1855 i915_gem_obj_ggtt_offset(obj), obj->base.size);
1856
1857 WARN_ON(i915_gem_obj_ggtt_bound(obj));
1858 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
1859 if (ret) {
1860 DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
1861 return ret;
1862 }
1863 vma->bound |= GLOBAL_BIND;
1864 }
1865
1866 /* Clear any non-preallocated blocks */
1867 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
1868 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1869 hole_start, hole_end);
1870 ggtt_vm->clear_range(ggtt_vm, hole_start,
1871 hole_end - hole_start, true);
1872 }
1873
1874 /* And finally clear the reserved guard page */
1875 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
1876
1877 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
1878 struct i915_hw_ppgtt *ppgtt;
1879
1880 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1881 if (!ppgtt)
1882 return -ENOMEM;
1883
1884 ret = __hw_ppgtt_init(dev, ppgtt);
1885 if (ret != 0)
1886 return ret;
1887
1888 dev_priv->mm.aliasing_ppgtt = ppgtt;
1889 }
1890
1891 return 0;
1892 }
1893
1894 void i915_gem_init_global_gtt(struct drm_device *dev)
1895 {
1896 struct drm_i915_private *dev_priv = dev->dev_private;
1897 unsigned long gtt_size, mappable_size;
1898
1899 gtt_size = dev_priv->gtt.base.total;
1900 mappable_size = dev_priv->gtt.mappable_end;
1901
1902 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1903 }
1904
1905 void i915_global_gtt_cleanup(struct drm_device *dev)
1906 {
1907 struct drm_i915_private *dev_priv = dev->dev_private;
1908 struct i915_address_space *vm = &dev_priv->gtt.base;
1909
1910 if (dev_priv->mm.aliasing_ppgtt) {
1911 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1912
1913 ppgtt->base.cleanup(&ppgtt->base);
1914 }
1915
1916 if (drm_mm_initialized(&vm->mm)) {
1917 if (intel_vgpu_active(dev))
1918 intel_vgt_deballoon();
1919
1920 drm_mm_takedown(&vm->mm);
1921 list_del(&vm->global_link);
1922 }
1923
1924 vm->cleanup(vm);
1925 }
1926
1927 static int setup_scratch_page(struct drm_device *dev)
1928 {
1929 struct drm_i915_private *dev_priv = dev->dev_private;
1930 struct page *page;
1931 dma_addr_t dma_addr;
1932
1933 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1934 if (page == NULL)
1935 return -ENOMEM;
1936 set_pages_uc(page, 1);
1937
1938 #ifdef CONFIG_INTEL_IOMMU
1939 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1940 PCI_DMA_BIDIRECTIONAL);
1941 if (pci_dma_mapping_error(dev->pdev, dma_addr))
1942 return -EINVAL;
1943 #else
1944 dma_addr = page_to_phys(page);
1945 #endif
1946 dev_priv->gtt.base.scratch.page = page;
1947 dev_priv->gtt.base.scratch.addr = dma_addr;
1948
1949 return 0;
1950 }
1951
1952 static void teardown_scratch_page(struct drm_device *dev)
1953 {
1954 struct drm_i915_private *dev_priv = dev->dev_private;
1955 struct page *page = dev_priv->gtt.base.scratch.page;
1956
1957 set_pages_wb(page, 1);
1958 pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
1959 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
1960 __free_page(page);
1961 }
1962
1963 static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1964 {
1965 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1966 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1967 return snb_gmch_ctl << 20;
1968 }
1969
1970 static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1971 {
1972 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1973 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1974 if (bdw_gmch_ctl)
1975 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1976
1977 #ifdef CONFIG_X86_32
1978 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
1979 if (bdw_gmch_ctl > 4)
1980 bdw_gmch_ctl = 4;
1981 #endif
1982
1983 return bdw_gmch_ctl << 20;
1984 }
1985
1986 static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
1987 {
1988 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
1989 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
1990
1991 if (gmch_ctrl)
1992 return 1 << (20 + gmch_ctrl);
1993
1994 return 0;
1995 }
1996
1997 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
1998 {
1999 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2000 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2001 return snb_gmch_ctl << 25; /* 32 MB units */
2002 }
2003
2004 static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2005 {
2006 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2007 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2008 return bdw_gmch_ctl << 25; /* 32 MB units */
2009 }
2010
2011 static size_t chv_get_stolen_size(u16 gmch_ctrl)
2012 {
2013 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2014 gmch_ctrl &= SNB_GMCH_GMS_MASK;
2015
2016 /*
2017 * 0x0 to 0x10: 32MB increments starting at 0MB
2018 * 0x11 to 0x16: 4MB increments starting at 8MB
2019 * 0x17 to 0x1d: 4MB increments start at 36MB
2020 */
2021 if (gmch_ctrl < 0x11)
2022 return gmch_ctrl << 25;
2023 else if (gmch_ctrl < 0x17)
2024 return (gmch_ctrl - 0x11 + 2) << 22;
2025 else
2026 return (gmch_ctrl - 0x17 + 9) << 22;
2027 }
2028
2029 static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2030 {
2031 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2032 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2033
2034 if (gen9_gmch_ctl < 0xf0)
2035 return gen9_gmch_ctl << 25; /* 32 MB units */
2036 else
2037 /* 4MB increments starting at 0xf0 for 4MB */
2038 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2039 }
2040
2041 static int ggtt_probe_common(struct drm_device *dev,
2042 size_t gtt_size)
2043 {
2044 struct drm_i915_private *dev_priv = dev->dev_private;
2045 phys_addr_t gtt_phys_addr;
2046 int ret;
2047
2048 /* For Modern GENs the PTEs and register space are split in the BAR */
2049 gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
2050 (pci_resource_len(dev->pdev, 0) / 2);
2051
2052 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
2053 if (!dev_priv->gtt.gsm) {
2054 DRM_ERROR("Failed to map the gtt page table\n");
2055 return -ENOMEM;
2056 }
2057
2058 ret = setup_scratch_page(dev);
2059 if (ret) {
2060 DRM_ERROR("Scratch setup failed\n");
2061 /* iounmap will also get called at remove, but meh */
2062 iounmap(dev_priv->gtt.gsm);
2063 }
2064
2065 return ret;
2066 }
2067
2068 /* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2069 * bits. When using advanced contexts each context stores its own PAT, but
2070 * writing this data shouldn't be harmful even in those cases. */
2071 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
2072 {
2073 uint64_t pat;
2074
2075 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2076 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2077 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2078 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2079 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2080 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2081 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2082 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2083
2084 if (!USES_PPGTT(dev_priv->dev))
2085 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2086 * so RTL will always use the value corresponding to
2087 * pat_sel = 000".
2088 * So let's disable cache for GGTT to avoid screen corruptions.
2089 * MOCS still can be used though.
2090 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2091 * before this patch, i.e. the same uncached + snooping access
2092 * like on gen6/7 seems to be in effect.
2093 * - So this just fixes blitter/render access. Again it looks
2094 * like it's not just uncached access, but uncached + snooping.
2095 * So we can still hold onto all our assumptions wrt cpu
2096 * clflushing on LLC machines.
2097 */
2098 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2099
2100 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2101 * write would work. */
2102 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2103 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2104 }
2105
2106 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2107 {
2108 uint64_t pat;
2109
2110 /*
2111 * Map WB on BDW to snooped on CHV.
2112 *
2113 * Only the snoop bit has meaning for CHV, the rest is
2114 * ignored.
2115 *
2116 * The hardware will never snoop for certain types of accesses:
2117 * - CPU GTT (GMADR->GGTT->no snoop->memory)
2118 * - PPGTT page tables
2119 * - some other special cycles
2120 *
2121 * As with BDW, we also need to consider the following for GT accesses:
2122 * "For GGTT, there is NO pat_sel[2:0] from the entry,
2123 * so RTL will always use the value corresponding to
2124 * pat_sel = 000".
2125 * Which means we must set the snoop bit in PAT entry 0
2126 * in order to keep the global status page working.
2127 */
2128 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2129 GEN8_PPAT(1, 0) |
2130 GEN8_PPAT(2, 0) |
2131 GEN8_PPAT(3, 0) |
2132 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2133 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2134 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2135 GEN8_PPAT(7, CHV_PPAT_SNOOP);
2136
2137 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2138 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2139 }
2140
2141 static int gen8_gmch_probe(struct drm_device *dev,
2142 size_t *gtt_total,
2143 size_t *stolen,
2144 phys_addr_t *mappable_base,
2145 unsigned long *mappable_end)
2146 {
2147 struct drm_i915_private *dev_priv = dev->dev_private;
2148 unsigned int gtt_size;
2149 u16 snb_gmch_ctl;
2150 int ret;
2151
2152 /* TODO: We're not aware of mappable constraints on gen8 yet */
2153 *mappable_base = pci_resource_start(dev->pdev, 2);
2154 *mappable_end = pci_resource_len(dev->pdev, 2);
2155
2156 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2157 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2158
2159 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2160
2161 if (INTEL_INFO(dev)->gen >= 9) {
2162 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
2163 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2164 } else if (IS_CHERRYVIEW(dev)) {
2165 *stolen = chv_get_stolen_size(snb_gmch_ctl);
2166 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
2167 } else {
2168 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
2169 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2170 }
2171
2172 *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
2173
2174 if (IS_CHERRYVIEW(dev))
2175 chv_setup_private_ppat(dev_priv);
2176 else
2177 bdw_setup_private_ppat(dev_priv);
2178
2179 ret = ggtt_probe_common(dev, gtt_size);
2180
2181 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
2182 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
2183
2184 return ret;
2185 }
2186
2187 static int gen6_gmch_probe(struct drm_device *dev,
2188 size_t *gtt_total,
2189 size_t *stolen,
2190 phys_addr_t *mappable_base,
2191 unsigned long *mappable_end)
2192 {
2193 struct drm_i915_private *dev_priv = dev->dev_private;
2194 unsigned int gtt_size;
2195 u16 snb_gmch_ctl;
2196 int ret;
2197
2198 *mappable_base = pci_resource_start(dev->pdev, 2);
2199 *mappable_end = pci_resource_len(dev->pdev, 2);
2200
2201 /* 64/512MB is the current min/max we actually know of, but this is just
2202 * a coarse sanity check.
2203 */
2204 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
2205 DRM_ERROR("Unknown GMADR size (%lx)\n",
2206 dev_priv->gtt.mappable_end);
2207 return -ENXIO;
2208 }
2209
2210 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2211 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
2212 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2213
2214 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
2215
2216 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
2217 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
2218
2219 ret = ggtt_probe_common(dev, gtt_size);
2220
2221 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2222 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
2223
2224 return ret;
2225 }
2226
2227 static void gen6_gmch_remove(struct i915_address_space *vm)
2228 {
2229
2230 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
2231
2232 iounmap(gtt->gsm);
2233 teardown_scratch_page(vm->dev);
2234 }
2235
2236 static int i915_gmch_probe(struct drm_device *dev,
2237 size_t *gtt_total,
2238 size_t *stolen,
2239 phys_addr_t *mappable_base,
2240 unsigned long *mappable_end)
2241 {
2242 struct drm_i915_private *dev_priv = dev->dev_private;
2243 int ret;
2244
2245 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2246 if (!ret) {
2247 DRM_ERROR("failed to set up gmch\n");
2248 return -EIO;
2249 }
2250
2251 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
2252
2253 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
2254 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
2255
2256 if (unlikely(dev_priv->gtt.do_idle_maps))
2257 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2258
2259 return 0;
2260 }
2261
2262 static void i915_gmch_remove(struct i915_address_space *vm)
2263 {
2264 intel_gmch_remove();
2265 }
2266
2267 int i915_gem_gtt_init(struct drm_device *dev)
2268 {
2269 struct drm_i915_private *dev_priv = dev->dev_private;
2270 struct i915_gtt *gtt = &dev_priv->gtt;
2271 int ret;
2272
2273 if (INTEL_INFO(dev)->gen <= 5) {
2274 gtt->gtt_probe = i915_gmch_probe;
2275 gtt->base.cleanup = i915_gmch_remove;
2276 } else if (INTEL_INFO(dev)->gen < 8) {
2277 gtt->gtt_probe = gen6_gmch_probe;
2278 gtt->base.cleanup = gen6_gmch_remove;
2279 if (IS_HASWELL(dev) && dev_priv->ellc_size)
2280 gtt->base.pte_encode = iris_pte_encode;
2281 else if (IS_HASWELL(dev))
2282 gtt->base.pte_encode = hsw_pte_encode;
2283 else if (IS_VALLEYVIEW(dev))
2284 gtt->base.pte_encode = byt_pte_encode;
2285 else if (INTEL_INFO(dev)->gen >= 7)
2286 gtt->base.pte_encode = ivb_pte_encode;
2287 else
2288 gtt->base.pte_encode = snb_pte_encode;
2289 } else {
2290 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2291 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
2292 }
2293
2294 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
2295 &gtt->mappable_base, &gtt->mappable_end);
2296 if (ret)
2297 return ret;
2298
2299 gtt->base.dev = dev;
2300
2301 /* GMADR is the PCI mmio aperture into the global GTT. */
2302 DRM_INFO("Memory usable by graphics device = %zdM\n",
2303 gtt->base.total >> 20);
2304 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2305 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2306 #ifdef CONFIG_INTEL_IOMMU
2307 if (intel_iommu_gfx_mapped)
2308 DRM_INFO("VT-d active for gfx access\n");
2309 #endif
2310 /*
2311 * i915.enable_ppgtt is read-only, so do an early pass to validate the
2312 * user's requested state against the hardware/driver capabilities. We
2313 * do this now so that we can print out any log messages once rather
2314 * than every time we check intel_enable_ppgtt().
2315 */
2316 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2317 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
2318
2319 return 0;
2320 }
2321
2322 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2323 struct i915_address_space *vm,
2324 const struct i915_ggtt_view *view)
2325 {
2326 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2327 if (vma == NULL)
2328 return ERR_PTR(-ENOMEM);
2329
2330 INIT_LIST_HEAD(&vma->vma_link);
2331 INIT_LIST_HEAD(&vma->mm_list);
2332 INIT_LIST_HEAD(&vma->exec_list);
2333 vma->vm = vm;
2334 vma->obj = obj;
2335 vma->ggtt_view = *view;
2336
2337 if (INTEL_INFO(vm->dev)->gen >= 6) {
2338 if (i915_is_ggtt(vm)) {
2339 vma->unbind_vma = ggtt_unbind_vma;
2340 vma->bind_vma = ggtt_bind_vma;
2341 } else {
2342 vma->unbind_vma = ppgtt_unbind_vma;
2343 vma->bind_vma = ppgtt_bind_vma;
2344 }
2345 } else {
2346 BUG_ON(!i915_is_ggtt(vm));
2347 vma->unbind_vma = i915_ggtt_unbind_vma;
2348 vma->bind_vma = i915_ggtt_bind_vma;
2349 }
2350
2351 list_add_tail(&vma->vma_link, &obj->vma_list);
2352 if (!i915_is_ggtt(vm))
2353 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
2354
2355 return vma;
2356 }
2357
2358 struct i915_vma *
2359 i915_gem_obj_lookup_or_create_vma_view(struct drm_i915_gem_object *obj,
2360 struct i915_address_space *vm,
2361 const struct i915_ggtt_view *view)
2362 {
2363 struct i915_vma *vma;
2364
2365 vma = i915_gem_obj_to_vma_view(obj, vm, view);
2366 if (!vma)
2367 vma = __i915_gem_vma_create(obj, vm, view);
2368
2369 return vma;
2370 }
2371
2372 static inline
2373 int i915_get_vma_pages(struct i915_vma *vma)
2374 {
2375 if (vma->ggtt_view.pages)
2376 return 0;
2377
2378 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
2379 vma->ggtt_view.pages = vma->obj->pages;
2380 else
2381 WARN_ONCE(1, "GGTT view %u not implemented!\n",
2382 vma->ggtt_view.type);
2383
2384 if (!vma->ggtt_view.pages) {
2385 DRM_ERROR("Failed to get pages for VMA view type %u!\n",
2386 vma->ggtt_view.type);
2387 return -EINVAL;
2388 }
2389
2390 return 0;
2391 }
2392
2393 /**
2394 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
2395 * @vma: VMA to map
2396 * @cache_level: mapping cache level
2397 * @flags: flags like global or local mapping
2398 *
2399 * DMA addresses are taken from the scatter-gather table of this object (or of
2400 * this VMA in case of non-default GGTT views) and PTE entries set up.
2401 * Note that DMA addresses are also the only part of the SG table we care about.
2402 */
2403 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
2404 u32 flags)
2405 {
2406 int ret = i915_get_vma_pages(vma);
2407
2408 if (ret)
2409 return ret;
2410
2411 vma->bind_vma(vma, cache_level, flags);
2412
2413 return 0;
2414 }
Linux kernel - Deliverables
This page took 0.115984 seconds and 5 git commands to generate.