2 * drivers/gpu/drm/omapdrm/omap_gem.c
4 * Copyright (C) 2011 Texas Instruments
5 * Author: Rob Clark <rob.clark@linaro.org>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published by
9 * the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
21 #include <linux/spinlock.h>
22 #include <linux/shmem_fs.h>
23 #include <drm/drm_vma_manager.h>
26 #include "omap_dmm_tiler.h"
28 /* remove these once drm core helpers are merged */
29 struct page
**_drm_gem_get_pages(struct drm_gem_object
*obj
, gfp_t gfpmask
);
30 void _drm_gem_put_pages(struct drm_gem_object
*obj
, struct page
**pages
,
31 bool dirty
, bool accessed
);
32 int _drm_gem_create_mmap_offset_size(struct drm_gem_object
*obj
, size_t size
);
35 * GEM buffer object implementation.
38 #define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
40 /* note: we use upper 8 bits of flags for driver-internal flags: */
41 #define OMAP_BO_DMA 0x01000000 /* actually is physically contiguous */
42 #define OMAP_BO_EXT_SYNC 0x02000000 /* externally allocated sync object */
43 #define OMAP_BO_EXT_MEM 0x04000000 /* externally allocated memory */
46 struct omap_gem_object
{
47 struct drm_gem_object base
;
49 struct list_head mm_list
;
53 /** width/height for tiled formats (rounded up to slot boundaries) */
54 uint16_t width
, height
;
56 /** roll applied when mapping to DMM */
60 * If buffer is allocated physically contiguous, the OMAP_BO_DMA flag
61 * is set and the paddr is valid. Also if the buffer is remapped in
62 * TILER and paddr_cnt > 0, then paddr is valid. But if you are using
63 * the physical address and OMAP_BO_DMA is not set, then you should
64 * be going thru omap_gem_{get,put}_paddr() to ensure the mapping is
65 * not removed from under your feet.
67 * Note that OMAP_BO_SCANOUT is a hint from userspace that DMA capable
68 * buffer is requested, but doesn't mean that it is. Use the
69 * OMAP_BO_DMA flag to determine if the buffer has a DMA capable
80 * tiler block used when buffer is remapped in DMM/TILER.
82 struct tiler_block
*block
;
85 * Array of backing pages, if allocated. Note that pages are never
86 * allocated for buffers originally allocated from contiguous memory
90 /** addresses corresponding to pages in above array */
94 * Virtual address, if mapped.
99 * sync-object allocated on demand (if needed)
101 * Per-buffer sync-object for tracking pending and completed hw/dma
102 * read and write operations. The layout in memory is dictated by
103 * the SGX firmware, which uses this information to stall the command
104 * stream if a surface is not ready yet.
106 * Note that when buffer is used by SGX, the sync-object needs to be
107 * allocated from a special heap of sync-objects. This way many sync
108 * objects can be packed in a page, and not waste GPU virtual address
109 * space. Because of this we have to have a omap_gem_set_sync_object()
110 * API to allow replacement of the syncobj after it has (potentially)
111 * already been allocated. A bit ugly but I haven't thought of a
112 * better alternative.
115 uint32_t write_pending
;
116 uint32_t write_complete
;
117 uint32_t read_pending
;
118 uint32_t read_complete
;
122 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
);
123 static uint64_t mmap_offset(struct drm_gem_object
*obj
);
125 /* To deal with userspace mmap'ings of 2d tiled buffers, which (a) are
126 * not necessarily pinned in TILER all the time, and (b) when they are
127 * they are not necessarily page aligned, we reserve one or more small
128 * regions in each of the 2d containers to use as a user-GART where we
129 * can create a second page-aligned mapping of parts of the buffer
130 * being accessed from userspace.
132 * Note that we could optimize slightly when we know that multiple
133 * tiler containers are backed by the same PAT.. but I'll leave that
136 #define NUM_USERGART_ENTRIES 2
137 struct usergart_entry
{
138 struct tiler_block
*block
; /* the reserved tiler block */
140 struct drm_gem_object
*obj
; /* the current pinned obj */
141 pgoff_t obj_pgoff
; /* page offset of obj currently
145 struct usergart_entry entry
[NUM_USERGART_ENTRIES
];
146 int height
; /* height in rows */
147 int height_shift
; /* ilog2(height in rows) */
148 int slot_shift
; /* ilog2(width per slot) */
149 int stride_pfn
; /* stride in pages */
150 int last
; /* index of last used entry */
153 static void evict_entry(struct drm_gem_object
*obj
,
154 enum tiler_fmt fmt
, struct usergart_entry
*entry
)
156 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
157 int n
= usergart
[fmt
].height
;
158 size_t size
= PAGE_SIZE
* n
;
159 loff_t off
= mmap_offset(obj
) +
160 (entry
->obj_pgoff
<< PAGE_SHIFT
);
161 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
165 /* if stride > than PAGE_SIZE then sparse mapping: */
166 for (i
= n
; i
> 0; i
--) {
167 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
169 off
+= PAGE_SIZE
* m
;
172 unmap_mapping_range(obj
->dev
->anon_inode
->i_mapping
,
179 /* Evict a buffer from usergart, if it is mapped there */
180 static void evict(struct drm_gem_object
*obj
)
182 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
184 if (omap_obj
->flags
& OMAP_BO_TILED
) {
185 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
191 for (i
= 0; i
< NUM_USERGART_ENTRIES
; i
++) {
192 struct usergart_entry
*entry
= &usergart
[fmt
].entry
[i
];
193 if (entry
->obj
== obj
)
194 evict_entry(obj
, fmt
, entry
);
199 /* GEM objects can either be allocated from contiguous memory (in which
200 * case obj->filp==NULL), or w/ shmem backing (obj->filp!=NULL). But non
201 * contiguous buffers can be remapped in TILER/DMM if they need to be
202 * contiguous... but we don't do this all the time to reduce pressure
203 * on TILER/DMM space when we know at allocation time that the buffer
204 * will need to be scanned out.
206 static inline bool is_shmem(struct drm_gem_object
*obj
)
208 return obj
->filp
!= NULL
;
212 * shmem buffers that are mapped cached can simulate coherency via using
213 * page faulting to keep track of dirty pages
215 static inline bool is_cached_coherent(struct drm_gem_object
*obj
)
217 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
218 return is_shmem(obj
) &&
219 ((omap_obj
->flags
& OMAP_BO_CACHE_MASK
) == OMAP_BO_CACHED
);
222 static DEFINE_SPINLOCK(sync_lock
);
224 /** ensure backing pages are allocated */
225 static int omap_gem_attach_pages(struct drm_gem_object
*obj
)
227 struct drm_device
*dev
= obj
->dev
;
228 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
230 int npages
= obj
->size
>> PAGE_SHIFT
;
234 WARN_ON(omap_obj
->pages
);
236 pages
= drm_gem_get_pages(obj
);
238 dev_err(obj
->dev
->dev
, "could not get pages: %ld\n", PTR_ERR(pages
));
239 return PTR_ERR(pages
);
242 /* for non-cached buffers, ensure the new pages are clean because
243 * DSS, GPU, etc. are not cache coherent:
245 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
246 addrs
= kmalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
252 for (i
= 0; i
< npages
; i
++) {
253 addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
],
254 0, PAGE_SIZE
, DMA_BIDIRECTIONAL
);
257 addrs
= kzalloc(npages
* sizeof(*addrs
), GFP_KERNEL
);
264 omap_obj
->addrs
= addrs
;
265 omap_obj
->pages
= pages
;
270 drm_gem_put_pages(obj
, pages
, true, false);
275 /** release backing pages */
276 static void omap_gem_detach_pages(struct drm_gem_object
*obj
)
278 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
280 /* for non-cached buffers, ensure the new pages are clean because
281 * DSS, GPU, etc. are not cache coherent:
283 if (omap_obj
->flags
& (OMAP_BO_WC
|OMAP_BO_UNCACHED
)) {
284 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
285 for (i
= 0; i
< npages
; i
++) {
286 dma_unmap_page(obj
->dev
->dev
, omap_obj
->addrs
[i
],
287 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
291 kfree(omap_obj
->addrs
);
292 omap_obj
->addrs
= NULL
;
294 drm_gem_put_pages(obj
, omap_obj
->pages
, true, false);
295 omap_obj
->pages
= NULL
;
298 /* get buffer flags */
299 uint32_t omap_gem_flags(struct drm_gem_object
*obj
)
301 return to_omap_bo(obj
)->flags
;
304 /** get mmap offset */
305 static uint64_t mmap_offset(struct drm_gem_object
*obj
)
307 struct drm_device
*dev
= obj
->dev
;
311 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
313 /* Make it mmapable */
314 size
= omap_gem_mmap_size(obj
);
315 ret
= drm_gem_create_mmap_offset_size(obj
, size
);
317 dev_err(dev
->dev
, "could not allocate mmap offset\n");
321 return drm_vma_node_offset_addr(&obj
->vma_node
);
324 uint64_t omap_gem_mmap_offset(struct drm_gem_object
*obj
)
327 mutex_lock(&obj
->dev
->struct_mutex
);
328 offset
= mmap_offset(obj
);
329 mutex_unlock(&obj
->dev
->struct_mutex
);
334 size_t omap_gem_mmap_size(struct drm_gem_object
*obj
)
336 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
337 size_t size
= obj
->size
;
339 if (omap_obj
->flags
& OMAP_BO_TILED
) {
340 /* for tiled buffers, the virtual size has stride rounded up
341 * to 4kb.. (to hide the fact that row n+1 might start 16kb or
342 * 32kb later!). But we don't back the entire buffer with
343 * pages, only the valid picture part.. so need to adjust for
344 * this in the size used to mmap and generate mmap offset
346 size
= tiler_vsize(gem2fmt(omap_obj
->flags
),
347 omap_obj
->width
, omap_obj
->height
);
353 /* get tiled size, returns -EINVAL if not tiled buffer */
354 int omap_gem_tiled_size(struct drm_gem_object
*obj
, uint16_t *w
, uint16_t *h
)
356 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
357 if (omap_obj
->flags
& OMAP_BO_TILED
) {
358 *w
= omap_obj
->width
;
359 *h
= omap_obj
->height
;
365 /* Normal handling for the case of faulting in non-tiled buffers */
366 static int fault_1d(struct drm_gem_object
*obj
,
367 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
369 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
373 /* We don't use vmf->pgoff since that has the fake offset: */
374 pgoff
= ((unsigned long)vmf
->virtual_address
-
375 vma
->vm_start
) >> PAGE_SHIFT
;
377 if (omap_obj
->pages
) {
378 omap_gem_cpu_sync(obj
, pgoff
);
379 pfn
= page_to_pfn(omap_obj
->pages
[pgoff
]);
381 BUG_ON(!(omap_obj
->flags
& OMAP_BO_DMA
));
382 pfn
= (omap_obj
->paddr
>> PAGE_SHIFT
) + pgoff
;
385 VERB("Inserting %p pfn %lx, pa %lx", vmf
->virtual_address
,
386 pfn
, pfn
<< PAGE_SHIFT
);
388 return vm_insert_mixed(vma
, (unsigned long)vmf
->virtual_address
, pfn
);
391 /* Special handling for the case of faulting in 2d tiled buffers */
392 static int fault_2d(struct drm_gem_object
*obj
,
393 struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
395 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
396 struct usergart_entry
*entry
;
397 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
398 struct page
*pages
[64]; /* XXX is this too much to have on stack? */
400 pgoff_t pgoff
, base_pgoff
;
405 * Note the height of the slot is also equal to the number of pages
406 * that need to be mapped in to fill 4kb wide CPU page. If the slot
407 * height is 64, then 64 pages fill a 4kb wide by 64 row region.
409 const int n
= usergart
[fmt
].height
;
410 const int n_shift
= usergart
[fmt
].height_shift
;
413 * If buffer width in bytes > PAGE_SIZE then the virtual stride is
414 * rounded up to next multiple of PAGE_SIZE.. this need to be taken
415 * into account in some of the math, so figure out virtual stride
418 const int m
= 1 + ((omap_obj
->width
<< fmt
) / PAGE_SIZE
);
420 /* We don't use vmf->pgoff since that has the fake offset: */
421 pgoff
= ((unsigned long)vmf
->virtual_address
-
422 vma
->vm_start
) >> PAGE_SHIFT
;
425 * Actual address we start mapping at is rounded down to previous slot
426 * boundary in the y direction:
428 base_pgoff
= round_down(pgoff
, m
<< n_shift
);
430 /* figure out buffer width in slots */
431 slots
= omap_obj
->width
>> usergart
[fmt
].slot_shift
;
433 vaddr
= vmf
->virtual_address
- ((pgoff
- base_pgoff
) << PAGE_SHIFT
);
435 entry
= &usergart
[fmt
].entry
[usergart
[fmt
].last
];
437 /* evict previous buffer using this usergart entry, if any: */
439 evict_entry(entry
->obj
, fmt
, entry
);
442 entry
->obj_pgoff
= base_pgoff
;
444 /* now convert base_pgoff to phys offset from virt offset: */
445 base_pgoff
= (base_pgoff
>> n_shift
) * slots
;
447 /* for wider-than 4k.. figure out which part of the slot-row we want: */
450 entry
->obj_pgoff
+= off
;
452 slots
= min(slots
- (off
<< n_shift
), n
);
453 base_pgoff
+= off
<< n_shift
;
454 vaddr
+= off
<< PAGE_SHIFT
;
458 * Map in pages. Beyond the valid pixel part of the buffer, we set
459 * pages[i] to NULL to get a dummy page mapped in.. if someone
460 * reads/writes it they will get random/undefined content, but at
461 * least it won't be corrupting whatever other random page used to
462 * be mapped in, or other undefined behavior.
464 memcpy(pages
, &omap_obj
->pages
[base_pgoff
],
465 sizeof(struct page
*) * slots
);
466 memset(pages
+ slots
, 0,
467 sizeof(struct page
*) * (n
- slots
));
469 ret
= tiler_pin(entry
->block
, pages
, ARRAY_SIZE(pages
), 0, true);
471 dev_err(obj
->dev
->dev
, "failed to pin: %d\n", ret
);
475 pfn
= entry
->paddr
>> PAGE_SHIFT
;
477 VERB("Inserting %p pfn %lx, pa %lx", vmf
->virtual_address
,
478 pfn
, pfn
<< PAGE_SHIFT
);
480 for (i
= n
; i
> 0; i
--) {
481 vm_insert_mixed(vma
, (unsigned long)vaddr
, pfn
);
482 pfn
+= usergart
[fmt
].stride_pfn
;
483 vaddr
+= PAGE_SIZE
* m
;
486 /* simple round-robin: */
487 usergart
[fmt
].last
= (usergart
[fmt
].last
+ 1) % NUM_USERGART_ENTRIES
;
493 * omap_gem_fault - pagefault handler for GEM objects
494 * @vma: the VMA of the GEM object
497 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
498 * does most of the work for us including the actual map/unmap calls
499 * but we need to do the actual page work.
501 * The VMA was set up by GEM. In doing so it also ensured that the
502 * vma->vm_private_data points to the GEM object that is backing this
505 int omap_gem_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
507 struct drm_gem_object
*obj
= vma
->vm_private_data
;
508 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
509 struct drm_device
*dev
= obj
->dev
;
513 /* Make sure we don't parallel update on a fault, nor move or remove
514 * something from beneath our feet
516 mutex_lock(&dev
->struct_mutex
);
518 /* if a shmem backed object, make sure we have pages attached now */
519 ret
= get_pages(obj
, &pages
);
523 /* where should we do corresponding put_pages().. we are mapping
524 * the original page, rather than thru a GART, so we can't rely
525 * on eviction to trigger this. But munmap() or all mappings should
526 * probably trigger put_pages()?
529 if (omap_obj
->flags
& OMAP_BO_TILED
)
530 ret
= fault_2d(obj
, vma
, vmf
);
532 ret
= fault_1d(obj
, vma
, vmf
);
536 mutex_unlock(&dev
->struct_mutex
);
541 return VM_FAULT_NOPAGE
;
545 return VM_FAULT_SIGBUS
;
549 /** We override mainly to fix up some of the vm mapping flags.. */
550 int omap_gem_mmap(struct file
*filp
, struct vm_area_struct
*vma
)
554 ret
= drm_gem_mmap(filp
, vma
);
556 DBG("mmap failed: %d", ret
);
560 return omap_gem_mmap_obj(vma
->vm_private_data
, vma
);
563 int omap_gem_mmap_obj(struct drm_gem_object
*obj
,
564 struct vm_area_struct
*vma
)
566 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
568 vma
->vm_flags
&= ~VM_PFNMAP
;
569 vma
->vm_flags
|= VM_MIXEDMAP
;
571 if (omap_obj
->flags
& OMAP_BO_WC
) {
572 vma
->vm_page_prot
= pgprot_writecombine(vm_get_page_prot(vma
->vm_flags
));
573 } else if (omap_obj
->flags
& OMAP_BO_UNCACHED
) {
574 vma
->vm_page_prot
= pgprot_noncached(vm_get_page_prot(vma
->vm_flags
));
577 * We do have some private objects, at least for scanout buffers
578 * on hardware without DMM/TILER. But these are allocated write-
581 if (WARN_ON(!obj
->filp
))
585 * Shunt off cached objs to shmem file so they have their own
586 * address_space (so unmap_mapping_range does what we want,
587 * in particular in the case of mmap'd dmabufs)
591 vma
->vm_file
= get_file(obj
->filp
);
593 vma
->vm_page_prot
= vm_get_page_prot(vma
->vm_flags
);
601 * omap_gem_dumb_create - create a dumb buffer
602 * @drm_file: our client file
604 * @args: the requested arguments copied from userspace
606 * Allocate a buffer suitable for use for a frame buffer of the
607 * form described by user space. Give userspace a handle by which
610 int omap_gem_dumb_create(struct drm_file
*file
, struct drm_device
*dev
,
611 struct drm_mode_create_dumb
*args
)
613 union omap_gem_size gsize
;
615 args
->pitch
= align_pitch(0, args
->width
, args
->bpp
);
616 args
->size
= PAGE_ALIGN(args
->pitch
* args
->height
);
618 gsize
= (union omap_gem_size
){
622 return omap_gem_new_handle(dev
, file
, gsize
,
623 OMAP_BO_SCANOUT
| OMAP_BO_WC
, &args
->handle
);
627 * omap_gem_dumb_map - buffer mapping for dumb interface
628 * @file: our drm client file
630 * @handle: GEM handle to the object (from dumb_create)
632 * Do the necessary setup to allow the mapping of the frame buffer
633 * into user memory. We don't have to do much here at the moment.
635 int omap_gem_dumb_map_offset(struct drm_file
*file
, struct drm_device
*dev
,
636 uint32_t handle
, uint64_t *offset
)
638 struct drm_gem_object
*obj
;
641 /* GEM does all our handle to object mapping */
642 obj
= drm_gem_object_lookup(dev
, file
, handle
);
648 *offset
= omap_gem_mmap_offset(obj
);
650 drm_gem_object_unreference_unlocked(obj
);
656 /* Set scrolling position. This allows us to implement fast scrolling
659 * Call only from non-atomic contexts.
661 int omap_gem_roll(struct drm_gem_object
*obj
, uint32_t roll
)
663 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
664 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
668 dev_err(obj
->dev
->dev
, "invalid roll: %d\n", roll
);
672 omap_obj
->roll
= roll
;
674 mutex_lock(&obj
->dev
->struct_mutex
);
676 /* if we aren't mapped yet, we don't need to do anything */
677 if (omap_obj
->block
) {
679 ret
= get_pages(obj
, &pages
);
682 ret
= tiler_pin(omap_obj
->block
, pages
, npages
, roll
, true);
684 dev_err(obj
->dev
->dev
, "could not repin: %d\n", ret
);
688 mutex_unlock(&obj
->dev
->struct_mutex
);
693 /* Sync the buffer for CPU access.. note pages should already be
694 * attached, ie. omap_gem_get_pages()
696 void omap_gem_cpu_sync(struct drm_gem_object
*obj
, int pgoff
)
698 struct drm_device
*dev
= obj
->dev
;
699 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
701 if (is_cached_coherent(obj
) && omap_obj
->addrs
[pgoff
]) {
702 dma_unmap_page(dev
->dev
, omap_obj
->addrs
[pgoff
],
703 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
704 omap_obj
->addrs
[pgoff
] = 0;
708 /* sync the buffer for DMA access */
709 void omap_gem_dma_sync(struct drm_gem_object
*obj
,
710 enum dma_data_direction dir
)
712 struct drm_device
*dev
= obj
->dev
;
713 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
715 if (is_cached_coherent(obj
)) {
716 int i
, npages
= obj
->size
>> PAGE_SHIFT
;
717 struct page
**pages
= omap_obj
->pages
;
720 for (i
= 0; i
< npages
; i
++) {
721 if (!omap_obj
->addrs
[i
]) {
722 omap_obj
->addrs
[i
] = dma_map_page(dev
->dev
, pages
[i
], 0,
723 PAGE_SIZE
, DMA_BIDIRECTIONAL
);
729 unmap_mapping_range(obj
->filp
->f_mapping
, 0,
730 omap_gem_mmap_size(obj
), 1);
735 /* Get physical address for DMA.. if 'remap' is true, and the buffer is not
736 * already contiguous, remap it to pin in physically contiguous memory.. (ie.
739 int omap_gem_get_paddr(struct drm_gem_object
*obj
,
740 dma_addr_t
*paddr
, bool remap
)
742 struct omap_drm_private
*priv
= obj
->dev
->dev_private
;
743 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
746 mutex_lock(&obj
->dev
->struct_mutex
);
748 if (remap
&& is_shmem(obj
) && priv
->has_dmm
) {
749 if (omap_obj
->paddr_cnt
== 0) {
751 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
752 enum tiler_fmt fmt
= gem2fmt(omap_obj
->flags
);
753 struct tiler_block
*block
;
755 BUG_ON(omap_obj
->block
);
757 ret
= get_pages(obj
, &pages
);
761 if (omap_obj
->flags
& OMAP_BO_TILED
) {
762 block
= tiler_reserve_2d(fmt
,
764 omap_obj
->height
, 0);
766 block
= tiler_reserve_1d(obj
->size
);
770 ret
= PTR_ERR(block
);
771 dev_err(obj
->dev
->dev
,
772 "could not remap: %d (%d)\n", ret
, fmt
);
776 /* TODO: enable async refill.. */
777 ret
= tiler_pin(block
, pages
, npages
,
778 omap_obj
->roll
, true);
780 tiler_release(block
);
781 dev_err(obj
->dev
->dev
,
782 "could not pin: %d\n", ret
);
786 omap_obj
->paddr
= tiler_ssptr(block
);
787 omap_obj
->block
= block
;
789 DBG("got paddr: %pad", &omap_obj
->paddr
);
792 omap_obj
->paddr_cnt
++;
794 *paddr
= omap_obj
->paddr
;
795 } else if (omap_obj
->flags
& OMAP_BO_DMA
) {
796 *paddr
= omap_obj
->paddr
;
803 mutex_unlock(&obj
->dev
->struct_mutex
);
808 /* Release physical address, when DMA is no longer being performed.. this
809 * could potentially unpin and unmap buffers from TILER
811 int omap_gem_put_paddr(struct drm_gem_object
*obj
)
813 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
816 mutex_lock(&obj
->dev
->struct_mutex
);
817 if (omap_obj
->paddr_cnt
> 0) {
818 omap_obj
->paddr_cnt
--;
819 if (omap_obj
->paddr_cnt
== 0) {
820 ret
= tiler_unpin(omap_obj
->block
);
822 dev_err(obj
->dev
->dev
,
823 "could not unpin pages: %d\n", ret
);
826 ret
= tiler_release(omap_obj
->block
);
828 dev_err(obj
->dev
->dev
,
829 "could not release unmap: %d\n", ret
);
831 omap_obj
->block
= NULL
;
835 mutex_unlock(&obj
->dev
->struct_mutex
);
839 /* Get rotated scanout address (only valid if already pinned), at the
840 * specified orientation and x,y offset from top-left corner of buffer
841 * (only valid for tiled 2d buffers)
843 int omap_gem_rotated_paddr(struct drm_gem_object
*obj
, uint32_t orient
,
844 int x
, int y
, dma_addr_t
*paddr
)
846 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
849 mutex_lock(&obj
->dev
->struct_mutex
);
850 if ((omap_obj
->paddr_cnt
> 0) && omap_obj
->block
&&
851 (omap_obj
->flags
& OMAP_BO_TILED
)) {
852 *paddr
= tiler_tsptr(omap_obj
->block
, orient
, x
, y
);
855 mutex_unlock(&obj
->dev
->struct_mutex
);
859 /* Get tiler stride for the buffer (only valid for 2d tiled buffers) */
860 int omap_gem_tiled_stride(struct drm_gem_object
*obj
, uint32_t orient
)
862 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
864 if (omap_obj
->flags
& OMAP_BO_TILED
)
865 ret
= tiler_stride(gem2fmt(omap_obj
->flags
), orient
);
869 /* acquire pages when needed (for example, for DMA where physically
870 * contiguous buffer is not required
872 static int get_pages(struct drm_gem_object
*obj
, struct page
***pages
)
874 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
877 if (is_shmem(obj
) && !omap_obj
->pages
) {
878 ret
= omap_gem_attach_pages(obj
);
880 dev_err(obj
->dev
->dev
, "could not attach pages\n");
885 /* TODO: even phys-contig.. we should have a list of pages? */
886 *pages
= omap_obj
->pages
;
891 /* if !remap, and we don't have pages backing, then fail, rather than
892 * increasing the pin count (which we don't really do yet anyways,
893 * because we don't support swapping pages back out). And 'remap'
894 * might not be quite the right name, but I wanted to keep it working
895 * similarly to omap_gem_get_paddr(). Note though that mutex is not
896 * aquired if !remap (because this can be called in atomic ctxt),
897 * but probably omap_gem_get_paddr() should be changed to work in the
898 * same way. If !remap, a matching omap_gem_put_pages() call is not
899 * required (and should not be made).
901 int omap_gem_get_pages(struct drm_gem_object
*obj
, struct page
***pages
,
906 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
907 if (!omap_obj
->pages
)
909 *pages
= omap_obj
->pages
;
912 mutex_lock(&obj
->dev
->struct_mutex
);
913 ret
= get_pages(obj
, pages
);
914 mutex_unlock(&obj
->dev
->struct_mutex
);
918 /* release pages when DMA no longer being performed */
919 int omap_gem_put_pages(struct drm_gem_object
*obj
)
921 /* do something here if we dynamically attach/detach pages.. at
922 * least they would no longer need to be pinned if everyone has
923 * released the pages..
928 /* Get kernel virtual address for CPU access.. this more or less only
929 * exists for omap_fbdev. This should be called with struct_mutex
932 void *omap_gem_vaddr(struct drm_gem_object
*obj
)
934 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
935 WARN_ON(!mutex_is_locked(&obj
->dev
->struct_mutex
));
936 if (!omap_obj
->vaddr
) {
938 int ret
= get_pages(obj
, &pages
);
941 omap_obj
->vaddr
= vmap(pages
, obj
->size
>> PAGE_SHIFT
,
942 VM_MAP
, pgprot_writecombine(PAGE_KERNEL
));
944 return omap_obj
->vaddr
;
948 /* re-pin objects in DMM in resume path: */
949 int omap_gem_resume(struct device
*dev
)
951 struct drm_device
*drm_dev
= dev_get_drvdata(dev
);
952 struct omap_drm_private
*priv
= drm_dev
->dev_private
;
953 struct omap_gem_object
*omap_obj
;
956 list_for_each_entry(omap_obj
, &priv
->obj_list
, mm_list
) {
957 if (omap_obj
->block
) {
958 struct drm_gem_object
*obj
= &omap_obj
->base
;
959 uint32_t npages
= obj
->size
>> PAGE_SHIFT
;
960 WARN_ON(!omap_obj
->pages
); /* this can't happen */
961 ret
= tiler_pin(omap_obj
->block
,
962 omap_obj
->pages
, npages
,
963 omap_obj
->roll
, true);
965 dev_err(dev
, "could not repin: %d\n", ret
);
975 #ifdef CONFIG_DEBUG_FS
976 void omap_gem_describe(struct drm_gem_object
*obj
, struct seq_file
*m
)
978 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
981 off
= drm_vma_node_start(&obj
->vma_node
);
983 seq_printf(m
, "%08x: %2d (%2d) %08llx %pad (%2d) %p %4d",
984 omap_obj
->flags
, obj
->name
, obj
->refcount
.refcount
.counter
,
985 off
, &omap_obj
->paddr
, omap_obj
->paddr_cnt
,
986 omap_obj
->vaddr
, omap_obj
->roll
);
988 if (omap_obj
->flags
& OMAP_BO_TILED
) {
989 seq_printf(m
, " %dx%d", omap_obj
->width
, omap_obj
->height
);
990 if (omap_obj
->block
) {
991 struct tcm_area
*area
= &omap_obj
->block
->area
;
992 seq_printf(m
, " (%dx%d, %dx%d)",
993 area
->p0
.x
, area
->p0
.y
,
994 area
->p1
.x
, area
->p1
.y
);
997 seq_printf(m
, " %d", obj
->size
);
1000 seq_printf(m
, "\n");
1003 void omap_gem_describe_objects(struct list_head
*list
, struct seq_file
*m
)
1005 struct omap_gem_object
*omap_obj
;
1009 list_for_each_entry(omap_obj
, list
, mm_list
) {
1010 struct drm_gem_object
*obj
= &omap_obj
->base
;
1012 omap_gem_describe(obj
, m
);
1017 seq_printf(m
, "Total %d objects, %zu bytes\n", count
, size
);
1021 /* Buffer Synchronization:
1024 struct omap_gem_sync_waiter
{
1025 struct list_head list
;
1026 struct omap_gem_object
*omap_obj
;
1027 enum omap_gem_op op
;
1028 uint32_t read_target
, write_target
;
1029 /* notify called w/ sync_lock held */
1030 void (*notify
)(void *arg
);
1034 /* list of omap_gem_sync_waiter.. the notify fxn gets called back when
1035 * the read and/or write target count is achieved which can call a user
1036 * callback (ex. to kick 3d and/or 2d), wakeup blocked task (prep for
1039 static LIST_HEAD(waiters
);
1041 static inline bool is_waiting(struct omap_gem_sync_waiter
*waiter
)
1043 struct omap_gem_object
*omap_obj
= waiter
->omap_obj
;
1044 if ((waiter
->op
& OMAP_GEM_READ
) &&
1045 (omap_obj
->sync
->write_complete
< waiter
->write_target
))
1047 if ((waiter
->op
& OMAP_GEM_WRITE
) &&
1048 (omap_obj
->sync
->read_complete
< waiter
->read_target
))
1053 /* macro for sync debug.. */
1055 #define SYNC(fmt, ...) do { if (SYNCDBG) \
1056 printk(KERN_ERR "%s:%d: "fmt"\n", \
1057 __func__, __LINE__, ##__VA_ARGS__); \
1061 static void sync_op_update(void)
1063 struct omap_gem_sync_waiter
*waiter
, *n
;
1064 list_for_each_entry_safe(waiter
, n
, &waiters
, list
) {
1065 if (!is_waiting(waiter
)) {
1066 list_del(&waiter
->list
);
1067 SYNC("notify: %p", waiter
);
1068 waiter
->notify(waiter
->arg
);
1074 static inline int sync_op(struct drm_gem_object
*obj
,
1075 enum omap_gem_op op
, bool start
)
1077 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1080 spin_lock(&sync_lock
);
1082 if (!omap_obj
->sync
) {
1083 omap_obj
->sync
= kzalloc(sizeof(*omap_obj
->sync
), GFP_ATOMIC
);
1084 if (!omap_obj
->sync
) {
1091 if (op
& OMAP_GEM_READ
)
1092 omap_obj
->sync
->read_pending
++;
1093 if (op
& OMAP_GEM_WRITE
)
1094 omap_obj
->sync
->write_pending
++;
1096 if (op
& OMAP_GEM_READ
)
1097 omap_obj
->sync
->read_complete
++;
1098 if (op
& OMAP_GEM_WRITE
)
1099 omap_obj
->sync
->write_complete
++;
1104 spin_unlock(&sync_lock
);
1109 /* it is a bit lame to handle updates in this sort of polling way, but
1110 * in case of PVR, the GPU can directly update read/write complete
1111 * values, and not really tell us which ones it updated.. this also
1112 * means that sync_lock is not quite sufficient. So we'll need to
1113 * do something a bit better when it comes time to add support for
1116 void omap_gem_op_update(void)
1118 spin_lock(&sync_lock
);
1120 spin_unlock(&sync_lock
);
1123 /* mark the start of read and/or write operation */
1124 int omap_gem_op_start(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1126 return sync_op(obj
, op
, true);
1129 int omap_gem_op_finish(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1131 return sync_op(obj
, op
, false);
1134 static DECLARE_WAIT_QUEUE_HEAD(sync_event
);
1136 static void sync_notify(void *arg
)
1138 struct task_struct
**waiter_task
= arg
;
1139 *waiter_task
= NULL
;
1140 wake_up_all(&sync_event
);
1143 int omap_gem_op_sync(struct drm_gem_object
*obj
, enum omap_gem_op op
)
1145 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1147 if (omap_obj
->sync
) {
1148 struct task_struct
*waiter_task
= current
;
1149 struct omap_gem_sync_waiter
*waiter
=
1150 kzalloc(sizeof(*waiter
), GFP_KERNEL
);
1155 waiter
->omap_obj
= omap_obj
;
1157 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1158 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1159 waiter
->notify
= sync_notify
;
1160 waiter
->arg
= &waiter_task
;
1162 spin_lock(&sync_lock
);
1163 if (is_waiting(waiter
)) {
1164 SYNC("waited: %p", waiter
);
1165 list_add_tail(&waiter
->list
, &waiters
);
1166 spin_unlock(&sync_lock
);
1167 ret
= wait_event_interruptible(sync_event
,
1168 (waiter_task
== NULL
));
1169 spin_lock(&sync_lock
);
1171 SYNC("interrupted: %p", waiter
);
1172 /* we were interrupted */
1173 list_del(&waiter
->list
);
1176 /* freed in sync_op_update() */
1180 spin_unlock(&sync_lock
);
1186 /* call fxn(arg), either synchronously or asynchronously if the op
1187 * is currently blocked.. fxn() can be called from any context
1189 * (TODO for now fxn is called back from whichever context calls
1190 * omap_gem_op_update().. but this could be better defined later
1193 * TODO more code in common w/ _sync()..
1195 int omap_gem_op_async(struct drm_gem_object
*obj
, enum omap_gem_op op
,
1196 void (*fxn
)(void *arg
), void *arg
)
1198 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1199 if (omap_obj
->sync
) {
1200 struct omap_gem_sync_waiter
*waiter
=
1201 kzalloc(sizeof(*waiter
), GFP_ATOMIC
);
1206 waiter
->omap_obj
= omap_obj
;
1208 waiter
->read_target
= omap_obj
->sync
->read_pending
;
1209 waiter
->write_target
= omap_obj
->sync
->write_pending
;
1210 waiter
->notify
= fxn
;
1213 spin_lock(&sync_lock
);
1214 if (is_waiting(waiter
)) {
1215 SYNC("waited: %p", waiter
);
1216 list_add_tail(&waiter
->list
, &waiters
);
1217 spin_unlock(&sync_lock
);
1221 spin_unlock(&sync_lock
);
1232 /* special API so PVR can update the buffer to use a sync-object allocated
1233 * from it's sync-obj heap. Only used for a newly allocated (from PVR's
1234 * perspective) sync-object, so we overwrite the new syncobj w/ values
1235 * from the already allocated syncobj (if there is one)
1237 int omap_gem_set_sync_object(struct drm_gem_object
*obj
, void *syncobj
)
1239 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1242 spin_lock(&sync_lock
);
1244 if ((omap_obj
->flags
& OMAP_BO_EXT_SYNC
) && !syncobj
) {
1245 /* clearing a previously set syncobj */
1246 syncobj
= kmemdup(omap_obj
->sync
, sizeof(*omap_obj
->sync
),
1252 omap_obj
->flags
&= ~OMAP_BO_EXT_SYNC
;
1253 omap_obj
->sync
= syncobj
;
1254 } else if (syncobj
&& !(omap_obj
->flags
& OMAP_BO_EXT_SYNC
)) {
1255 /* replacing an existing syncobj */
1256 if (omap_obj
->sync
) {
1257 memcpy(syncobj
, omap_obj
->sync
, sizeof(*omap_obj
->sync
));
1258 kfree(omap_obj
->sync
);
1260 omap_obj
->flags
|= OMAP_BO_EXT_SYNC
;
1261 omap_obj
->sync
= syncobj
;
1265 spin_unlock(&sync_lock
);
1269 /* don't call directly.. called from GEM core when it is time to actually
1272 void omap_gem_free_object(struct drm_gem_object
*obj
)
1274 struct drm_device
*dev
= obj
->dev
;
1275 struct omap_gem_object
*omap_obj
= to_omap_bo(obj
);
1279 WARN_ON(!mutex_is_locked(&dev
->struct_mutex
));
1281 list_del(&omap_obj
->mm_list
);
1283 drm_gem_free_mmap_offset(obj
);
1285 /* this means the object is still pinned.. which really should
1286 * not happen. I think..
1288 WARN_ON(omap_obj
->paddr_cnt
> 0);
1290 /* don't free externally allocated backing memory */
1291 if (!(omap_obj
->flags
& OMAP_BO_EXT_MEM
)) {
1292 if (omap_obj
->pages
)
1293 omap_gem_detach_pages(obj
);
1295 if (!is_shmem(obj
)) {
1296 dma_free_writecombine(dev
->dev
, obj
->size
,
1297 omap_obj
->vaddr
, omap_obj
->paddr
);
1298 } else if (omap_obj
->vaddr
) {
1299 vunmap(omap_obj
->vaddr
);
1303 /* don't free externally allocated syncobj */
1304 if (!(omap_obj
->flags
& OMAP_BO_EXT_SYNC
))
1305 kfree(omap_obj
->sync
);
1307 drm_gem_object_release(obj
);
1312 /* convenience method to construct a GEM buffer object, and userspace handle */
1313 int omap_gem_new_handle(struct drm_device
*dev
, struct drm_file
*file
,
1314 union omap_gem_size gsize
, uint32_t flags
, uint32_t *handle
)
1316 struct drm_gem_object
*obj
;
1319 obj
= omap_gem_new(dev
, gsize
, flags
);
1323 ret
= drm_gem_handle_create(file
, obj
, handle
);
1325 drm_gem_object_release(obj
);
1326 kfree(obj
); /* TODO isn't there a dtor to call? just copying i915 */
1330 /* drop reference from allocate - handle holds it now */
1331 drm_gem_object_unreference_unlocked(obj
);
1336 /* GEM buffer object constructor */
1337 struct drm_gem_object
*omap_gem_new(struct drm_device
*dev
,
1338 union omap_gem_size gsize
, uint32_t flags
)
1340 struct omap_drm_private
*priv
= dev
->dev_private
;
1341 struct omap_gem_object
*omap_obj
;
1342 struct drm_gem_object
*obj
= NULL
;
1343 struct address_space
*mapping
;
1347 if (flags
& OMAP_BO_TILED
) {
1349 dev_err(dev
->dev
, "Tiled buffers require DMM\n");
1353 /* tiled buffers are always shmem paged backed.. when they are
1354 * scanned out, they are remapped into DMM/TILER
1356 flags
&= ~OMAP_BO_SCANOUT
;
1358 /* currently don't allow cached buffers.. there is some caching
1359 * stuff that needs to be handled better
1361 flags
&= ~(OMAP_BO_CACHED
|OMAP_BO_UNCACHED
);
1362 flags
|= OMAP_BO_WC
;
1364 /* align dimensions to slot boundaries... */
1365 tiler_align(gem2fmt(flags
),
1366 &gsize
.tiled
.width
, &gsize
.tiled
.height
);
1368 /* ...and calculate size based on aligned dimensions */
1369 size
= tiler_size(gem2fmt(flags
),
1370 gsize
.tiled
.width
, gsize
.tiled
.height
);
1372 size
= PAGE_ALIGN(gsize
.bytes
);
1375 omap_obj
= kzalloc(sizeof(*omap_obj
), GFP_KERNEL
);
1379 list_add(&omap_obj
->mm_list
, &priv
->obj_list
);
1381 obj
= &omap_obj
->base
;
1383 if ((flags
& OMAP_BO_SCANOUT
) && !priv
->has_dmm
) {
1384 /* attempt to allocate contiguous memory if we don't
1385 * have DMM for remappign discontiguous buffers
1387 omap_obj
->vaddr
= dma_alloc_writecombine(dev
->dev
, size
,
1388 &omap_obj
->paddr
, GFP_KERNEL
);
1389 if (omap_obj
->vaddr
)
1390 flags
|= OMAP_BO_DMA
;
1394 omap_obj
->flags
= flags
;
1396 if (flags
& OMAP_BO_TILED
) {
1397 omap_obj
->width
= gsize
.tiled
.width
;
1398 omap_obj
->height
= gsize
.tiled
.height
;
1401 if (flags
& (OMAP_BO_DMA
|OMAP_BO_EXT_MEM
)) {
1402 drm_gem_private_object_init(dev
, obj
, size
);
1404 ret
= drm_gem_object_init(dev
, obj
, size
);
1408 mapping
= file_inode(obj
->filp
)->i_mapping
;
1409 mapping_set_gfp_mask(mapping
, GFP_USER
| __GFP_DMA32
);
1416 omap_gem_free_object(obj
);
1421 /* init/cleanup.. if DMM is used, we need to set some stuff up.. */
1422 void omap_gem_init(struct drm_device
*dev
)
1424 struct omap_drm_private
*priv
= dev
->dev_private
;
1425 const enum tiler_fmt fmts
[] = {
1426 TILFMT_8BIT
, TILFMT_16BIT
, TILFMT_32BIT
1430 if (!dmm_is_available()) {
1431 /* DMM only supported on OMAP4 and later, so this isn't fatal */
1432 dev_warn(dev
->dev
, "DMM not available, disable DMM support\n");
1436 usergart
= kcalloc(3, sizeof(*usergart
), GFP_KERNEL
);
1440 /* reserve 4k aligned/wide regions for userspace mappings: */
1441 for (i
= 0; i
< ARRAY_SIZE(fmts
); i
++) {
1442 uint16_t h
= 1, w
= PAGE_SIZE
>> i
;
1443 tiler_align(fmts
[i
], &w
, &h
);
1444 /* note: since each region is 1 4kb page wide, and minimum
1445 * number of rows, the height ends up being the same as the
1446 * # of pages in the region
1448 usergart
[i
].height
= h
;
1449 usergart
[i
].height_shift
= ilog2(h
);
1450 usergart
[i
].stride_pfn
= tiler_stride(fmts
[i
], 0) >> PAGE_SHIFT
;
1451 usergart
[i
].slot_shift
= ilog2((PAGE_SIZE
/ h
) >> i
);
1452 for (j
= 0; j
< NUM_USERGART_ENTRIES
; j
++) {
1453 struct usergart_entry
*entry
= &usergart
[i
].entry
[j
];
1454 struct tiler_block
*block
=
1455 tiler_reserve_2d(fmts
[i
], w
, h
,
1457 if (IS_ERR(block
)) {
1459 "reserve failed: %d, %d, %ld\n",
1460 i
, j
, PTR_ERR(block
));
1463 entry
->paddr
= tiler_ssptr(block
);
1464 entry
->block
= block
;
1466 DBG("%d:%d: %dx%d: paddr=%pad stride=%d", i
, j
, w
, h
,
1468 usergart
[i
].stride_pfn
<< PAGE_SHIFT
);
1472 priv
->has_dmm
= true;
1475 void omap_gem_deinit(struct drm_device
*dev
)
1477 /* I believe we can rely on there being no more outstanding GEM
1478 * objects which could depend on usergart/dmm at this point.
This page took 0.091187 seconds and 5 git commands to generate.