1 /* drivers/android/pmem.c
3 * Copyright (C) 2007 Google, Inc.
5 * This software is licensed under the terms of the GNU General Public
6 * License version 2, as published by the Free Software Foundation, and
7 * may be copied, distributed, and modified under those terms.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
16 #include <linux/miscdevice.h>
17 #include <linux/platform_device.h>
19 #include <linux/file.h>
21 #include <linux/list.h>
22 #include <linux/debugfs.h>
23 #include <linux/android_pmem.h>
24 #include <linux/mempolicy.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
28 #include <linux/uaccess.h>
29 #include <asm/cacheflush.h>
31 #define PMEM_MAX_DEVICES 10
32 #define PMEM_MAX_ORDER 128
33 #define PMEM_MIN_ALLOC PAGE_SIZE
37 /* indicates that a refernce to this file has been taken via get_pmem_file,
38 * the file should not be released until put_pmem_file is called */
39 #define PMEM_FLAGS_BUSY 0x1
40 /* indicates that this is a suballocation of a larger master range */
41 #define PMEM_FLAGS_CONNECTED (0x1 << 1)
42 /* indicates this is a master and not a sub allocation and that it is mmaped */
43 #define PMEM_FLAGS_MASTERMAP (0x1 << 2)
44 /* submap and unsubmap flags indicate:
45 * 00: subregion has never been mmaped
46 * 10: subregion has been mmaped, reference to the mm was taken
47 * 11: subretion has ben released, refernece to the mm still held
48 * 01: subretion has been released, reference to the mm has been released
50 #define PMEM_FLAGS_SUBMAP (0x1 << 3)
51 #define PMEM_FLAGS_UNSUBMAP (0x1 << 4)
55 /* in alloc mode: an index into the bitmap
56 * in no_alloc mode: the size of the allocation */
58 /* see flags above for descriptions */
60 /* protects this data field, if the mm_mmap sem will be held at the
61 * same time as this sem, the mm sem must be taken first (as this is
62 * the order for vma_open and vma_close ops */
63 struct rw_semaphore sem
;
64 /* info about the mmaping process */
65 struct vm_area_struct
*vma
;
66 /* task struct of the mapping process */
67 struct task_struct
*task
;
68 /* process id of teh mapping process */
70 /* file descriptor of the master */
72 /* file struct of the master */
73 struct file
*master_file
;
74 /* a list of currently available regions if this is a suballocation */
75 struct list_head region_list
;
76 /* a linked list of data so we can access them for debugging */
77 struct list_head list
;
84 unsigned allocated
:1; /* 1 if allocated, 0 if free */
85 unsigned order
:7; /* size of the region in pmem space */
88 struct pmem_region_node
{
89 struct pmem_region region
;
90 struct list_head list
;
93 #define PMEM_DEBUG_MSGS 0
95 #define DLOG(fmt, args...) \
96 do { printk(KERN_INFO "[%s:%s:%d] "fmt, __FILE__, __func__, __LINE__, \
100 #define DLOG(x...) do {} while (0)
104 struct miscdevice dev
;
105 /* physical start address of the remaped pmem space */
107 /* vitual start address of the remaped pmem space */
108 unsigned char __iomem
*vbase
;
109 /* total size of the pmem space */
111 /* number of entries in the pmem space */
112 unsigned long num_entries
;
113 /* pfn of the garbage page in memory */
114 unsigned long garbage_pfn
;
115 /* index of the garbage page in the pmem space */
117 /* the bitmap for the region indicating which entries are allocated
118 * and which are free */
119 struct pmem_bits
*bitmap
;
120 /* indicates the region should not be managed with an allocator */
121 unsigned no_allocator
;
122 /* indicates maps of this region should be cached, if a mix of
123 * cached and uncached is desired, set this and open the device with
124 * O_SYNC to get an uncached region */
127 /* in no_allocator mode the first mapper gets the whole space and sets
130 /* for debugging, creates a list of pmem file structs, the
131 * data_list_sem should be taken before pmem_data->sem if both are
133 struct semaphore data_list_sem
;
134 struct list_head data_list
;
135 /* pmem_sem protects the bitmap array
136 * a write lock should be held when modifying entries in bitmap
137 * a read lock should be held when reading data from bits or
138 * dereferencing a pointer into bitmap
140 * pmem_data->sem protects the pmem data of a particular file
141 * Many of the function that require the pmem_data->sem have a non-
142 * locking version for when the caller is already holding that sem.
144 * IF YOU TAKE BOTH LOCKS TAKE THEM IN THIS ORDER:
145 * down(pmem_data->sem) => down(bitmap_sem)
147 struct rw_semaphore bitmap_sem
;
149 long (*ioctl
)(struct file
*, unsigned int, unsigned long);
150 int (*release
)(struct inode
*, struct file
*);
153 static struct pmem_info pmem
[PMEM_MAX_DEVICES
];
156 #define PMEM_IS_FREE(id, index) (!(pmem[id].bitmap[index].allocated))
157 #define PMEM_ORDER(id, index) pmem[id].bitmap[index].order
158 #define PMEM_BUDDY_INDEX(id, index) (index ^ (1 << PMEM_ORDER(id, index)))
159 #define PMEM_NEXT_INDEX(id, index) (index + (1 << PMEM_ORDER(id, index)))
160 #define PMEM_OFFSET(index) (index * PMEM_MIN_ALLOC)
161 #define PMEM_START_ADDR(id, index) (PMEM_OFFSET(index) + pmem[id].base)
162 #define PMEM_LEN(id, index) ((1 << PMEM_ORDER(id, index)) * PMEM_MIN_ALLOC)
163 #define PMEM_END_ADDR(id, index) (PMEM_START_ADDR(id, index) + \
165 #define PMEM_START_VADDR(id, index) (PMEM_OFFSET(id, index) + pmem[id].vbase)
166 #define PMEM_END_VADDR(id, index) (PMEM_START_VADDR(id, index) + \
168 #define PMEM_REVOKED(data) (data->flags & PMEM_FLAGS_REVOKED)
169 #define PMEM_IS_PAGE_ALIGNED(addr) (!((addr) & (~PAGE_MASK)))
170 #define PMEM_IS_SUBMAP(data) ((data->flags & PMEM_FLAGS_SUBMAP) && \
171 (!(data->flags & PMEM_FLAGS_UNSUBMAP)))
173 static int pmem_release(struct inode
*, struct file
*);
174 static int pmem_mmap(struct file
*, struct vm_area_struct
*);
175 static int pmem_open(struct inode
*, struct file
*);
176 static long pmem_ioctl(struct file
*, unsigned int, unsigned long);
178 const struct file_operations pmem_fops
= {
179 .release
= pmem_release
,
182 .unlocked_ioctl
= pmem_ioctl
,
183 .llseek
= noop_llseek
,
186 static int get_id(struct file
*file
)
188 return MINOR(file
->f_dentry
->d_inode
->i_rdev
);
191 static int is_pmem_file(struct file
*file
)
195 if (unlikely(!file
|| !file
->f_dentry
|| !file
->f_dentry
->d_inode
))
198 if (unlikely(id
>= PMEM_MAX_DEVICES
))
200 if (unlikely(file
->f_dentry
->d_inode
->i_rdev
!=
201 MKDEV(MISC_MAJOR
, pmem
[id
].dev
.minor
)))
206 static int has_allocation(struct file
*file
)
208 struct pmem_data
*data
;
209 /* check is_pmem_file first if not accessed via pmem_file_ops */
211 if (unlikely(!file
->private_data
))
213 data
= file
->private_data
;
214 if (unlikely(data
->index
< 0))
219 static int is_master_owner(struct file
*file
)
221 struct file
*master_file
;
222 struct pmem_data
*data
;
223 int put_needed
, ret
= 0;
225 if (!is_pmem_file(file
) || !has_allocation(file
))
227 data
= file
->private_data
;
228 if (PMEM_FLAGS_MASTERMAP
& data
->flags
)
230 master_file
= fget_light(data
->master_fd
, &put_needed
);
231 if (master_file
&& data
->master_file
== master_file
)
233 fput_light(master_file
, put_needed
);
237 static int pmem_free(int id
, int index
)
239 /* caller should hold the write lock on pmem_sem! */
240 int buddy
, curr
= index
;
241 DLOG("index %d\n", index
);
243 if (pmem
[id
].no_allocator
) {
244 pmem
[id
].allocated
= 0;
247 /* clean up the bitmap, merging any buddies */
248 pmem
[id
].bitmap
[curr
].allocated
= 0;
249 /* find a slots buddy Buddy# = Slot# ^ (1 << order)
250 * if the buddy is also free merge them
251 * repeat until the buddy is not free or end of the bitmap is reached
254 buddy
= PMEM_BUDDY_INDEX(id
, curr
);
255 if (PMEM_IS_FREE(id
, buddy
) &&
256 PMEM_ORDER(id
, buddy
) == PMEM_ORDER(id
, curr
)) {
257 PMEM_ORDER(id
, buddy
)++;
258 PMEM_ORDER(id
, curr
)++;
259 curr
= min(buddy
, curr
);
263 } while (curr
< pmem
[id
].num_entries
);
268 static void pmem_revoke(struct file
*file
, struct pmem_data
*data
);
270 static int pmem_release(struct inode
*inode
, struct file
*file
)
272 struct pmem_data
*data
= file
->private_data
;
273 struct pmem_region_node
*region_node
;
274 struct list_head
*elt
, *elt2
;
275 int id
= get_id(file
), ret
= 0;
278 down(&pmem
[id
].data_list_sem
);
279 /* if this file is a master, revoke all the memory in the connected
281 if (PMEM_FLAGS_MASTERMAP
& data
->flags
) {
282 struct pmem_data
*sub_data
;
283 list_for_each(elt
, &pmem
[id
].data_list
) {
284 sub_data
= list_entry(elt
, struct pmem_data
, list
);
285 down_read(&sub_data
->sem
);
286 if (PMEM_IS_SUBMAP(sub_data
) &&
287 file
== sub_data
->master_file
) {
288 up_read(&sub_data
->sem
);
289 pmem_revoke(file
, sub_data
);
291 up_read(&sub_data
->sem
);
294 list_del(&data
->list
);
295 up(&pmem
[id
].data_list_sem
);
298 down_write(&data
->sem
);
300 /* if its not a conencted file and it has an allocation, free it */
301 if (!(PMEM_FLAGS_CONNECTED
& data
->flags
) && has_allocation(file
)) {
302 down_write(&pmem
[id
].bitmap_sem
);
303 ret
= pmem_free(id
, data
->index
);
304 up_write(&pmem
[id
].bitmap_sem
);
307 /* if this file is a submap (mapped, connected file), downref the
309 if (PMEM_FLAGS_SUBMAP
& data
->flags
)
311 put_task_struct(data
->task
);
315 file
->private_data
= NULL
;
317 list_for_each_safe(elt
, elt2
, &data
->region_list
) {
318 region_node
= list_entry(elt
, struct pmem_region_node
, list
);
322 BUG_ON(!list_empty(&data
->region_list
));
324 up_write(&data
->sem
);
326 if (pmem
[id
].release
)
327 ret
= pmem
[id
].release(inode
, file
);
332 static int pmem_open(struct inode
*inode
, struct file
*file
)
334 struct pmem_data
*data
;
335 int id
= get_id(file
);
338 DLOG("current %u file %p(%d)\n", current
->pid
, file
, file_count(file
));
339 /* setup file->private_data to indicate its unmapped */
340 /* you can only open a pmem device one time */
341 if (file
->private_data
!= NULL
)
343 data
= kmalloc(sizeof(struct pmem_data
), GFP_KERNEL
);
345 printk("pmem: unable to allocate memory for pmem metadata.");
353 data
->master_file
= NULL
;
357 INIT_LIST_HEAD(&data
->region_list
);
358 init_rwsem(&data
->sem
);
360 file
->private_data
= data
;
361 INIT_LIST_HEAD(&data
->list
);
363 down(&pmem
[id
].data_list_sem
);
364 list_add(&data
->list
, &pmem
[id
].data_list
);
365 up(&pmem
[id
].data_list_sem
);
369 static unsigned long pmem_order(unsigned long len
)
373 len
= (len
+ PMEM_MIN_ALLOC
- 1)/PMEM_MIN_ALLOC
;
375 for (i
= 0; i
< sizeof(len
)*8; i
++)
381 static int pmem_allocate(int id
, unsigned long len
)
383 /* caller should hold the write lock on pmem_sem! */
384 /* return the corresponding pdata[] entry */
386 int end
= pmem
[id
].num_entries
;
388 unsigned long order
= pmem_order(len
);
390 if (pmem
[id
].no_allocator
) {
391 DLOG("no allocator");
392 if ((len
> pmem
[id
].size
) || pmem
[id
].allocated
)
394 pmem
[id
].allocated
= 1;
398 if (order
> PMEM_MAX_ORDER
)
400 DLOG("order %lx\n", order
);
402 /* look through the bitmap:
403 * if you find a free slot of the correct order use it
404 * otherwise, use the best fit (smallest with size > order) slot
407 if (PMEM_IS_FREE(id
, curr
)) {
408 if (PMEM_ORDER(id
, curr
) == (unsigned char)order
) {
409 /* set the not free bit and clear others */
413 if (PMEM_ORDER(id
, curr
) > (unsigned char)order
&&
415 PMEM_ORDER(id
, curr
) < PMEM_ORDER(id
, best_fit
)))
418 curr
= PMEM_NEXT_INDEX(id
, curr
);
421 /* if best_fit < 0, there are no suitable slots,
425 printk("pmem: no space left to allocate!\n");
429 /* now partition the best fit:
430 * split the slot into 2 buddies of order - 1
431 * repeat until the slot is of the correct order
433 while (PMEM_ORDER(id
, best_fit
) > (unsigned char)order
) {
435 PMEM_ORDER(id
, best_fit
) -= 1;
436 buddy
= PMEM_BUDDY_INDEX(id
, best_fit
);
437 PMEM_ORDER(id
, buddy
) = PMEM_ORDER(id
, best_fit
);
439 pmem
[id
].bitmap
[best_fit
].allocated
= 1;
443 static pgprot_t
phys_mem_access_prot(struct file
*file
, pgprot_t vma_prot
)
445 int id
= get_id(file
);
446 #ifdef pgprot_noncached
447 if (pmem
[id
].cached
== 0 || file
->f_flags
& O_SYNC
)
448 return pgprot_noncached(vma_prot
);
450 #ifdef pgprot_ext_buffered
451 else if (pmem
[id
].buffered
)
452 return pgprot_ext_buffered(vma_prot
);
457 static unsigned long pmem_start_addr(int id
, struct pmem_data
*data
)
459 if (pmem
[id
].no_allocator
)
460 return PMEM_START_ADDR(id
, 0);
462 return PMEM_START_ADDR(id
, data
->index
);
466 static void *pmem_start_vaddr(int id
, struct pmem_data
*data
)
468 return pmem_start_addr(id
, data
) - pmem
[id
].base
+ pmem
[id
].vbase
;
471 static unsigned long pmem_len(int id
, struct pmem_data
*data
)
473 if (pmem
[id
].no_allocator
)
476 return PMEM_LEN(id
, data
->index
);
479 static int pmem_map_garbage(int id
, struct vm_area_struct
*vma
,
480 struct pmem_data
*data
, unsigned long offset
,
483 int i
, garbage_pages
= len
>> PAGE_SHIFT
;
485 vma
->vm_flags
|= VM_IO
| VM_RESERVED
| VM_PFNMAP
| VM_SHARED
| VM_WRITE
;
486 for (i
= 0; i
< garbage_pages
; i
++) {
487 if (vm_insert_pfn(vma
, vma
->vm_start
+ offset
+ (i
* PAGE_SIZE
),
488 pmem
[id
].garbage_pfn
))
494 static int pmem_unmap_pfn_range(int id
, struct vm_area_struct
*vma
,
495 struct pmem_data
*data
, unsigned long offset
,
499 DLOG("unmap offset %lx len %lx\n", offset
, len
);
501 BUG_ON(!PMEM_IS_PAGE_ALIGNED(len
));
503 garbage_pages
= len
>> PAGE_SHIFT
;
504 zap_page_range(vma
, vma
->vm_start
+ offset
, len
, NULL
);
505 pmem_map_garbage(id
, vma
, data
, offset
, len
);
509 static int pmem_map_pfn_range(int id
, struct vm_area_struct
*vma
,
510 struct pmem_data
*data
, unsigned long offset
,
513 DLOG("map offset %lx len %lx\n", offset
, len
);
514 BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma
->vm_start
));
515 BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma
->vm_end
));
516 BUG_ON(!PMEM_IS_PAGE_ALIGNED(len
));
517 BUG_ON(!PMEM_IS_PAGE_ALIGNED(offset
));
519 if (io_remap_pfn_range(vma
, vma
->vm_start
+ offset
,
520 (pmem_start_addr(id
, data
) + offset
) >> PAGE_SHIFT
,
521 len
, vma
->vm_page_prot
)) {
527 static int pmem_remap_pfn_range(int id
, struct vm_area_struct
*vma
,
528 struct pmem_data
*data
, unsigned long offset
,
531 /* hold the mm semp for the vma you are modifying when you call this */
533 zap_page_range(vma
, vma
->vm_start
+ offset
, len
, NULL
);
534 return pmem_map_pfn_range(id
, vma
, data
, offset
, len
);
537 static void pmem_vma_open(struct vm_area_struct
*vma
)
539 struct file
*file
= vma
->vm_file
;
540 struct pmem_data
*data
= file
->private_data
;
541 int id
= get_id(file
);
542 /* this should never be called as we don't support copying pmem
544 BUG_ON(!has_allocation(file
));
545 down_write(&data
->sem
);
546 /* remap the garbage pages, forkers don't get access to the data */
547 pmem_unmap_pfn_range(id
, vma
, data
, 0, vma
->vm_start
- vma
->vm_end
);
548 up_write(&data
->sem
);
551 static void pmem_vma_close(struct vm_area_struct
*vma
)
553 struct file
*file
= vma
->vm_file
;
554 struct pmem_data
*data
= file
->private_data
;
556 DLOG("current %u ppid %u file %p count %d\n", current
->pid
,
557 current
->parent
->pid
, file
, file_count(file
));
558 if (unlikely(!is_pmem_file(file
) || !has_allocation(file
))) {
559 printk(KERN_WARNING
"pmem: something is very wrong, you are "
560 "closing a vm backing an allocation that doesn't "
564 down_write(&data
->sem
);
565 if (data
->vma
== vma
) {
567 if ((data
->flags
& PMEM_FLAGS_CONNECTED
) &&
568 (data
->flags
& PMEM_FLAGS_SUBMAP
))
569 data
->flags
|= PMEM_FLAGS_UNSUBMAP
;
571 /* the kernel is going to free this vma now anyway */
572 up_write(&data
->sem
);
575 static struct vm_operations_struct vm_ops
= {
576 .open
= pmem_vma_open
,
577 .close
= pmem_vma_close
,
580 static int pmem_mmap(struct file
*file
, struct vm_area_struct
*vma
)
582 struct pmem_data
*data
;
584 unsigned long vma_size
= vma
->vm_end
- vma
->vm_start
;
585 int ret
= 0, id
= get_id(file
);
587 if (vma
->vm_pgoff
|| !PMEM_IS_PAGE_ALIGNED(vma_size
)) {
589 printk(KERN_ERR
"pmem: mmaps must be at offset zero, aligned"
590 " and a multiple of pages_size.\n");
595 data
= file
->private_data
;
596 down_write(&data
->sem
);
597 /* check this file isn't already mmaped, for submaps check this file
598 * has never been mmaped */
599 if ((data
->flags
& PMEM_FLAGS_MASTERMAP
) ||
600 (data
->flags
& PMEM_FLAGS_SUBMAP
) ||
601 (data
->flags
& PMEM_FLAGS_UNSUBMAP
)) {
603 printk(KERN_ERR
"pmem: you can only mmap a pmem file once, "
604 "this file is already mmaped. %x\n", data
->flags
);
609 /* if file->private_data == unalloced, alloc*/
610 if (data
&& data
->index
== -1) {
611 down_write(&pmem
[id
].bitmap_sem
);
612 index
= pmem_allocate(id
, vma
->vm_end
- vma
->vm_start
);
613 up_write(&pmem
[id
].bitmap_sem
);
616 /* either no space was available or an error occured */
617 if (!has_allocation(file
)) {
619 printk("pmem: could not find allocation for map.\n");
623 if (pmem_len(id
, data
) < vma_size
) {
625 printk(KERN_WARNING
"pmem: mmap size [%lu] does not match"
626 "size of backing region [%lu].\n", vma_size
,
633 vma
->vm_pgoff
= pmem_start_addr(id
, data
) >> PAGE_SHIFT
;
634 vma
->vm_page_prot
= phys_mem_access_prot(file
, vma
->vm_page_prot
);
636 if (data
->flags
& PMEM_FLAGS_CONNECTED
) {
637 struct pmem_region_node
*region_node
;
638 struct list_head
*elt
;
639 if (pmem_map_garbage(id
, vma
, data
, 0, vma_size
)) {
640 printk("pmem: mmap failed in kernel!\n");
644 list_for_each(elt
, &data
->region_list
) {
645 region_node
= list_entry(elt
, struct pmem_region_node
,
647 DLOG("remapping file: %p %lx %lx\n", file
,
648 region_node
->region
.offset
,
649 region_node
->region
.len
);
650 if (pmem_remap_pfn_range(id
, vma
, data
,
651 region_node
->region
.offset
,
652 region_node
->region
.len
)) {
657 data
->flags
|= PMEM_FLAGS_SUBMAP
;
658 get_task_struct(current
->group_leader
);
659 data
->task
= current
->group_leader
;
662 data
->pid
= current
->pid
;
664 DLOG("submmapped file %p vma %p pid %u\n", file
, vma
,
667 if (pmem_map_pfn_range(id
, vma
, data
, 0, vma_size
)) {
668 printk(KERN_INFO
"pmem: mmap failed in kernel!\n");
672 data
->flags
|= PMEM_FLAGS_MASTERMAP
;
673 data
->pid
= current
->pid
;
675 vma
->vm_ops
= &vm_ops
;
677 up_write(&data
->sem
);
681 /* the following are the api for accessing pmem regions by other drivers
682 * from inside the kernel */
683 int get_pmem_user_addr(struct file
*file
, unsigned long *start
,
686 struct pmem_data
*data
;
687 if (!is_pmem_file(file
) || !has_allocation(file
)) {
689 printk(KERN_INFO
"pmem: requested pmem data from invalid"
694 data
= file
->private_data
;
695 down_read(&data
->sem
);
697 *start
= data
->vma
->vm_start
;
698 *len
= data
->vma
->vm_end
- data
->vma
->vm_start
;
707 int get_pmem_addr(struct file
*file
, unsigned long *start
,
708 unsigned long *vstart
, unsigned long *len
)
710 struct pmem_data
*data
;
713 if (!is_pmem_file(file
) || !has_allocation(file
))
716 data
= file
->private_data
;
717 if (data
->index
== -1) {
719 printk(KERN_INFO
"pmem: requested pmem data from file with no "
726 down_read(&data
->sem
);
727 *start
= pmem_start_addr(id
, data
);
728 *len
= pmem_len(id
, data
);
729 *vstart
= (unsigned long)pmem_start_vaddr(id
, data
);
732 down_write(&data
->sem
);
734 up_write(&data
->sem
);
739 int get_pmem_file(int fd
, unsigned long *start
, unsigned long *vstart
,
740 unsigned long *len
, struct file
**filp
)
745 if (unlikely(file
== NULL
)) {
746 printk(KERN_INFO
"pmem: requested data from file descriptor "
747 "that doesn't exist.");
751 if (get_pmem_addr(file
, start
, vstart
, len
))
762 void put_pmem_file(struct file
*file
)
764 struct pmem_data
*data
;
767 if (!is_pmem_file(file
))
770 data
= file
->private_data
;
772 down_write(&data
->sem
);
773 if (data
->ref
== 0) {
774 printk("pmem: pmem_put > pmem_get %s (pid %d)\n",
775 pmem
[id
].dev
.name
, data
->pid
);
779 up_write(&data
->sem
);
784 void flush_pmem_file(struct file
*file
, unsigned long offset
, unsigned long len
)
786 struct pmem_data
*data
;
789 struct pmem_region_node
*region_node
;
790 struct list_head
*elt
;
791 void *flush_start
, *flush_end
;
793 if (!is_pmem_file(file
) || !has_allocation(file
))
797 data
= file
->private_data
;
798 if (!pmem
[id
].cached
)
801 down_read(&data
->sem
);
802 vaddr
= pmem_start_vaddr(id
, data
);
803 /* if this isn't a submmapped file, flush the whole thing */
804 if (unlikely(!(data
->flags
& PMEM_FLAGS_CONNECTED
))) {
805 dmac_flush_range(vaddr
, vaddr
+ pmem_len(id
, data
));
808 /* otherwise, flush the region of the file we are drawing */
809 list_for_each(elt
, &data
->region_list
) {
810 region_node
= list_entry(elt
, struct pmem_region_node
, list
);
811 if ((offset
>= region_node
->region
.offset
) &&
812 ((offset
+ len
) <= (region_node
->region
.offset
+
813 region_node
->region
.len
))) {
814 flush_start
= vaddr
+ region_node
->region
.offset
;
815 flush_end
= flush_start
+ region_node
->region
.len
;
816 dmac_flush_range(flush_start
, flush_end
);
824 static int pmem_connect(unsigned long connect
, struct file
*file
)
826 struct pmem_data
*data
= file
->private_data
;
827 struct pmem_data
*src_data
;
828 struct file
*src_file
;
829 int ret
= 0, put_needed
;
831 down_write(&data
->sem
);
832 /* retrieve the src file and check it is a pmem file with an alloc */
833 src_file
= fget_light(connect
, &put_needed
);
834 DLOG("connect %p to %p\n", file
, src_file
);
836 printk(KERN_INFO
"pmem: src file not found!\n");
840 if (unlikely(!is_pmem_file(src_file
) || !has_allocation(src_file
))) {
841 printk(KERN_INFO
"pmem: src file is not a pmem file or has no "
846 src_data
= src_file
->private_data
;
848 if (has_allocation(file
) && (data
->index
!= src_data
->index
)) {
849 printk(KERN_INFO
"pmem: file is already mapped but doesn't "
850 "match this src_file!\n");
854 data
->index
= src_data
->index
;
855 data
->flags
|= PMEM_FLAGS_CONNECTED
;
856 data
->master_fd
= connect
;
857 data
->master_file
= src_file
;
860 fput_light(src_file
, put_needed
);
862 up_write(&data
->sem
);
866 static void pmem_unlock_data_and_mm(struct pmem_data
*data
,
867 struct mm_struct
*mm
)
869 up_write(&data
->sem
);
871 up_write(&mm
->mmap_sem
);
876 static int pmem_lock_data_and_mm(struct file
*file
, struct pmem_data
*data
,
877 struct mm_struct
**locked_mm
)
880 struct mm_struct
*mm
= NULL
;
883 down_read(&data
->sem
);
884 if (PMEM_IS_SUBMAP(data
)) {
885 mm
= get_task_mm(data
->task
);
888 printk(KERN_DEBUG
"pmem: can't remap task is gone!\n");
897 down_write(&mm
->mmap_sem
);
899 down_write(&data
->sem
);
900 /* check that the file didn't get mmaped before we could take the
901 * data sem, this should be safe b/c you can only submap each file
903 if (PMEM_IS_SUBMAP(data
) && !mm
) {
904 pmem_unlock_data_and_mm(data
, mm
);
905 up_write(&data
->sem
);
908 /* now check that vma.mm is still there, it could have been
909 * deleted by vma_close before we could get the data->sem */
910 if ((data
->flags
& PMEM_FLAGS_UNSUBMAP
) && (mm
!= NULL
)) {
911 /* might as well release this */
912 if (data
->flags
& PMEM_FLAGS_SUBMAP
) {
913 put_task_struct(data
->task
);
915 /* lower the submap flag to show the mm is gone */
916 data
->flags
&= ~(PMEM_FLAGS_SUBMAP
);
918 pmem_unlock_data_and_mm(data
, mm
);
925 int pmem_remap(struct pmem_region
*region
, struct file
*file
,
929 struct pmem_region_node
*region_node
;
930 struct mm_struct
*mm
= NULL
;
931 struct list_head
*elt
, *elt2
;
932 int id
= get_id(file
);
933 struct pmem_data
*data
= file
->private_data
;
935 /* pmem region must be aligned on a page boundry */
936 if (unlikely(!PMEM_IS_PAGE_ALIGNED(region
->offset
) ||
937 !PMEM_IS_PAGE_ALIGNED(region
->len
))) {
939 printk(KERN_DEBUG
"pmem: request for unaligned pmem "
940 "suballocation %lx %lx\n", region
->offset
, region
->len
);
945 /* if userspace requests a region of len 0, there's nothing to do */
946 if (region
->len
== 0)
949 /* lock the mm and data */
950 ret
= pmem_lock_data_and_mm(file
, data
, &mm
);
954 /* only the owner of the master file can remap the client fds
956 if (!is_master_owner(file
)) {
958 printk("pmem: remap requested from non-master process\n");
964 /* check that the requested range is within the src allocation */
965 if (unlikely((region
->offset
> pmem_len(id
, data
)) ||
966 (region
->len
> pmem_len(id
, data
)) ||
967 (region
->offset
+ region
->len
> pmem_len(id
, data
)))) {
969 printk(KERN_INFO
"pmem: suballoc doesn't fit in src_file!\n");
975 if (operation
== PMEM_MAP
) {
976 region_node
= kmalloc(sizeof(struct pmem_region_node
),
981 printk(KERN_INFO
"No space to allocate metadata!");
985 region_node
->region
= *region
;
986 list_add(®ion_node
->list
, &data
->region_list
);
987 } else if (operation
== PMEM_UNMAP
) {
989 list_for_each_safe(elt
, elt2
, &data
->region_list
) {
990 region_node
= list_entry(elt
, struct pmem_region_node
,
992 if (region
->len
== 0 ||
993 (region_node
->region
.offset
== region
->offset
&&
994 region_node
->region
.len
== region
->len
)) {
1002 printk("pmem: Unmap region does not map any mapped "
1010 if (data
->vma
&& PMEM_IS_SUBMAP(data
)) {
1011 if (operation
== PMEM_MAP
)
1012 ret
= pmem_remap_pfn_range(id
, data
->vma
, data
,
1013 region
->offset
, region
->len
);
1014 else if (operation
== PMEM_UNMAP
)
1015 ret
= pmem_unmap_pfn_range(id
, data
->vma
, data
,
1016 region
->offset
, region
->len
);
1020 pmem_unlock_data_and_mm(data
, mm
);
1024 static void pmem_revoke(struct file
*file
, struct pmem_data
*data
)
1026 struct pmem_region_node
*region_node
;
1027 struct list_head
*elt
, *elt2
;
1028 struct mm_struct
*mm
= NULL
;
1029 int id
= get_id(file
);
1032 data
->master_file
= NULL
;
1033 ret
= pmem_lock_data_and_mm(file
, data
, &mm
);
1034 /* if lock_data_and_mm fails either the task that mapped the fd, or
1035 * the vma that mapped it have already gone away, nothing more
1036 * needs to be done */
1039 /* unmap everything */
1040 /* delete the regions and region list nothing is mapped any more */
1042 list_for_each_safe(elt
, elt2
, &data
->region_list
) {
1043 region_node
= list_entry(elt
, struct pmem_region_node
,
1045 pmem_unmap_pfn_range(id
, data
->vma
, data
,
1046 region_node
->region
.offset
,
1047 region_node
->region
.len
);
1051 /* delete the master file */
1052 pmem_unlock_data_and_mm(data
, mm
);
1055 static void pmem_get_size(struct pmem_region
*region
, struct file
*file
)
1057 struct pmem_data
*data
= file
->private_data
;
1058 int id
= get_id(file
);
1060 if (!has_allocation(file
)) {
1065 region
->offset
= pmem_start_addr(id
, data
);
1066 region
->len
= pmem_len(id
, data
);
1068 DLOG("offset %lx len %lx\n", region
->offset
, region
->len
);
1072 static long pmem_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
1074 struct pmem_data
*data
;
1075 int id
= get_id(file
);
1080 struct pmem_region region
;
1082 if (!has_allocation(file
)) {
1086 data
= file
->private_data
;
1087 region
.offset
= pmem_start_addr(id
, data
);
1088 region
.len
= pmem_len(id
, data
);
1090 printk(KERN_INFO
"pmem: request for physical address "
1091 "of pmem region from process %d.\n", current
->pid
);
1092 if (copy_to_user((void __user
*)arg
, ®ion
,
1093 sizeof(struct pmem_region
)))
1099 struct pmem_region region
;
1100 if (copy_from_user(®ion
, (void __user
*)arg
,
1101 sizeof(struct pmem_region
)))
1103 data
= file
->private_data
;
1104 return pmem_remap(®ion
, file
, PMEM_MAP
);
1109 struct pmem_region region
;
1110 if (copy_from_user(®ion
, (void __user
*)arg
,
1111 sizeof(struct pmem_region
)))
1113 data
= file
->private_data
;
1114 return pmem_remap(®ion
, file
, PMEM_UNMAP
);
1119 struct pmem_region region
;
1121 pmem_get_size(®ion
, file
);
1122 if (copy_to_user((void __user
*)arg
, ®ion
,
1123 sizeof(struct pmem_region
)))
1127 case PMEM_GET_TOTAL_SIZE
:
1129 struct pmem_region region
;
1130 DLOG("get total size\n");
1133 region
.len
= pmem
[id
].size
;
1134 if (copy_to_user((void __user
*)arg
, ®ion
,
1135 sizeof(struct pmem_region
)))
1141 if (has_allocation(file
))
1143 data
= file
->private_data
;
1144 data
->index
= pmem_allocate(id
, arg
);
1149 return pmem_connect(arg
, file
);
1153 return pmem
[id
].ioctl(file
, cmd
, arg
);
1160 static ssize_t
debug_open(struct inode
*inode
, struct file
*file
)
1162 file
->private_data
= inode
->i_private
;
1166 static ssize_t
debug_read(struct file
*file
, char __user
*buf
, size_t count
,
1169 struct list_head
*elt
, *elt2
;
1170 struct pmem_data
*data
;
1171 struct pmem_region_node
*region_node
;
1172 int id
= (int)file
->private_data
;
1173 const int debug_bufmax
= 4096;
1174 static char buffer
[4096];
1177 DLOG("debug open\n");
1178 n
= scnprintf(buffer
, debug_bufmax
,
1179 "pid #: mapped regions (offset, len) (offset,len)...\n");
1181 down(&pmem
[id
].data_list_sem
);
1182 list_for_each(elt
, &pmem
[id
].data_list
) {
1183 data
= list_entry(elt
, struct pmem_data
, list
);
1184 down_read(&data
->sem
);
1185 n
+= scnprintf(buffer
+ n
, debug_bufmax
- n
, "pid %u:",
1187 list_for_each(elt2
, &data
->region_list
) {
1188 region_node
= list_entry(elt2
, struct pmem_region_node
,
1190 n
+= scnprintf(buffer
+ n
, debug_bufmax
- n
,
1192 region_node
->region
.offset
,
1193 region_node
->region
.len
);
1195 n
+= scnprintf(buffer
+ n
, debug_bufmax
- n
, "\n");
1196 up_read(&data
->sem
);
1198 up(&pmem
[id
].data_list_sem
);
1202 return simple_read_from_buffer(buf
, count
, ppos
, buffer
, n
);
1205 static struct file_operations debug_fops
= {
1208 .llseek
= default_llseek
,
1213 static struct miscdevice pmem_dev
= {
1219 int pmem_setup(struct android_pmem_platform_data
*pdata
,
1220 long (*ioctl
)(struct file
*, unsigned int, unsigned long),
1221 int (*release
)(struct inode
*, struct file
*))
1228 pmem
[id
].no_allocator
= pdata
->no_allocator
;
1229 pmem
[id
].cached
= pdata
->cached
;
1230 pmem
[id
].buffered
= pdata
->buffered
;
1231 pmem
[id
].base
= pdata
->start
;
1232 pmem
[id
].size
= pdata
->size
;
1233 pmem
[id
].ioctl
= ioctl
;
1234 pmem
[id
].release
= release
;
1235 init_rwsem(&pmem
[id
].bitmap_sem
);
1236 sema_init(&pmem
[id
].data_list_sem
, 1);
1237 INIT_LIST_HEAD(&pmem
[id
].data_list
);
1238 pmem
[id
].dev
.name
= pdata
->name
;
1239 pmem
[id
].dev
.minor
= id
;
1240 pmem
[id
].dev
.fops
= &pmem_fops
;
1241 printk(KERN_INFO
"%s: %d init\n", pdata
->name
, pdata
->cached
);
1243 err
= misc_register(&pmem
[id
].dev
);
1245 printk(KERN_ALERT
"Unable to register pmem driver!\n");
1246 goto err_cant_register_device
;
1248 pmem
[id
].num_entries
= pmem
[id
].size
/ PMEM_MIN_ALLOC
;
1250 pmem
[id
].bitmap
= kcalloc(pmem
[id
].num_entries
,
1251 sizeof(struct pmem_bits
), GFP_KERNEL
);
1252 if (!pmem
[id
].bitmap
)
1253 goto err_no_mem_for_metadata
;
1255 for (i
= sizeof(pmem
[id
].num_entries
) * 8 - 1; i
>= 0; i
--) {
1256 if ((pmem
[id
].num_entries
) & 1<<i
) {
1257 PMEM_ORDER(id
, index
) = i
;
1258 index
= PMEM_NEXT_INDEX(id
, index
);
1262 if (pmem
[id
].cached
)
1263 pmem
[id
].vbase
= ioremap_cached(pmem
[id
].base
,
1265 #ifdef ioremap_ext_buffered
1266 else if (pmem
[id
].buffered
)
1267 pmem
[id
].vbase
= ioremap_ext_buffered(pmem
[id
].base
,
1271 pmem
[id
].vbase
= ioremap(pmem
[id
].base
, pmem
[id
].size
);
1273 if (pmem
[id
].vbase
== 0)
1274 goto error_cant_remap
;
1276 pmem
[id
].garbage_pfn
= page_to_pfn(alloc_page(GFP_KERNEL
));
1277 if (pmem
[id
].no_allocator
)
1278 pmem
[id
].allocated
= 0;
1281 debugfs_create_file(pdata
->name
, S_IFREG
| S_IRUGO
, NULL
, (void *)id
,
1286 kfree(pmem
[id
].bitmap
);
1287 err_no_mem_for_metadata
:
1288 misc_deregister(&pmem
[id
].dev
);
1289 err_cant_register_device
:
1293 static int pmem_probe(struct platform_device
*pdev
)
1295 struct android_pmem_platform_data
*pdata
;
1297 if (!pdev
|| !pdev
->dev
.platform_data
) {
1298 printk(KERN_ALERT
"Unable to probe pmem!\n");
1301 pdata
= pdev
->dev
.platform_data
;
1302 return pmem_setup(pdata
, NULL
, NULL
);
1306 static int pmem_remove(struct platform_device
*pdev
)
1309 __free_page(pfn_to_page(pmem
[id
].garbage_pfn
));
1310 misc_deregister(&pmem
[id
].dev
);
1314 static struct platform_driver pmem_driver
= {
1315 .probe
= pmem_probe
,
1316 .remove
= pmem_remove
,
1317 .driver
= { .name
= "android_pmem" }
1321 static int __init
pmem_init(void)
1323 return platform_driver_register(&pmem_driver
);
1326 static void __exit
pmem_exit(void)
1328 platform_driver_unregister(&pmem_driver
);
1331 module_init(pmem_init
);
1332 module_exit(pmem_exit
);