1 /******************************************************************************
4 * Device for accessing (in user-space) pages that have been granted by other
7 * Copyright (c) 2006-2007, D G Murray.
8 * (c) 2009 Gerd Hoffmann <kraxel@redhat.com>
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #include <linux/module.h>
23 #include <linux/kernel.h>
24 #include <linux/init.h>
25 #include <linux/miscdevice.h>
28 #include <linux/mman.h>
29 #include <linux/mmu_notifier.h>
30 #include <linux/types.h>
31 #include <linux/uaccess.h>
32 #include <linux/sched.h>
33 #include <linux/spinlock.h>
34 #include <linux/slab.h>
35 #include <linux/highmem.h>
38 #include <xen/grant_table.h>
39 #include <xen/gntdev.h>
40 #include <xen/events.h>
41 #include <asm/xen/hypervisor.h>
42 #include <asm/xen/hypercall.h>
43 #include <asm/xen/page.h>
45 MODULE_LICENSE("GPL");
46 MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, "
47 "Gerd Hoffmann <kraxel@redhat.com>");
48 MODULE_DESCRIPTION("User-space granted page access driver");
50 static int limit
= 1024*1024;
51 module_param(limit
, int, 0644);
52 MODULE_PARM_DESC(limit
, "Maximum number of grants that may be mapped by "
55 static atomic_t pages_mapped
= ATOMIC_INIT(0);
57 static int use_ptemod
;
60 struct list_head maps
;
61 /* lock protects maps from concurrent changes */
64 struct mmu_notifier mn
;
69 /* Address relative to the start of the grant_map */
75 struct list_head next
;
76 struct vm_area_struct
*vma
;
81 struct unmap_notify notify
;
82 struct ioctl_gntdev_grant_ref
*grants
;
83 struct gnttab_map_grant_ref
*map_ops
;
84 struct gnttab_unmap_grant_ref
*unmap_ops
;
88 static int unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
);
90 /* ------------------------------------------------------------------ */
92 static void gntdev_print_maps(struct gntdev_priv
*priv
,
93 char *text
, int text_index
)
96 struct grant_map
*map
;
98 pr_debug("%s: maps list (priv %p)\n", __func__
, priv
);
99 list_for_each_entry(map
, &priv
->maps
, next
)
100 pr_debug(" index %2d, count %2d %s\n",
101 map
->index
, map
->count
,
102 map
->index
== text_index
&& text
? text
: "");
106 static struct grant_map
*gntdev_alloc_map(struct gntdev_priv
*priv
, int count
)
108 struct grant_map
*add
;
111 add
= kzalloc(sizeof(struct grant_map
), GFP_KERNEL
);
115 add
->grants
= kzalloc(sizeof(add
->grants
[0]) * count
, GFP_KERNEL
);
116 add
->map_ops
= kzalloc(sizeof(add
->map_ops
[0]) * count
, GFP_KERNEL
);
117 add
->unmap_ops
= kzalloc(sizeof(add
->unmap_ops
[0]) * count
, GFP_KERNEL
);
118 add
->pages
= kzalloc(sizeof(add
->pages
[0]) * count
, GFP_KERNEL
);
119 if (NULL
== add
->grants
||
120 NULL
== add
->map_ops
||
121 NULL
== add
->unmap_ops
||
125 for (i
= 0; i
< count
; i
++) {
126 add
->pages
[i
] = alloc_page(GFP_KERNEL
| __GFP_HIGHMEM
);
127 if (add
->pages
[i
] == NULL
)
129 add
->map_ops
[i
].handle
= -1;
130 add
->unmap_ops
[i
].handle
= -1;
135 atomic_set(&add
->users
, 1);
141 for (i
= 0; i
< count
; i
++) {
143 __free_page(add
->pages
[i
]);
148 kfree(add
->unmap_ops
);
153 static void gntdev_add_map(struct gntdev_priv
*priv
, struct grant_map
*add
)
155 struct grant_map
*map
;
157 list_for_each_entry(map
, &priv
->maps
, next
) {
158 if (add
->index
+ add
->count
< map
->index
) {
159 list_add_tail(&add
->next
, &map
->next
);
162 add
->index
= map
->index
+ map
->count
;
164 list_add_tail(&add
->next
, &priv
->maps
);
167 gntdev_print_maps(priv
, "[new]", add
->index
);
170 static struct grant_map
*gntdev_find_map_index(struct gntdev_priv
*priv
,
171 int index
, int count
)
173 struct grant_map
*map
;
175 list_for_each_entry(map
, &priv
->maps
, next
) {
176 if (map
->index
!= index
)
178 if (count
&& map
->count
!= count
)
185 static void gntdev_put_map(struct grant_map
*map
)
192 if (!atomic_dec_and_test(&map
->users
))
195 atomic_sub(map
->count
, &pages_mapped
);
197 if (map
->notify
.flags
& UNMAP_NOTIFY_SEND_EVENT
) {
198 notify_remote_via_evtchn(map
->notify
.event
);
203 unmap_grant_pages(map
, 0, map
->count
);
205 for (i
= 0; i
< map
->count
; i
++) {
206 uint32_t check
, *tmp
;
209 /* XXX When unmapping in an HVM domain, Xen will
210 * sometimes end up mapping the GFN to an invalid MFN.
211 * In this case, writes will be discarded and reads will
212 * return all 0xFF bytes. Leak these unusable GFNs
213 * until Xen supports fixing their p2m mapping.
215 * Confirmed present in Xen 4.1-RC3 with HVM source
217 tmp
= kmap(map
->pages
[i
]);
221 kunmap(map
->pages
[i
]);
222 if (check
== 0xdeaddead)
223 __free_page(map
->pages
[i
]);
225 pr_debug("Discard page %d=%ld\n", i
,
226 page_to_pfn(map
->pages
[i
]));
232 kfree(map
->unmap_ops
);
236 /* ------------------------------------------------------------------ */
238 static int find_grant_ptes(pte_t
*pte
, pgtable_t token
,
239 unsigned long addr
, void *data
)
241 struct grant_map
*map
= data
;
242 unsigned int pgnr
= (addr
- map
->vma
->vm_start
) >> PAGE_SHIFT
;
243 int flags
= map
->flags
| GNTMAP_application_map
| GNTMAP_contains_pte
;
246 BUG_ON(pgnr
>= map
->count
);
247 pte_maddr
= arbitrary_virt_to_machine(pte
).maddr
;
249 gnttab_set_map_op(&map
->map_ops
[pgnr
], pte_maddr
, flags
,
250 map
->grants
[pgnr
].ref
,
251 map
->grants
[pgnr
].domid
);
252 gnttab_set_unmap_op(&map
->unmap_ops
[pgnr
], pte_maddr
, flags
,
257 static int map_grant_pages(struct grant_map
*map
)
263 /* Note: it could already be mapped */
264 if (map
->map_ops
[0].handle
!= -1)
266 for (i
= 0; i
< map
->count
; i
++) {
268 pfn_to_kaddr(page_to_pfn(map
->pages
[i
]));
269 gnttab_set_map_op(&map
->map_ops
[i
], addr
, map
->flags
,
271 map
->grants
[i
].domid
);
272 gnttab_set_unmap_op(&map
->unmap_ops
[i
], addr
,
273 map
->flags
, -1 /* handle */);
277 pr_debug("map %d+%d\n", map
->index
, map
->count
);
278 err
= gnttab_map_refs(map
->map_ops
, map
->pages
, map
->count
);
282 for (i
= 0; i
< map
->count
; i
++) {
283 if (map
->map_ops
[i
].status
)
286 BUG_ON(map
->map_ops
[i
].handle
== -1);
287 map
->unmap_ops
[i
].handle
= map
->map_ops
[i
].handle
;
288 pr_debug("map handle=%d\n", map
->map_ops
[i
].handle
);
294 static int __unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
)
298 if (map
->notify
.flags
& UNMAP_NOTIFY_CLEAR_BYTE
) {
299 int pgno
= (map
->notify
.addr
>> PAGE_SHIFT
);
300 if (pgno
>= offset
&& pgno
< offset
+ pages
&& use_ptemod
) {
302 tmp
= map
->vma
->vm_start
+ map
->notify
.addr
;
303 err
= copy_to_user(tmp
, &err
, 1);
306 map
->notify
.flags
&= ~UNMAP_NOTIFY_CLEAR_BYTE
;
307 } else if (pgno
>= offset
&& pgno
< offset
+ pages
) {
308 uint8_t *tmp
= kmap(map
->pages
[pgno
]);
309 tmp
[map
->notify
.addr
& (PAGE_SIZE
-1)] = 0;
310 kunmap(map
->pages
[pgno
]);
311 map
->notify
.flags
&= ~UNMAP_NOTIFY_CLEAR_BYTE
;
315 err
= gnttab_unmap_refs(map
->unmap_ops
+ offset
, map
->pages
+ offset
, pages
);
319 for (i
= 0; i
< pages
; i
++) {
320 if (map
->unmap_ops
[offset
+i
].status
)
322 pr_debug("unmap handle=%d st=%d\n",
323 map
->unmap_ops
[offset
+i
].handle
,
324 map
->unmap_ops
[offset
+i
].status
);
325 map
->unmap_ops
[offset
+i
].handle
= -1;
330 static int unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
)
334 pr_debug("unmap %d+%d [%d+%d]\n", map
->index
, map
->count
, offset
, pages
);
336 /* It is possible the requested range will have a "hole" where we
337 * already unmapped some of the grants. Only unmap valid ranges.
339 while (pages
&& !err
) {
340 while (pages
&& map
->unmap_ops
[offset
].handle
== -1) {
345 while (range
< pages
) {
346 if (map
->unmap_ops
[offset
+range
].handle
== -1) {
352 err
= __unmap_grant_pages(map
, offset
, range
);
360 /* ------------------------------------------------------------------ */
362 static void gntdev_vma_close(struct vm_area_struct
*vma
)
364 struct grant_map
*map
= vma
->vm_private_data
;
366 pr_debug("close %p\n", vma
);
368 vma
->vm_private_data
= NULL
;
372 static struct vm_operations_struct gntdev_vmops
= {
373 .close
= gntdev_vma_close
,
376 /* ------------------------------------------------------------------ */
378 static void mn_invl_range_start(struct mmu_notifier
*mn
,
379 struct mm_struct
*mm
,
380 unsigned long start
, unsigned long end
)
382 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
383 struct grant_map
*map
;
384 unsigned long mstart
, mend
;
387 spin_lock(&priv
->lock
);
388 list_for_each_entry(map
, &priv
->maps
, next
) {
391 if (map
->vma
->vm_start
>= end
)
393 if (map
->vma
->vm_end
<= start
)
395 mstart
= max(start
, map
->vma
->vm_start
);
396 mend
= min(end
, map
->vma
->vm_end
);
397 pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
398 map
->index
, map
->count
,
399 map
->vma
->vm_start
, map
->vma
->vm_end
,
400 start
, end
, mstart
, mend
);
401 err
= unmap_grant_pages(map
,
402 (mstart
- map
->vma
->vm_start
) >> PAGE_SHIFT
,
403 (mend
- mstart
) >> PAGE_SHIFT
);
406 spin_unlock(&priv
->lock
);
409 static void mn_invl_page(struct mmu_notifier
*mn
,
410 struct mm_struct
*mm
,
411 unsigned long address
)
413 mn_invl_range_start(mn
, mm
, address
, address
+ PAGE_SIZE
);
416 static void mn_release(struct mmu_notifier
*mn
,
417 struct mm_struct
*mm
)
419 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
420 struct grant_map
*map
;
423 spin_lock(&priv
->lock
);
424 list_for_each_entry(map
, &priv
->maps
, next
) {
427 pr_debug("map %d+%d (%lx %lx)\n",
428 map
->index
, map
->count
,
429 map
->vma
->vm_start
, map
->vma
->vm_end
);
430 err
= unmap_grant_pages(map
, /* offset */ 0, map
->count
);
433 spin_unlock(&priv
->lock
);
436 struct mmu_notifier_ops gntdev_mmu_ops
= {
437 .release
= mn_release
,
438 .invalidate_page
= mn_invl_page
,
439 .invalidate_range_start
= mn_invl_range_start
,
442 /* ------------------------------------------------------------------ */
444 static int gntdev_open(struct inode
*inode
, struct file
*flip
)
446 struct gntdev_priv
*priv
;
449 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
453 INIT_LIST_HEAD(&priv
->maps
);
454 spin_lock_init(&priv
->lock
);
457 priv
->mm
= get_task_mm(current
);
462 priv
->mn
.ops
= &gntdev_mmu_ops
;
463 ret
= mmu_notifier_register(&priv
->mn
, priv
->mm
);
472 flip
->private_data
= priv
;
473 pr_debug("priv %p\n", priv
);
478 static int gntdev_release(struct inode
*inode
, struct file
*flip
)
480 struct gntdev_priv
*priv
= flip
->private_data
;
481 struct grant_map
*map
;
483 pr_debug("priv %p\n", priv
);
485 spin_lock(&priv
->lock
);
486 while (!list_empty(&priv
->maps
)) {
487 map
= list_entry(priv
->maps
.next
, struct grant_map
, next
);
488 list_del(&map
->next
);
491 spin_unlock(&priv
->lock
);
494 mmu_notifier_unregister(&priv
->mn
, priv
->mm
);
499 static long gntdev_ioctl_map_grant_ref(struct gntdev_priv
*priv
,
500 struct ioctl_gntdev_map_grant_ref __user
*u
)
502 struct ioctl_gntdev_map_grant_ref op
;
503 struct grant_map
*map
;
506 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
508 pr_debug("priv %p, add %d\n", priv
, op
.count
);
509 if (unlikely(op
.count
<= 0))
513 map
= gntdev_alloc_map(priv
, op
.count
);
517 if (unlikely(atomic_add_return(op
.count
, &pages_mapped
) > limit
)) {
518 pr_debug("can't map: over limit\n");
523 if (copy_from_user(map
->grants
, &u
->refs
,
524 sizeof(map
->grants
[0]) * op
.count
) != 0) {
529 spin_lock(&priv
->lock
);
530 gntdev_add_map(priv
, map
);
531 op
.index
= map
->index
<< PAGE_SHIFT
;
532 spin_unlock(&priv
->lock
);
534 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
540 static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv
*priv
,
541 struct ioctl_gntdev_unmap_grant_ref __user
*u
)
543 struct ioctl_gntdev_unmap_grant_ref op
;
544 struct grant_map
*map
;
547 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
549 pr_debug("priv %p, del %d+%d\n", priv
, (int)op
.index
, (int)op
.count
);
551 spin_lock(&priv
->lock
);
552 map
= gntdev_find_map_index(priv
, op
.index
>> PAGE_SHIFT
, op
.count
);
554 list_del(&map
->next
);
558 spin_unlock(&priv
->lock
);
562 static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv
*priv
,
563 struct ioctl_gntdev_get_offset_for_vaddr __user
*u
)
565 struct ioctl_gntdev_get_offset_for_vaddr op
;
566 struct vm_area_struct
*vma
;
567 struct grant_map
*map
;
569 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
571 pr_debug("priv %p, offset for vaddr %lx\n", priv
, (unsigned long)op
.vaddr
);
573 vma
= find_vma(current
->mm
, op
.vaddr
);
574 if (!vma
|| vma
->vm_ops
!= &gntdev_vmops
)
577 map
= vma
->vm_private_data
;
581 op
.offset
= map
->index
<< PAGE_SHIFT
;
582 op
.count
= map
->count
;
584 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
589 static long gntdev_ioctl_notify(struct gntdev_priv
*priv
, void __user
*u
)
591 struct ioctl_gntdev_unmap_notify op
;
592 struct grant_map
*map
;
595 if (copy_from_user(&op
, u
, sizeof(op
)))
598 if (op
.action
& ~(UNMAP_NOTIFY_CLEAR_BYTE
|UNMAP_NOTIFY_SEND_EVENT
))
601 spin_lock(&priv
->lock
);
603 list_for_each_entry(map
, &priv
->maps
, next
) {
604 uint64_t begin
= map
->index
<< PAGE_SHIFT
;
605 uint64_t end
= (map
->index
+ map
->count
) << PAGE_SHIFT
;
606 if (op
.index
>= begin
&& op
.index
< end
)
613 if ((op
.action
& UNMAP_NOTIFY_CLEAR_BYTE
) &&
614 (map
->flags
& GNTMAP_readonly
)) {
619 map
->notify
.flags
= op
.action
;
620 map
->notify
.addr
= op
.index
- (map
->index
<< PAGE_SHIFT
);
621 map
->notify
.event
= op
.event_channel_port
;
624 spin_unlock(&priv
->lock
);
628 static long gntdev_ioctl(struct file
*flip
,
629 unsigned int cmd
, unsigned long arg
)
631 struct gntdev_priv
*priv
= flip
->private_data
;
632 void __user
*ptr
= (void __user
*)arg
;
635 case IOCTL_GNTDEV_MAP_GRANT_REF
:
636 return gntdev_ioctl_map_grant_ref(priv
, ptr
);
638 case IOCTL_GNTDEV_UNMAP_GRANT_REF
:
639 return gntdev_ioctl_unmap_grant_ref(priv
, ptr
);
641 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR
:
642 return gntdev_ioctl_get_offset_for_vaddr(priv
, ptr
);
644 case IOCTL_GNTDEV_SET_UNMAP_NOTIFY
:
645 return gntdev_ioctl_notify(priv
, ptr
);
648 pr_debug("priv %p, unknown cmd %x\n", priv
, cmd
);
655 static int gntdev_mmap(struct file
*flip
, struct vm_area_struct
*vma
)
657 struct gntdev_priv
*priv
= flip
->private_data
;
658 int index
= vma
->vm_pgoff
;
659 int count
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
660 struct grant_map
*map
;
661 int i
, err
= -EINVAL
;
663 if ((vma
->vm_flags
& VM_WRITE
) && !(vma
->vm_flags
& VM_SHARED
))
666 pr_debug("map %d+%d at %lx (pgoff %lx)\n",
667 index
, count
, vma
->vm_start
, vma
->vm_pgoff
);
669 spin_lock(&priv
->lock
);
670 map
= gntdev_find_map_index(priv
, index
, count
);
673 if (use_ptemod
&& map
->vma
)
675 if (use_ptemod
&& priv
->mm
!= vma
->vm_mm
) {
676 printk(KERN_WARNING
"Huh? Other mm?\n");
680 atomic_inc(&map
->users
);
682 vma
->vm_ops
= &gntdev_vmops
;
684 vma
->vm_flags
|= VM_RESERVED
|VM_DONTCOPY
|VM_DONTEXPAND
|VM_PFNMAP
;
686 vma
->vm_private_data
= map
;
692 if ((vma
->vm_flags
& VM_WRITE
) &&
693 (map
->flags
& GNTMAP_readonly
))
696 map
->flags
= GNTMAP_host_map
;
697 if (!(vma
->vm_flags
& VM_WRITE
))
698 map
->flags
|= GNTMAP_readonly
;
701 spin_unlock(&priv
->lock
);
704 err
= apply_to_page_range(vma
->vm_mm
, vma
->vm_start
,
705 vma
->vm_end
- vma
->vm_start
,
706 find_grant_ptes
, map
);
708 printk(KERN_WARNING
"find_grant_ptes() failure.\n");
713 err
= map_grant_pages(map
);
718 for (i
= 0; i
< count
; i
++) {
719 err
= vm_insert_page(vma
, vma
->vm_start
+ i
*PAGE_SIZE
,
729 spin_unlock(&priv
->lock
);
739 static const struct file_operations gntdev_fops
= {
740 .owner
= THIS_MODULE
,
742 .release
= gntdev_release
,
744 .unlocked_ioctl
= gntdev_ioctl
747 static struct miscdevice gntdev_miscdev
= {
748 .minor
= MISC_DYNAMIC_MINOR
,
749 .name
= "xen/gntdev",
750 .fops
= &gntdev_fops
,
753 /* ------------------------------------------------------------------ */
755 static int __init
gntdev_init(void)
762 use_ptemod
= xen_pv_domain();
764 err
= misc_register(&gntdev_miscdev
);
766 printk(KERN_ERR
"Could not register gntdev device\n");
772 static void __exit
gntdev_exit(void)
774 misc_deregister(&gntdev_miscdev
);
777 module_init(gntdev_init
);
778 module_exit(gntdev_exit
);
780 /* ------------------------------------------------------------------ */