1 /******************************************************************************
4 * Device for accessing (in user-space) pages that have been granted by other
7 * Copyright (c) 2006-2007, D G Murray.
8 * (c) 2009 Gerd Hoffmann <kraxel@redhat.com>
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #include <linux/module.h>
23 #include <linux/kernel.h>
24 #include <linux/init.h>
25 #include <linux/miscdevice.h>
28 #include <linux/mman.h>
29 #include <linux/mmu_notifier.h>
30 #include <linux/types.h>
31 #include <linux/uaccess.h>
32 #include <linux/sched.h>
33 #include <linux/spinlock.h>
34 #include <linux/slab.h>
37 #include <xen/grant_table.h>
38 #include <xen/gntdev.h>
39 #include <asm/xen/hypervisor.h>
40 #include <asm/xen/hypercall.h>
41 #include <asm/xen/page.h>
43 MODULE_LICENSE("GPL");
44 MODULE_AUTHOR("Derek G. Murray <Derek.Murray@cl.cam.ac.uk>, "
45 "Gerd Hoffmann <kraxel@redhat.com>");
46 MODULE_DESCRIPTION("User-space granted page access driver");
48 static int limit
= 1024;
49 module_param(limit
, int, 0644);
50 MODULE_PARM_DESC(limit
, "Maximum number of grants that may be mapped at "
51 "once by a gntdev instance");
54 struct list_head maps
;
57 /* lock protects maps from concurrent changes */
60 struct mmu_notifier mn
;
64 struct list_head next
;
65 struct gntdev_priv
*priv
;
66 struct vm_area_struct
*vma
;
71 struct ioctl_gntdev_grant_ref
*grants
;
72 struct gnttab_map_grant_ref
*map_ops
;
73 struct gnttab_unmap_grant_ref
*unmap_ops
;
76 /* ------------------------------------------------------------------ */
78 static void gntdev_print_maps(struct gntdev_priv
*priv
,
79 char *text
, int text_index
)
82 struct grant_map
*map
;
84 pr_debug("maps list (priv %p, usage %d/%d)\n",
85 priv
, priv
->used
, priv
->limit
);
87 list_for_each_entry(map
, &priv
->maps
, next
)
88 pr_debug(" index %2d, count %2d %s\n",
89 map
->index
, map
->count
,
90 map
->index
== text_index
&& text
? text
: "");
94 static struct grant_map
*gntdev_alloc_map(struct gntdev_priv
*priv
, int count
)
96 struct grant_map
*add
;
98 add
= kzalloc(sizeof(struct grant_map
), GFP_KERNEL
);
102 add
->grants
= kzalloc(sizeof(add
->grants
[0]) * count
, GFP_KERNEL
);
103 add
->map_ops
= kzalloc(sizeof(add
->map_ops
[0]) * count
, GFP_KERNEL
);
104 add
->unmap_ops
= kzalloc(sizeof(add
->unmap_ops
[0]) * count
, GFP_KERNEL
);
105 if (NULL
== add
->grants
||
106 NULL
== add
->map_ops
||
107 NULL
== add
->unmap_ops
)
114 if (add
->count
+ priv
->used
> priv
->limit
)
122 kfree(add
->unmap_ops
);
127 static void gntdev_add_map(struct gntdev_priv
*priv
, struct grant_map
*add
)
129 struct grant_map
*map
;
131 list_for_each_entry(map
, &priv
->maps
, next
) {
132 if (add
->index
+ add
->count
< map
->index
) {
133 list_add_tail(&add
->next
, &map
->next
);
136 add
->index
= map
->index
+ map
->count
;
138 list_add_tail(&add
->next
, &priv
->maps
);
141 priv
->used
+= add
->count
;
142 gntdev_print_maps(priv
, "[new]", add
->index
);
145 static struct grant_map
*gntdev_find_map_index(struct gntdev_priv
*priv
,
146 int index
, int count
)
148 struct grant_map
*map
;
150 list_for_each_entry(map
, &priv
->maps
, next
) {
151 if (map
->index
!= index
)
153 if (map
->count
!= count
)
160 static struct grant_map
*gntdev_find_map_vaddr(struct gntdev_priv
*priv
,
163 struct grant_map
*map
;
165 list_for_each_entry(map
, &priv
->maps
, next
) {
168 if (vaddr
< map
->vma
->vm_start
)
170 if (vaddr
>= map
->vma
->vm_end
)
177 static int gntdev_del_map(struct grant_map
*map
)
183 for (i
= 0; i
< map
->count
; i
++)
184 if (map
->unmap_ops
[i
].handle
)
187 map
->priv
->used
-= map
->count
;
188 list_del(&map
->next
);
192 static void gntdev_free_map(struct grant_map
*map
)
198 kfree(map
->unmap_ops
);
202 /* ------------------------------------------------------------------ */
204 static int find_grant_ptes(pte_t
*pte
, pgtable_t token
,
205 unsigned long addr
, void *data
)
207 struct grant_map
*map
= data
;
208 unsigned int pgnr
= (addr
- map
->vma
->vm_start
) >> PAGE_SHIFT
;
211 BUG_ON(pgnr
>= map
->count
);
212 pte_maddr
= arbitrary_virt_to_machine(pte
).maddr
;
214 gnttab_set_map_op(&map
->map_ops
[pgnr
], pte_maddr
,
215 GNTMAP_contains_pte
| map
->flags
,
216 map
->grants
[pgnr
].ref
,
217 map
->grants
[pgnr
].domid
);
218 gnttab_set_unmap_op(&map
->unmap_ops
[pgnr
], pte_maddr
,
219 GNTMAP_contains_pte
| map
->flags
,
224 static int map_grant_pages(struct grant_map
*map
)
228 pr_debug("map %d+%d\n", map
->index
, map
->count
);
229 err
= HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref
,
230 map
->map_ops
, map
->count
);
234 for (i
= 0; i
< map
->count
; i
++) {
235 if (map
->map_ops
[i
].status
)
237 map
->unmap_ops
[i
].handle
= map
->map_ops
[i
].handle
;
242 static int unmap_grant_pages(struct grant_map
*map
, int offset
, int pages
)
246 pr_debug("map %d+%d [%d+%d]\n", map
->index
, map
->count
, offset
, pages
);
247 err
= HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref
,
248 map
->unmap_ops
+ offset
, pages
);
252 for (i
= 0; i
< pages
; i
++) {
253 if (map
->unmap_ops
[offset
+i
].status
)
255 map
->unmap_ops
[offset
+i
].handle
= 0;
260 /* ------------------------------------------------------------------ */
262 static void gntdev_vma_close(struct vm_area_struct
*vma
)
264 struct grant_map
*map
= vma
->vm_private_data
;
266 pr_debug("close %p\n", vma
);
269 vma
->vm_private_data
= NULL
;
272 static int gntdev_vma_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
274 pr_debug("vaddr %p, pgoff %ld (shouldn't happen)\n",
275 vmf
->virtual_address
, vmf
->pgoff
);
276 vmf
->flags
= VM_FAULT_ERROR
;
280 static struct vm_operations_struct gntdev_vmops
= {
281 .close
= gntdev_vma_close
,
282 .fault
= gntdev_vma_fault
,
285 /* ------------------------------------------------------------------ */
287 static void mn_invl_range_start(struct mmu_notifier
*mn
,
288 struct mm_struct
*mm
,
289 unsigned long start
, unsigned long end
)
291 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
292 struct grant_map
*map
;
293 unsigned long mstart
, mend
;
296 spin_lock(&priv
->lock
);
297 list_for_each_entry(map
, &priv
->maps
, next
) {
302 if (map
->vma
->vm_start
>= end
)
304 if (map
->vma
->vm_end
<= start
)
306 mstart
= max(start
, map
->vma
->vm_start
);
307 mend
= min(end
, map
->vma
->vm_end
);
308 pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
309 map
->index
, map
->count
,
310 map
->vma
->vm_start
, map
->vma
->vm_end
,
311 start
, end
, mstart
, mend
);
312 err
= unmap_grant_pages(map
,
313 (mstart
- map
->vma
->vm_start
) >> PAGE_SHIFT
,
314 (mend
- mstart
) >> PAGE_SHIFT
);
317 spin_unlock(&priv
->lock
);
320 static void mn_invl_page(struct mmu_notifier
*mn
,
321 struct mm_struct
*mm
,
322 unsigned long address
)
324 mn_invl_range_start(mn
, mm
, address
, address
+ PAGE_SIZE
);
327 static void mn_release(struct mmu_notifier
*mn
,
328 struct mm_struct
*mm
)
330 struct gntdev_priv
*priv
= container_of(mn
, struct gntdev_priv
, mn
);
331 struct grant_map
*map
;
334 spin_lock(&priv
->lock
);
335 list_for_each_entry(map
, &priv
->maps
, next
) {
338 pr_debug("map %d+%d (%lx %lx)\n",
339 map
->index
, map
->count
,
340 map
->vma
->vm_start
, map
->vma
->vm_end
);
341 err
= unmap_grant_pages(map
, /* offset */ 0, map
->count
);
344 spin_unlock(&priv
->lock
);
347 struct mmu_notifier_ops gntdev_mmu_ops
= {
348 .release
= mn_release
,
349 .invalidate_page
= mn_invl_page
,
350 .invalidate_range_start
= mn_invl_range_start
,
353 /* ------------------------------------------------------------------ */
355 static int gntdev_open(struct inode
*inode
, struct file
*flip
)
357 struct gntdev_priv
*priv
;
360 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
364 INIT_LIST_HEAD(&priv
->maps
);
365 spin_lock_init(&priv
->lock
);
368 priv
->mm
= get_task_mm(current
);
373 priv
->mn
.ops
= &gntdev_mmu_ops
;
374 ret
= mmu_notifier_register(&priv
->mn
, priv
->mm
);
382 flip
->private_data
= priv
;
383 pr_debug("priv %p\n", priv
);
388 static int gntdev_release(struct inode
*inode
, struct file
*flip
)
390 struct gntdev_priv
*priv
= flip
->private_data
;
391 struct grant_map
*map
;
394 pr_debug("priv %p\n", priv
);
396 spin_lock(&priv
->lock
);
397 while (!list_empty(&priv
->maps
)) {
398 map
= list_entry(priv
->maps
.next
, struct grant_map
, next
);
399 err
= gntdev_del_map(map
);
401 gntdev_free_map(map
);
404 spin_unlock(&priv
->lock
);
406 mmu_notifier_unregister(&priv
->mn
, priv
->mm
);
411 static long gntdev_ioctl_map_grant_ref(struct gntdev_priv
*priv
,
412 struct ioctl_gntdev_map_grant_ref __user
*u
)
414 struct ioctl_gntdev_map_grant_ref op
;
415 struct grant_map
*map
;
418 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
420 pr_debug("priv %p, add %d\n", priv
, op
.count
);
421 if (unlikely(op
.count
<= 0))
423 if (unlikely(op
.count
> priv
->limit
))
427 map
= gntdev_alloc_map(priv
, op
.count
);
430 if (copy_from_user(map
->grants
, &u
->refs
,
431 sizeof(map
->grants
[0]) * op
.count
) != 0) {
432 gntdev_free_map(map
);
436 spin_lock(&priv
->lock
);
437 gntdev_add_map(priv
, map
);
438 op
.index
= map
->index
<< PAGE_SHIFT
;
439 spin_unlock(&priv
->lock
);
441 if (copy_to_user(u
, &op
, sizeof(op
)) != 0) {
442 spin_lock(&priv
->lock
);
444 spin_unlock(&priv
->lock
);
445 gntdev_free_map(map
);
451 static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv
*priv
,
452 struct ioctl_gntdev_unmap_grant_ref __user
*u
)
454 struct ioctl_gntdev_unmap_grant_ref op
;
455 struct grant_map
*map
;
458 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
460 pr_debug("priv %p, del %d+%d\n", priv
, (int)op
.index
, (int)op
.count
);
462 spin_lock(&priv
->lock
);
463 map
= gntdev_find_map_index(priv
, op
.index
>> PAGE_SHIFT
, op
.count
);
465 err
= gntdev_del_map(map
);
466 spin_unlock(&priv
->lock
);
468 gntdev_free_map(map
);
472 static long gntdev_ioctl_get_offset_for_vaddr(struct gntdev_priv
*priv
,
473 struct ioctl_gntdev_get_offset_for_vaddr __user
*u
)
475 struct ioctl_gntdev_get_offset_for_vaddr op
;
476 struct grant_map
*map
;
478 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
480 pr_debug("priv %p, offset for vaddr %lx\n", priv
, (unsigned long)op
.vaddr
);
482 spin_lock(&priv
->lock
);
483 map
= gntdev_find_map_vaddr(priv
, op
.vaddr
);
485 map
->vma
->vm_start
!= op
.vaddr
) {
486 spin_unlock(&priv
->lock
);
489 op
.offset
= map
->index
<< PAGE_SHIFT
;
490 op
.count
= map
->count
;
491 spin_unlock(&priv
->lock
);
493 if (copy_to_user(u
, &op
, sizeof(op
)) != 0)
498 static long gntdev_ioctl_set_max_grants(struct gntdev_priv
*priv
,
499 struct ioctl_gntdev_set_max_grants __user
*u
)
501 struct ioctl_gntdev_set_max_grants op
;
503 if (copy_from_user(&op
, u
, sizeof(op
)) != 0)
505 pr_debug("priv %p, limit %d\n", priv
, op
.count
);
506 if (op
.count
> limit
)
509 spin_lock(&priv
->lock
);
510 priv
->limit
= op
.count
;
511 spin_unlock(&priv
->lock
);
515 static long gntdev_ioctl(struct file
*flip
,
516 unsigned int cmd
, unsigned long arg
)
518 struct gntdev_priv
*priv
= flip
->private_data
;
519 void __user
*ptr
= (void __user
*)arg
;
522 case IOCTL_GNTDEV_MAP_GRANT_REF
:
523 return gntdev_ioctl_map_grant_ref(priv
, ptr
);
525 case IOCTL_GNTDEV_UNMAP_GRANT_REF
:
526 return gntdev_ioctl_unmap_grant_ref(priv
, ptr
);
528 case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR
:
529 return gntdev_ioctl_get_offset_for_vaddr(priv
, ptr
);
531 case IOCTL_GNTDEV_SET_MAX_GRANTS
:
532 return gntdev_ioctl_set_max_grants(priv
, ptr
);
535 pr_debug("priv %p, unknown cmd %x\n", priv
, cmd
);
542 static int gntdev_mmap(struct file
*flip
, struct vm_area_struct
*vma
)
544 struct gntdev_priv
*priv
= flip
->private_data
;
545 int index
= vma
->vm_pgoff
;
546 int count
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
547 struct grant_map
*map
;
550 if ((vma
->vm_flags
& VM_WRITE
) && !(vma
->vm_flags
& VM_SHARED
))
553 pr_debug("map %d+%d at %lx (pgoff %lx)\n",
554 index
, count
, vma
->vm_start
, vma
->vm_pgoff
);
556 spin_lock(&priv
->lock
);
557 map
= gntdev_find_map_index(priv
, index
, count
);
562 if (priv
->mm
!= vma
->vm_mm
) {
563 printk(KERN_WARNING
"Huh? Other mm?\n");
567 vma
->vm_ops
= &gntdev_vmops
;
569 vma
->vm_flags
|= VM_RESERVED
|VM_DONTCOPY
|VM_DONTEXPAND
|VM_PFNMAP
;
571 vma
->vm_private_data
= map
;
574 map
->flags
= GNTMAP_host_map
| GNTMAP_application_map
;
575 if (!(vma
->vm_flags
& VM_WRITE
))
576 map
->flags
|= GNTMAP_readonly
;
578 spin_unlock(&priv
->lock
);
580 err
= apply_to_page_range(vma
->vm_mm
, vma
->vm_start
,
581 vma
->vm_end
- vma
->vm_start
,
582 find_grant_ptes
, map
);
584 printk(KERN_WARNING
"find_grant_ptes() failure.\n");
588 err
= map_grant_pages(map
);
590 printk(KERN_WARNING
"map_grant_pages() failure.\n");
599 spin_unlock(&priv
->lock
);
603 static const struct file_operations gntdev_fops
= {
604 .owner
= THIS_MODULE
,
606 .release
= gntdev_release
,
608 .unlocked_ioctl
= gntdev_ioctl
611 static struct miscdevice gntdev_miscdev
= {
612 .minor
= MISC_DYNAMIC_MINOR
,
613 .name
= "xen/gntdev",
614 .fops
= &gntdev_fops
,
617 /* ------------------------------------------------------------------ */
619 static int __init
gntdev_init(void)
626 err
= misc_register(&gntdev_miscdev
);
628 printk(KERN_ERR
"Could not register gntdev device\n");
634 static void __exit
gntdev_exit(void)
636 misc_deregister(&gntdev_miscdev
);
639 module_init(gntdev_init
);
640 module_exit(gntdev_exit
);
642 /* ------------------------------------------------------------------ */