2 * User-space Probes (UProbes)
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright (C) IBM Corporation, 2008-2011
24 #include <linux/kernel.h>
25 #include <linux/highmem.h>
26 #include <linux/pagemap.h> /* read_mapping_page */
27 #include <linux/slab.h>
28 #include <linux/sched.h>
29 #include <linux/rmap.h> /* anon_vma_prepare */
30 #include <linux/mmu_notifier.h> /* set_pte_at_notify */
31 #include <linux/swap.h> /* try_to_free_swap */
33 #include <linux/uprobes.h>
35 static struct rb_root uprobes_tree
= RB_ROOT
;
37 static DEFINE_SPINLOCK(uprobes_treelock
); /* serialize rbtree access */
39 #define UPROBES_HASH_SZ 13
41 /* serialize (un)register */
42 static struct mutex uprobes_mutex
[UPROBES_HASH_SZ
];
44 #define uprobes_hash(v) (&uprobes_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
46 /* serialize uprobe->pending_list */
47 static struct mutex uprobes_mmap_mutex
[UPROBES_HASH_SZ
];
48 #define uprobes_mmap_hash(v) (&uprobes_mmap_mutex[((unsigned long)(v)) % UPROBES_HASH_SZ])
51 * uprobe_events allows us to skip the uprobe_mmap if there are no uprobe
52 * events active at this time. Probably a fine grained per inode count is
55 static atomic_t uprobe_events
= ATOMIC_INIT(0);
58 * Maintain a temporary per vma info that can be used to search if a vma
59 * has already been handled. This structure is introduced since extending
60 * vm_area_struct wasnt recommended.
63 struct list_head probe_list
;
69 struct rb_node rb_node
; /* node in the rb tree */
71 struct rw_semaphore consumer_rwsem
;
72 struct list_head pending_list
;
73 struct uprobe_consumer
*consumers
;
74 struct inode
*inode
; /* Also hold a ref to inode */
77 struct arch_uprobe arch
;
81 * valid_vma: Verify if the specified vma is an executable vma
82 * Relax restrictions while unregistering: vm_flags might have
83 * changed after breakpoint was inserted.
84 * - is_register: indicates if we are in register context.
85 * - Return 1 if the specified virtual address is in an
88 static bool valid_vma(struct vm_area_struct
*vma
, bool is_register
)
96 if ((vma
->vm_flags
& (VM_READ
|VM_WRITE
|VM_EXEC
|VM_SHARED
)) == (VM_READ
|VM_EXEC
))
102 static loff_t
vma_address(struct vm_area_struct
*vma
, loff_t offset
)
106 vaddr
= vma
->vm_start
+ offset
;
107 vaddr
-= vma
->vm_pgoff
<< PAGE_SHIFT
;
113 * __replace_page - replace page in vma by new page.
114 * based on replace_page in mm/ksm.c
116 * @vma: vma that holds the pte pointing to page
117 * @page: the cowed page we are replacing by kpage
118 * @kpage: the modified page we replace page by
120 * Returns 0 on success, -EFAULT on failure.
122 static int __replace_page(struct vm_area_struct
*vma
, struct page
*page
, struct page
*kpage
)
124 struct mm_struct
*mm
= vma
->vm_mm
;
133 addr
= page_address_in_vma(page
, vma
);
137 pgd
= pgd_offset(mm
, addr
);
138 if (!pgd_present(*pgd
))
141 pud
= pud_offset(pgd
, addr
);
142 if (!pud_present(*pud
))
145 pmd
= pmd_offset(pud
, addr
);
146 if (!pmd_present(*pmd
))
149 ptep
= pte_offset_map_lock(mm
, pmd
, addr
, &ptl
);
154 page_add_new_anon_rmap(kpage
, vma
, addr
);
156 flush_cache_page(vma
, addr
, pte_pfn(*ptep
));
157 ptep_clear_flush(vma
, addr
, ptep
);
158 set_pte_at_notify(mm
, addr
, ptep
, mk_pte(kpage
, vma
->vm_page_prot
));
160 page_remove_rmap(page
);
161 if (!page_mapped(page
))
162 try_to_free_swap(page
);
164 pte_unmap_unlock(ptep
, ptl
);
172 * is_bkpt_insn - check if instruction is breakpoint instruction.
173 * @insn: instruction to be checked.
174 * Default implementation of is_bkpt_insn
175 * Returns true if @insn is a breakpoint instruction.
177 bool __weak
is_bkpt_insn(uprobe_opcode_t
*insn
)
179 return *insn
== UPROBES_BKPT_INSN
;
184 * Expect the breakpoint instruction to be the smallest size instruction for
185 * the architecture. If an arch has variable length instruction and the
186 * breakpoint instruction is not of the smallest length instruction
187 * supported by that architecture then we need to modify read_opcode /
188 * write_opcode accordingly. This would never be a problem for archs that
189 * have fixed length instructions.
193 * write_opcode - write the opcode at a given virtual address.
194 * @mm: the probed process address space.
195 * @arch_uprobe: the breakpointing information.
196 * @vaddr: the virtual address to store the opcode.
197 * @opcode: opcode to be written at @vaddr.
199 * Called with mm->mmap_sem held (for read and with a reference to
202 * For mm @mm, write the opcode at @vaddr.
203 * Return 0 (success) or a negative errno.
205 static int write_opcode(struct mm_struct
*mm
, struct arch_uprobe
*auprobe
,
206 unsigned long vaddr
, uprobe_opcode_t opcode
)
208 struct page
*old_page
, *new_page
;
209 struct address_space
*mapping
;
210 void *vaddr_old
, *vaddr_new
;
211 struct vm_area_struct
*vma
;
212 struct uprobe
*uprobe
;
216 /* Read the page with vaddr into memory */
217 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 0, &old_page
, &vma
);
224 * We are interested in text pages only. Our pages of interest
225 * should be mapped for read and execute only. We desist from
226 * adding probes in write mapped pages since the breakpoints
227 * might end up in the file copy.
229 if (!valid_vma(vma
, is_bkpt_insn(&opcode
)))
232 uprobe
= container_of(auprobe
, struct uprobe
, arch
);
233 mapping
= uprobe
->inode
->i_mapping
;
234 if (mapping
!= vma
->vm_file
->f_mapping
)
237 addr
= vma_address(vma
, uprobe
->offset
);
238 if (vaddr
!= (unsigned long)addr
)
242 new_page
= alloc_page_vma(GFP_HIGHUSER_MOVABLE
, vma
, vaddr
);
246 __SetPageUptodate(new_page
);
249 * lock page will serialize against do_wp_page()'s
250 * PageAnon() handling
253 /* copy the page now that we've got it stable */
254 vaddr_old
= kmap_atomic(old_page
);
255 vaddr_new
= kmap_atomic(new_page
);
257 memcpy(vaddr_new
, vaddr_old
, PAGE_SIZE
);
259 /* poke the new insn in, ASSUMES we don't cross page boundary */
261 BUG_ON(vaddr
+ UPROBES_BKPT_INSN_SIZE
> PAGE_SIZE
);
262 memcpy(vaddr_new
+ vaddr
, &opcode
, UPROBES_BKPT_INSN_SIZE
);
264 kunmap_atomic(vaddr_new
);
265 kunmap_atomic(vaddr_old
);
267 ret
= anon_vma_prepare(vma
);
272 ret
= __replace_page(vma
, old_page
, new_page
);
273 unlock_page(new_page
);
276 unlock_page(old_page
);
277 page_cache_release(new_page
);
286 * read_opcode - read the opcode at a given virtual address.
287 * @mm: the probed process address space.
288 * @vaddr: the virtual address to read the opcode.
289 * @opcode: location to store the read opcode.
291 * Called with mm->mmap_sem held (for read and with a reference to
294 * For mm @mm, read the opcode at @vaddr and store it in @opcode.
295 * Return 0 (success) or a negative errno.
297 static int read_opcode(struct mm_struct
*mm
, unsigned long vaddr
, uprobe_opcode_t
*opcode
)
303 ret
= get_user_pages(NULL
, mm
, vaddr
, 1, 0, 0, &page
, NULL
);
308 vaddr_new
= kmap_atomic(page
);
310 memcpy(opcode
, vaddr_new
+ vaddr
, UPROBES_BKPT_INSN_SIZE
);
311 kunmap_atomic(vaddr_new
);
319 static int is_bkpt_at_addr(struct mm_struct
*mm
, unsigned long vaddr
)
321 uprobe_opcode_t opcode
;
324 result
= read_opcode(mm
, vaddr
, &opcode
);
328 if (is_bkpt_insn(&opcode
))
335 * set_bkpt - store breakpoint at a given address.
336 * @mm: the probed process address space.
337 * @uprobe: the probepoint information.
338 * @vaddr: the virtual address to insert the opcode.
340 * For mm @mm, store the breakpoint instruction at @vaddr.
341 * Return 0 (success) or a negative errno.
343 int __weak
set_bkpt(struct mm_struct
*mm
, struct arch_uprobe
*auprobe
, unsigned long vaddr
)
347 result
= is_bkpt_at_addr(mm
, vaddr
);
354 return write_opcode(mm
, auprobe
, vaddr
, UPROBES_BKPT_INSN
);
358 * set_orig_insn - Restore the original instruction.
359 * @mm: the probed process address space.
360 * @uprobe: the probepoint information.
361 * @vaddr: the virtual address to insert the opcode.
362 * @verify: if true, verify existance of breakpoint instruction.
364 * For mm @mm, restore the original opcode (opcode) at @vaddr.
365 * Return 0 (success) or a negative errno.
368 set_orig_insn(struct mm_struct
*mm
, struct arch_uprobe
*auprobe
, unsigned long vaddr
, bool verify
)
373 result
= is_bkpt_at_addr(mm
, vaddr
);
380 return write_opcode(mm
, auprobe
, vaddr
, *(uprobe_opcode_t
*)auprobe
->insn
);
383 static int match_uprobe(struct uprobe
*l
, struct uprobe
*r
)
385 if (l
->inode
< r
->inode
)
388 if (l
->inode
> r
->inode
)
391 if (l
->offset
< r
->offset
)
394 if (l
->offset
> r
->offset
)
400 static struct uprobe
*__find_uprobe(struct inode
*inode
, loff_t offset
)
402 struct uprobe u
= { .inode
= inode
, .offset
= offset
};
403 struct rb_node
*n
= uprobes_tree
.rb_node
;
404 struct uprobe
*uprobe
;
408 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
409 match
= match_uprobe(&u
, uprobe
);
411 atomic_inc(&uprobe
->ref
);
424 * Find a uprobe corresponding to a given inode:offset
425 * Acquires uprobes_treelock
427 static struct uprobe
*find_uprobe(struct inode
*inode
, loff_t offset
)
429 struct uprobe
*uprobe
;
432 spin_lock_irqsave(&uprobes_treelock
, flags
);
433 uprobe
= __find_uprobe(inode
, offset
);
434 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
439 static struct uprobe
*__insert_uprobe(struct uprobe
*uprobe
)
441 struct rb_node
**p
= &uprobes_tree
.rb_node
;
442 struct rb_node
*parent
= NULL
;
448 u
= rb_entry(parent
, struct uprobe
, rb_node
);
449 match
= match_uprobe(uprobe
, u
);
456 p
= &parent
->rb_left
;
458 p
= &parent
->rb_right
;
463 rb_link_node(&uprobe
->rb_node
, parent
, p
);
464 rb_insert_color(&uprobe
->rb_node
, &uprobes_tree
);
465 /* get access + creation ref */
466 atomic_set(&uprobe
->ref
, 2);
472 * Acquire uprobes_treelock.
473 * Matching uprobe already exists in rbtree;
474 * increment (access refcount) and return the matching uprobe.
476 * No matching uprobe; insert the uprobe in rb_tree;
477 * get a double refcount (access + creation) and return NULL.
479 static struct uprobe
*insert_uprobe(struct uprobe
*uprobe
)
484 spin_lock_irqsave(&uprobes_treelock
, flags
);
485 u
= __insert_uprobe(uprobe
);
486 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
491 static void put_uprobe(struct uprobe
*uprobe
)
493 if (atomic_dec_and_test(&uprobe
->ref
))
497 static struct uprobe
*alloc_uprobe(struct inode
*inode
, loff_t offset
)
499 struct uprobe
*uprobe
, *cur_uprobe
;
501 uprobe
= kzalloc(sizeof(struct uprobe
), GFP_KERNEL
);
505 uprobe
->inode
= igrab(inode
);
506 uprobe
->offset
= offset
;
507 init_rwsem(&uprobe
->consumer_rwsem
);
508 INIT_LIST_HEAD(&uprobe
->pending_list
);
510 /* add to uprobes_tree, sorted on inode:offset */
511 cur_uprobe
= insert_uprobe(uprobe
);
513 /* a uprobe exists for this inode:offset combination */
519 atomic_inc(&uprobe_events
);
525 /* Returns the previous consumer */
526 static struct uprobe_consumer
*
527 consumer_add(struct uprobe
*uprobe
, struct uprobe_consumer
*consumer
)
529 down_write(&uprobe
->consumer_rwsem
);
530 consumer
->next
= uprobe
->consumers
;
531 uprobe
->consumers
= consumer
;
532 up_write(&uprobe
->consumer_rwsem
);
534 return consumer
->next
;
538 * For uprobe @uprobe, delete the consumer @consumer.
539 * Return true if the @consumer is deleted successfully
542 static bool consumer_del(struct uprobe
*uprobe
, struct uprobe_consumer
*consumer
)
544 struct uprobe_consumer
**con
;
547 down_write(&uprobe
->consumer_rwsem
);
548 for (con
= &uprobe
->consumers
; *con
; con
= &(*con
)->next
) {
549 if (*con
== consumer
) {
550 *con
= consumer
->next
;
555 up_write(&uprobe
->consumer_rwsem
);
560 static int __copy_insn(struct address_space
*mapping
,
561 struct vm_area_struct
*vma
, char *insn
,
562 unsigned long nbytes
, unsigned long offset
)
564 struct file
*filp
= vma
->vm_file
;
573 idx
= (unsigned long)(offset
>> PAGE_CACHE_SHIFT
);
574 off1
= offset
&= ~PAGE_MASK
;
577 * Ensure that the page that has the original instruction is
578 * populated and in page-cache.
580 page
= read_mapping_page(mapping
, idx
, filp
);
582 return PTR_ERR(page
);
584 vaddr
= kmap_atomic(page
);
585 memcpy(insn
, vaddr
+ off1
, nbytes
);
586 kunmap_atomic(vaddr
);
587 page_cache_release(page
);
592 static int copy_insn(struct uprobe
*uprobe
, struct vm_area_struct
*vma
, unsigned long addr
)
594 struct address_space
*mapping
;
595 unsigned long nbytes
;
599 nbytes
= PAGE_SIZE
- addr
;
600 mapping
= uprobe
->inode
->i_mapping
;
602 /* Instruction at end of binary; copy only available bytes */
603 if (uprobe
->offset
+ MAX_UINSN_BYTES
> uprobe
->inode
->i_size
)
604 bytes
= uprobe
->inode
->i_size
- uprobe
->offset
;
606 bytes
= MAX_UINSN_BYTES
;
608 /* Instruction at the page-boundary; copy bytes in second page */
609 if (nbytes
< bytes
) {
610 if (__copy_insn(mapping
, vma
, uprobe
->arch
.insn
+ nbytes
,
611 bytes
- nbytes
, uprobe
->offset
+ nbytes
))
616 return __copy_insn(mapping
, vma
, uprobe
->arch
.insn
, bytes
, uprobe
->offset
);
619 static int install_breakpoint(struct mm_struct
*mm
, struct uprobe
*uprobe
,
620 struct vm_area_struct
*vma
, loff_t vaddr
)
626 * If probe is being deleted, unregister thread could be done with
627 * the vma-rmap-walk through. Adding a probe now can be fatal since
628 * nobody will be able to cleanup. Also we could be from fork or
629 * mremap path, where the probe might have already been inserted.
630 * Hence behave as if probe already existed.
632 if (!uprobe
->consumers
)
635 addr
= (unsigned long)vaddr
;
637 if (!(uprobe
->flags
& UPROBES_COPY_INSN
)) {
638 ret
= copy_insn(uprobe
, vma
, addr
);
642 if (is_bkpt_insn((uprobe_opcode_t
*)uprobe
->arch
.insn
))
645 ret
= arch_uprobes_analyze_insn(mm
, &uprobe
->arch
);
649 uprobe
->flags
|= UPROBES_COPY_INSN
;
651 ret
= set_bkpt(mm
, &uprobe
->arch
, addr
);
656 static void remove_breakpoint(struct mm_struct
*mm
, struct uprobe
*uprobe
, loff_t vaddr
)
658 set_orig_insn(mm
, &uprobe
->arch
, (unsigned long)vaddr
, true);
661 static void delete_uprobe(struct uprobe
*uprobe
)
665 spin_lock_irqsave(&uprobes_treelock
, flags
);
666 rb_erase(&uprobe
->rb_node
, &uprobes_tree
);
667 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
670 atomic_dec(&uprobe_events
);
673 static struct vma_info
*__find_next_vma_info(struct list_head
*head
,
674 loff_t offset
, struct address_space
*mapping
,
675 struct vma_info
*vi
, bool is_register
)
677 struct prio_tree_iter iter
;
678 struct vm_area_struct
*vma
;
679 struct vma_info
*tmpvi
;
684 pgoff
= offset
>> PAGE_SHIFT
;
686 vma_prio_tree_foreach(vma
, &iter
, &mapping
->i_mmap
, pgoff
, pgoff
) {
687 if (!valid_vma(vma
, is_register
))
691 vaddr
= vma_address(vma
, offset
);
693 list_for_each_entry(tmpvi
, head
, probe_list
) {
694 if (tmpvi
->mm
== vma
->vm_mm
&& tmpvi
->vaddr
== vaddr
) {
701 * Another vma needs a probe to be installed. However skip
702 * installing the probe if the vma is about to be unlinked.
704 if (!existing_vma
&& atomic_inc_not_zero(&vma
->vm_mm
->mm_users
)) {
707 list_add(&vi
->probe_list
, head
);
717 * Iterate in the rmap prio tree and find a vma where a probe has not
720 static struct vma_info
*
721 find_next_vma_info(struct list_head
*head
, loff_t offset
, struct address_space
*mapping
,
724 struct vma_info
*vi
, *retvi
;
726 vi
= kzalloc(sizeof(struct vma_info
), GFP_KERNEL
);
728 return ERR_PTR(-ENOMEM
);
730 mutex_lock(&mapping
->i_mmap_mutex
);
731 retvi
= __find_next_vma_info(head
, offset
, mapping
, vi
, is_register
);
732 mutex_unlock(&mapping
->i_mmap_mutex
);
740 static int register_for_each_vma(struct uprobe
*uprobe
, bool is_register
)
742 struct list_head try_list
;
743 struct vm_area_struct
*vma
;
744 struct address_space
*mapping
;
745 struct vma_info
*vi
, *tmpvi
;
746 struct mm_struct
*mm
;
750 mapping
= uprobe
->inode
->i_mapping
;
751 INIT_LIST_HEAD(&try_list
);
756 vi
= find_next_vma_info(&try_list
, uprobe
->offset
, mapping
, is_register
);
766 down_read(&mm
->mmap_sem
);
767 vma
= find_vma(mm
, (unsigned long)vi
->vaddr
);
768 if (!vma
|| !valid_vma(vma
, is_register
)) {
769 list_del(&vi
->probe_list
);
771 up_read(&mm
->mmap_sem
);
775 vaddr
= vma_address(vma
, uprobe
->offset
);
776 if (vma
->vm_file
->f_mapping
->host
!= uprobe
->inode
||
777 vaddr
!= vi
->vaddr
) {
778 list_del(&vi
->probe_list
);
780 up_read(&mm
->mmap_sem
);
786 ret
= install_breakpoint(mm
, uprobe
, vma
, vi
->vaddr
);
788 remove_breakpoint(mm
, uprobe
, vi
->vaddr
);
790 up_read(&mm
->mmap_sem
);
793 if (ret
&& ret
== -EEXIST
)
800 list_for_each_entry_safe(vi
, tmpvi
, &try_list
, probe_list
) {
801 list_del(&vi
->probe_list
);
808 static int __uprobe_register(struct uprobe
*uprobe
)
810 return register_for_each_vma(uprobe
, true);
813 static void __uprobe_unregister(struct uprobe
*uprobe
)
815 if (!register_for_each_vma(uprobe
, false))
816 delete_uprobe(uprobe
);
818 /* TODO : cant unregister? schedule a worker thread */
822 * uprobe_register - register a probe
823 * @inode: the file in which the probe has to be placed.
824 * @offset: offset from the start of the file.
825 * @consumer: information on howto handle the probe..
827 * Apart from the access refcount, uprobe_register() takes a creation
828 * refcount (thro alloc_uprobe) if and only if this @uprobe is getting
829 * inserted into the rbtree (i.e first consumer for a @inode:@offset
830 * tuple). Creation refcount stops uprobe_unregister from freeing the
831 * @uprobe even before the register operation is complete. Creation
832 * refcount is released when the last @consumer for the @uprobe
835 * Return errno if it cannot successully install probes
836 * else return 0 (success)
838 int uprobe_register(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*consumer
)
840 struct uprobe
*uprobe
;
843 if (!inode
|| !consumer
|| consumer
->next
)
846 if (offset
> i_size_read(inode
))
850 mutex_lock(uprobes_hash(inode
));
851 uprobe
= alloc_uprobe(inode
, offset
);
853 if (uprobe
&& !consumer_add(uprobe
, consumer
)) {
854 ret
= __uprobe_register(uprobe
);
856 uprobe
->consumers
= NULL
;
857 __uprobe_unregister(uprobe
);
859 uprobe
->flags
|= UPROBES_RUN_HANDLER
;
863 mutex_unlock(uprobes_hash(inode
));
870 * uprobe_unregister - unregister a already registered probe.
871 * @inode: the file in which the probe has to be removed.
872 * @offset: offset from the start of the file.
873 * @consumer: identify which probe if multiple probes are colocated.
875 void uprobe_unregister(struct inode
*inode
, loff_t offset
, struct uprobe_consumer
*consumer
)
877 struct uprobe
*uprobe
;
879 if (!inode
|| !consumer
)
882 uprobe
= find_uprobe(inode
, offset
);
886 mutex_lock(uprobes_hash(inode
));
888 if (consumer_del(uprobe
, consumer
)) {
889 if (!uprobe
->consumers
) {
890 __uprobe_unregister(uprobe
);
891 uprobe
->flags
&= ~UPROBES_RUN_HANDLER
;
895 mutex_unlock(uprobes_hash(inode
));
901 * Of all the nodes that correspond to the given inode, return the node
902 * with the least offset.
904 static struct rb_node
*find_least_offset_node(struct inode
*inode
)
906 struct uprobe u
= { .inode
= inode
, .offset
= 0};
907 struct rb_node
*n
= uprobes_tree
.rb_node
;
908 struct rb_node
*close_node
= NULL
;
909 struct uprobe
*uprobe
;
913 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
914 match
= match_uprobe(&u
, uprobe
);
916 if (uprobe
->inode
== inode
)
932 * For a given inode, build a list of probes that need to be inserted.
934 static void build_probe_list(struct inode
*inode
, struct list_head
*head
)
936 struct uprobe
*uprobe
;
940 spin_lock_irqsave(&uprobes_treelock
, flags
);
942 n
= find_least_offset_node(inode
);
944 for (; n
; n
= rb_next(n
)) {
945 uprobe
= rb_entry(n
, struct uprobe
, rb_node
);
946 if (uprobe
->inode
!= inode
)
949 list_add(&uprobe
->pending_list
, head
);
950 atomic_inc(&uprobe
->ref
);
953 spin_unlock_irqrestore(&uprobes_treelock
, flags
);
957 * Called from mmap_region.
958 * called with mm->mmap_sem acquired.
960 * Return -ve no if we fail to insert probes and we cannot
962 * Return 0 otherwise. i.e:
964 * - successful insertion of probes
965 * - (or) no possible probes to be inserted.
966 * - (or) insertion of probes failed but we can bail-out.
968 int uprobe_mmap(struct vm_area_struct
*vma
)
970 struct list_head tmp_list
;
971 struct uprobe
*uprobe
, *u
;
975 if (!atomic_read(&uprobe_events
) || !valid_vma(vma
, true))
978 inode
= vma
->vm_file
->f_mapping
->host
;
982 INIT_LIST_HEAD(&tmp_list
);
983 mutex_lock(uprobes_mmap_hash(inode
));
984 build_probe_list(inode
, &tmp_list
);
988 list_for_each_entry_safe(uprobe
, u
, &tmp_list
, pending_list
) {
991 list_del(&uprobe
->pending_list
);
993 vaddr
= vma_address(vma
, uprobe
->offset
);
994 if (vaddr
>= vma
->vm_start
&& vaddr
< vma
->vm_end
) {
995 ret
= install_breakpoint(vma
->vm_mm
, uprobe
, vma
, vaddr
);
996 /* Ignore double add: */
1004 mutex_unlock(uprobes_mmap_hash(inode
));
1009 static int __init
init_uprobes(void)
1013 for (i
= 0; i
< UPROBES_HASH_SZ
; i
++) {
1014 mutex_init(&uprobes_mutex
[i
]);
1015 mutex_init(&uprobes_mmap_mutex
[i
]);
1020 static void __exit
exit_uprobes(void)
1024 module_init(init_uprobes
);
1025 module_exit(exit_uprobes
);