4 * (C) Copyright 1994 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
16 #include <linux/highmem.h>
17 #include <linux/security.h>
18 #include <linux/mempolicy.h>
19 #include <linux/personality.h>
20 #include <linux/syscalls.h>
21 #include <linux/swap.h>
22 #include <linux/swapops.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/migrate.h>
25 #include <linux/perf_event.h>
26 #include <asm/uaccess.h>
27 #include <asm/pgtable.h>
28 #include <asm/cacheflush.h>
29 #include <asm/tlbflush.h>
32 static inline pgprot_t
pgprot_modify(pgprot_t oldprot
, pgprot_t newprot
)
38 static unsigned long change_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
39 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
40 int dirty_accountable
, int prot_numa
, bool *ret_all_same_nidpid
)
42 struct mm_struct
*mm
= vma
->vm_mm
;
45 unsigned long pages
= 0;
46 bool all_same_nidpid
= true;
50 pte
= pte_offset_map_lock(mm
, pmd
, addr
, &ptl
);
51 arch_enter_lazy_mmu_mode();
54 if (pte_present(oldpte
)) {
58 ptent
= ptep_modify_prot_start(mm
, addr
, pte
);
60 ptent
= pte_modify(ptent
, newprot
);
65 page
= vm_normal_page(vma
, addr
, oldpte
);
67 int nidpid
= page_nidpid_last(page
);
68 int this_nid
= nidpid_to_nid(nidpid
);
69 int this_pid
= nidpid_to_pid(nidpid
);
75 if (last_nid
!= this_nid
||
76 last_pid
!= this_pid
) {
77 all_same_nidpid
= false;
80 if (!pte_numa(oldpte
)) {
81 ptent
= pte_mknuma(ptent
);
88 * Avoid taking write faults for pages we know to be
91 if (dirty_accountable
&& pte_dirty(ptent
)) {
92 ptent
= pte_mkwrite(ptent
);
98 ptep_modify_prot_commit(mm
, addr
, pte
, ptent
);
99 } else if (IS_ENABLED(CONFIG_MIGRATION
) && !pte_file(oldpte
)) {
100 swp_entry_t entry
= pte_to_swp_entry(oldpte
);
102 if (is_write_migration_entry(entry
)) {
104 * A protection check is difficult so
105 * just be safe and disable write
107 make_migration_entry_read(&entry
);
108 set_pte_at(mm
, addr
, pte
,
109 swp_entry_to_pte(entry
));
114 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
115 arch_leave_lazy_mmu_mode();
116 pte_unmap_unlock(pte
- 1, ptl
);
118 *ret_all_same_nidpid
= all_same_nidpid
;
122 #ifdef CONFIG_NUMA_BALANCING
123 static inline void change_pmd_protnuma(struct mm_struct
*mm
, unsigned long addr
,
126 spin_lock(&mm
->page_table_lock
);
127 set_pmd_at(mm
, addr
& PMD_MASK
, pmd
, pmd_mknuma(*pmd
));
128 spin_unlock(&mm
->page_table_lock
);
131 static inline void change_pmd_protnuma(struct mm_struct
*mm
, unsigned long addr
,
136 #endif /* CONFIG_NUMA_BALANCING */
138 static inline unsigned long change_pmd_range(struct vm_area_struct
*vma
,
139 pud_t
*pud
, unsigned long addr
, unsigned long end
,
140 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
144 unsigned long pages
= 0;
145 bool all_same_nidpid
;
147 pmd
= pmd_offset(pud
, addr
);
149 next
= pmd_addr_end(addr
, end
);
150 if (pmd_trans_huge(*pmd
)) {
151 if (next
- addr
!= HPAGE_PMD_SIZE
)
152 split_huge_page_pmd(vma
, addr
, pmd
);
154 int nr_ptes
= change_huge_pmd(vma
, pmd
, addr
,
158 if (nr_ptes
== HPAGE_PMD_NR
)
166 if (pmd_none_or_clear_bad(pmd
))
168 pages
+= change_pte_range(vma
, pmd
, addr
, next
, newprot
,
169 dirty_accountable
, prot_numa
, &all_same_nidpid
);
172 * If we are changing protections for NUMA hinting faults then
173 * set pmd_numa if the examined pages were all on the same
174 * node. This allows a regular PMD to be handled as one fault
175 * and effectively batches the taking of the PTL
177 if (prot_numa
&& all_same_nidpid
)
178 change_pmd_protnuma(vma
->vm_mm
, addr
, pmd
);
179 } while (pmd
++, addr
= next
, addr
!= end
);
184 static inline unsigned long change_pud_range(struct vm_area_struct
*vma
,
185 pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
186 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
190 unsigned long pages
= 0;
192 pud
= pud_offset(pgd
, addr
);
194 next
= pud_addr_end(addr
, end
);
195 if (pud_none_or_clear_bad(pud
))
197 pages
+= change_pmd_range(vma
, pud
, addr
, next
, newprot
,
198 dirty_accountable
, prot_numa
);
199 } while (pud
++, addr
= next
, addr
!= end
);
204 static unsigned long change_protection_range(struct vm_area_struct
*vma
,
205 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
206 int dirty_accountable
, int prot_numa
)
208 struct mm_struct
*mm
= vma
->vm_mm
;
211 unsigned long start
= addr
;
212 unsigned long pages
= 0;
215 pgd
= pgd_offset(mm
, addr
);
216 flush_cache_range(vma
, addr
, end
);
218 next
= pgd_addr_end(addr
, end
);
219 if (pgd_none_or_clear_bad(pgd
))
221 pages
+= change_pud_range(vma
, pgd
, addr
, next
, newprot
,
222 dirty_accountable
, prot_numa
);
223 } while (pgd
++, addr
= next
, addr
!= end
);
225 /* Only flush the TLB if we actually modified any entries: */
227 flush_tlb_range(vma
, start
, end
);
232 unsigned long change_protection(struct vm_area_struct
*vma
, unsigned long start
,
233 unsigned long end
, pgprot_t newprot
,
234 int dirty_accountable
, int prot_numa
)
236 struct mm_struct
*mm
= vma
->vm_mm
;
239 mmu_notifier_invalidate_range_start(mm
, start
, end
);
240 if (is_vm_hugetlb_page(vma
))
241 pages
= hugetlb_change_protection(vma
, start
, end
, newprot
);
243 pages
= change_protection_range(vma
, start
, end
, newprot
, dirty_accountable
, prot_numa
);
244 mmu_notifier_invalidate_range_end(mm
, start
, end
);
250 mprotect_fixup(struct vm_area_struct
*vma
, struct vm_area_struct
**pprev
,
251 unsigned long start
, unsigned long end
, unsigned long newflags
)
253 struct mm_struct
*mm
= vma
->vm_mm
;
254 unsigned long oldflags
= vma
->vm_flags
;
255 long nrpages
= (end
- start
) >> PAGE_SHIFT
;
256 unsigned long charged
= 0;
259 int dirty_accountable
= 0;
261 if (newflags
== oldflags
) {
267 * If we make a private mapping writable we increase our commit;
268 * but (without finer accounting) cannot reduce our commit if we
269 * make it unwritable again. hugetlb mapping were accounted for
270 * even if read-only so there is no need to account for them here
272 if (newflags
& VM_WRITE
) {
273 if (!(oldflags
& (VM_ACCOUNT
|VM_WRITE
|VM_HUGETLB
|
274 VM_SHARED
|VM_NORESERVE
))) {
276 if (security_vm_enough_memory_mm(mm
, charged
))
278 newflags
|= VM_ACCOUNT
;
283 * First try to merge with previous and/or next vma.
285 pgoff
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
286 *pprev
= vma_merge(mm
, *pprev
, start
, end
, newflags
,
287 vma
->anon_vma
, vma
->vm_file
, pgoff
, vma_policy(vma
));
295 if (start
!= vma
->vm_start
) {
296 error
= split_vma(mm
, vma
, start
, 1);
301 if (end
!= vma
->vm_end
) {
302 error
= split_vma(mm
, vma
, end
, 0);
309 * vm_flags and vm_page_prot are protected by the mmap_sem
310 * held in write mode.
312 vma
->vm_flags
= newflags
;
313 vma
->vm_page_prot
= pgprot_modify(vma
->vm_page_prot
,
314 vm_get_page_prot(newflags
));
316 if (vma_wants_writenotify(vma
)) {
317 vma
->vm_page_prot
= vm_get_page_prot(newflags
& ~VM_SHARED
);
318 dirty_accountable
= 1;
321 change_protection(vma
, start
, end
, vma
->vm_page_prot
,
322 dirty_accountable
, 0);
324 vm_stat_account(mm
, oldflags
, vma
->vm_file
, -nrpages
);
325 vm_stat_account(mm
, newflags
, vma
->vm_file
, nrpages
);
326 perf_event_mmap(vma
);
330 vm_unacct_memory(charged
);
334 SYSCALL_DEFINE3(mprotect
, unsigned long, start
, size_t, len
,
337 unsigned long vm_flags
, nstart
, end
, tmp
, reqprot
;
338 struct vm_area_struct
*vma
, *prev
;
340 const int grows
= prot
& (PROT_GROWSDOWN
|PROT_GROWSUP
);
341 prot
&= ~(PROT_GROWSDOWN
|PROT_GROWSUP
);
342 if (grows
== (PROT_GROWSDOWN
|PROT_GROWSUP
)) /* can't be both */
345 if (start
& ~PAGE_MASK
)
349 len
= PAGE_ALIGN(len
);
353 if (!arch_validate_prot(prot
))
358 * Does the application expect PROT_READ to imply PROT_EXEC:
360 if ((prot
& PROT_READ
) && (current
->personality
& READ_IMPLIES_EXEC
))
363 vm_flags
= calc_vm_prot_bits(prot
);
365 down_write(¤t
->mm
->mmap_sem
);
367 vma
= find_vma(current
->mm
, start
);
372 if (unlikely(grows
& PROT_GROWSDOWN
)) {
373 if (vma
->vm_start
>= end
)
375 start
= vma
->vm_start
;
377 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
380 if (vma
->vm_start
> start
)
382 if (unlikely(grows
& PROT_GROWSUP
)) {
385 if (!(vma
->vm_flags
& VM_GROWSUP
))
389 if (start
> vma
->vm_start
)
392 for (nstart
= start
; ; ) {
393 unsigned long newflags
;
395 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
398 newflags
|= (vma
->vm_flags
& ~(VM_READ
| VM_WRITE
| VM_EXEC
));
400 /* newflags >> 4 shift VM_MAY% in place of VM_% */
401 if ((newflags
& ~(newflags
>> 4)) & (VM_READ
| VM_WRITE
| VM_EXEC
)) {
406 error
= security_file_mprotect(vma
, reqprot
, prot
);
413 error
= mprotect_fixup(vma
, &prev
, nstart
, tmp
, newflags
);
418 if (nstart
< prev
->vm_end
)
419 nstart
= prev
->vm_end
;
424 if (!vma
|| vma
->vm_start
!= nstart
) {
430 up_write(¤t
->mm
->mmap_sem
);
This page took 0.045825 seconds and 6 git commands to generate.