4 * (C) Copyright 1994 Linus Torvalds
5 * (C) Copyright 2002 Christoph Hellwig
7 * Address space accounting code <alan@lxorguk.ukuu.org.uk>
8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved
12 #include <linux/hugetlb.h>
13 #include <linux/shm.h>
14 #include <linux/mman.h>
16 #include <linux/highmem.h>
17 #include <linux/security.h>
18 #include <linux/mempolicy.h>
19 #include <linux/personality.h>
20 #include <linux/syscalls.h>
21 #include <linux/swap.h>
22 #include <linux/swapops.h>
23 #include <linux/mmu_notifier.h>
24 #include <linux/migrate.h>
25 #include <linux/perf_event.h>
26 #include <linux/ksm.h>
27 #include <asm/uaccess.h>
28 #include <asm/pgtable.h>
29 #include <asm/cacheflush.h>
30 #include <asm/tlbflush.h>
35 * For a prot_numa update we only hold mmap_sem for read so there is a
36 * potential race with faulting where a pmd was temporarily none. This
37 * function checks for a transhuge pmd under the appropriate lock. It
38 * returns a pte if it was successfully locked or NULL if it raced with
39 * a transhuge insertion.
41 static pte_t
*lock_pte_protection(struct vm_area_struct
*vma
, pmd_t
*pmd
,
42 unsigned long addr
, int prot_numa
, spinlock_t
**ptl
)
47 /* !prot_numa is protected by mmap_sem held for write */
49 return pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, ptl
);
51 pmdl
= pmd_lock(vma
->vm_mm
, pmd
);
52 if (unlikely(pmd_trans_huge(*pmd
) || pmd_none(*pmd
))) {
57 pte
= pte_offset_map_lock(vma
->vm_mm
, pmd
, addr
, ptl
);
62 static unsigned long change_pte_range(struct vm_area_struct
*vma
, pmd_t
*pmd
,
63 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
64 int dirty_accountable
, int prot_numa
)
66 struct mm_struct
*mm
= vma
->vm_mm
;
69 unsigned long pages
= 0;
71 pte
= lock_pte_protection(vma
, pmd
, addr
, prot_numa
, &ptl
);
75 arch_enter_lazy_mmu_mode();
78 if (pte_present(oldpte
)) {
80 bool preserve_write
= prot_numa
&& pte_write(oldpte
);
83 * Avoid trapping faults against the zero or KSM
84 * pages. See similar comment in change_huge_pmd.
89 page
= vm_normal_page(vma
, addr
, oldpte
);
90 if (!page
|| PageKsm(page
))
93 /* Avoid TLB flush if possible */
94 if (pte_protnone(oldpte
))
98 ptent
= ptep_modify_prot_start(mm
, addr
, pte
);
99 ptent
= pte_modify(ptent
, newprot
);
101 ptent
= pte_mkwrite(ptent
);
103 /* Avoid taking write faults for known dirty pages */
104 if (dirty_accountable
&& pte_dirty(ptent
) &&
105 (pte_soft_dirty(ptent
) ||
106 !(vma
->vm_flags
& VM_SOFTDIRTY
))) {
107 ptent
= pte_mkwrite(ptent
);
109 ptep_modify_prot_commit(mm
, addr
, pte
, ptent
);
111 } else if (IS_ENABLED(CONFIG_MIGRATION
)) {
112 swp_entry_t entry
= pte_to_swp_entry(oldpte
);
114 if (is_write_migration_entry(entry
)) {
117 * A protection check is difficult so
118 * just be safe and disable write
120 make_migration_entry_read(&entry
);
121 newpte
= swp_entry_to_pte(entry
);
122 if (pte_swp_soft_dirty(oldpte
))
123 newpte
= pte_swp_mksoft_dirty(newpte
);
124 set_pte_at(mm
, addr
, pte
, newpte
);
129 } while (pte
++, addr
+= PAGE_SIZE
, addr
!= end
);
130 arch_leave_lazy_mmu_mode();
131 pte_unmap_unlock(pte
- 1, ptl
);
136 static inline unsigned long change_pmd_range(struct vm_area_struct
*vma
,
137 pud_t
*pud
, unsigned long addr
, unsigned long end
,
138 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
141 struct mm_struct
*mm
= vma
->vm_mm
;
143 unsigned long pages
= 0;
144 unsigned long nr_huge_updates
= 0;
145 unsigned long mni_start
= 0;
147 pmd
= pmd_offset(pud
, addr
);
149 unsigned long this_pages
;
151 next
= pmd_addr_end(addr
, end
);
152 if (!pmd_trans_huge(*pmd
) && !pmd_devmap(*pmd
)
153 && pmd_none_or_clear_bad(pmd
))
156 /* invoke the mmu notifier if the pmd is populated */
159 mmu_notifier_invalidate_range_start(mm
, mni_start
, end
);
162 if (pmd_trans_huge(*pmd
) || pmd_devmap(*pmd
)) {
163 if (next
- addr
!= HPAGE_PMD_SIZE
) {
164 split_huge_pmd(vma
, pmd
, addr
);
168 int nr_ptes
= change_huge_pmd(vma
, pmd
, addr
,
172 if (nr_ptes
== HPAGE_PMD_NR
) {
173 pages
+= HPAGE_PMD_NR
;
177 /* huge pmd was handled */
181 /* fall through, the trans huge pmd just split */
183 this_pages
= change_pte_range(vma
, pmd
, addr
, next
, newprot
,
184 dirty_accountable
, prot_numa
);
186 } while (pmd
++, addr
= next
, addr
!= end
);
189 mmu_notifier_invalidate_range_end(mm
, mni_start
, end
);
192 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES
, nr_huge_updates
);
196 static inline unsigned long change_pud_range(struct vm_area_struct
*vma
,
197 pgd_t
*pgd
, unsigned long addr
, unsigned long end
,
198 pgprot_t newprot
, int dirty_accountable
, int prot_numa
)
202 unsigned long pages
= 0;
204 pud
= pud_offset(pgd
, addr
);
206 next
= pud_addr_end(addr
, end
);
207 if (pud_none_or_clear_bad(pud
))
209 pages
+= change_pmd_range(vma
, pud
, addr
, next
, newprot
,
210 dirty_accountable
, prot_numa
);
211 } while (pud
++, addr
= next
, addr
!= end
);
216 static unsigned long change_protection_range(struct vm_area_struct
*vma
,
217 unsigned long addr
, unsigned long end
, pgprot_t newprot
,
218 int dirty_accountable
, int prot_numa
)
220 struct mm_struct
*mm
= vma
->vm_mm
;
223 unsigned long start
= addr
;
224 unsigned long pages
= 0;
227 pgd
= pgd_offset(mm
, addr
);
228 flush_cache_range(vma
, addr
, end
);
229 set_tlb_flush_pending(mm
);
231 next
= pgd_addr_end(addr
, end
);
232 if (pgd_none_or_clear_bad(pgd
))
234 pages
+= change_pud_range(vma
, pgd
, addr
, next
, newprot
,
235 dirty_accountable
, prot_numa
);
236 } while (pgd
++, addr
= next
, addr
!= end
);
238 /* Only flush the TLB if we actually modified any entries: */
240 flush_tlb_range(vma
, start
, end
);
241 clear_tlb_flush_pending(mm
);
246 unsigned long change_protection(struct vm_area_struct
*vma
, unsigned long start
,
247 unsigned long end
, pgprot_t newprot
,
248 int dirty_accountable
, int prot_numa
)
252 if (is_vm_hugetlb_page(vma
))
253 pages
= hugetlb_change_protection(vma
, start
, end
, newprot
);
255 pages
= change_protection_range(vma
, start
, end
, newprot
, dirty_accountable
, prot_numa
);
261 mprotect_fixup(struct vm_area_struct
*vma
, struct vm_area_struct
**pprev
,
262 unsigned long start
, unsigned long end
, unsigned long newflags
)
264 struct mm_struct
*mm
= vma
->vm_mm
;
265 unsigned long oldflags
= vma
->vm_flags
;
266 long nrpages
= (end
- start
) >> PAGE_SHIFT
;
267 unsigned long charged
= 0;
270 int dirty_accountable
= 0;
272 if (newflags
== oldflags
) {
278 * If we make a private mapping writable we increase our commit;
279 * but (without finer accounting) cannot reduce our commit if we
280 * make it unwritable again. hugetlb mapping were accounted for
281 * even if read-only so there is no need to account for them here
283 if (newflags
& VM_WRITE
) {
284 /* Check space limits when area turns into data. */
285 if (!may_expand_vm(mm
, newflags
, nrpages
) &&
286 may_expand_vm(mm
, oldflags
, nrpages
))
288 if (!(oldflags
& (VM_ACCOUNT
|VM_WRITE
|VM_HUGETLB
|
289 VM_SHARED
|VM_NORESERVE
))) {
291 if (security_vm_enough_memory_mm(mm
, charged
))
293 newflags
|= VM_ACCOUNT
;
298 * First try to merge with previous and/or next vma.
300 pgoff
= vma
->vm_pgoff
+ ((start
- vma
->vm_start
) >> PAGE_SHIFT
);
301 *pprev
= vma_merge(mm
, *pprev
, start
, end
, newflags
,
302 vma
->anon_vma
, vma
->vm_file
, pgoff
, vma_policy(vma
),
303 vma
->vm_userfaultfd_ctx
);
311 if (start
!= vma
->vm_start
) {
312 error
= split_vma(mm
, vma
, start
, 1);
317 if (end
!= vma
->vm_end
) {
318 error
= split_vma(mm
, vma
, end
, 0);
325 * vm_flags and vm_page_prot are protected by the mmap_sem
326 * held in write mode.
328 vma
->vm_flags
= newflags
;
329 dirty_accountable
= vma_wants_writenotify(vma
);
330 vma_set_page_prot(vma
);
332 change_protection(vma
, start
, end
, vma
->vm_page_prot
,
333 dirty_accountable
, 0);
336 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
339 if ((oldflags
& (VM_WRITE
| VM_SHARED
| VM_LOCKED
)) == VM_LOCKED
&&
340 (newflags
& VM_WRITE
)) {
341 populate_vma_page_range(vma
, start
, end
, NULL
);
344 vm_stat_account(mm
, oldflags
, -nrpages
);
345 vm_stat_account(mm
, newflags
, nrpages
);
346 perf_event_mmap(vma
);
350 vm_unacct_memory(charged
);
354 SYSCALL_DEFINE3(mprotect
, unsigned long, start
, size_t, len
,
357 unsigned long vm_flags
, nstart
, end
, tmp
, reqprot
;
358 struct vm_area_struct
*vma
, *prev
;
360 const int grows
= prot
& (PROT_GROWSDOWN
|PROT_GROWSUP
);
361 prot
&= ~(PROT_GROWSDOWN
|PROT_GROWSUP
);
362 if (grows
== (PROT_GROWSDOWN
|PROT_GROWSUP
)) /* can't be both */
365 if (start
& ~PAGE_MASK
)
369 len
= PAGE_ALIGN(len
);
373 if (!arch_validate_prot(prot
))
378 * Does the application expect PROT_READ to imply PROT_EXEC:
380 if ((prot
& PROT_READ
) && (current
->personality
& READ_IMPLIES_EXEC
))
383 vm_flags
= calc_vm_prot_bits(prot
);
385 down_write(¤t
->mm
->mmap_sem
);
387 vma
= find_vma(current
->mm
, start
);
392 if (unlikely(grows
& PROT_GROWSDOWN
)) {
393 if (vma
->vm_start
>= end
)
395 start
= vma
->vm_start
;
397 if (!(vma
->vm_flags
& VM_GROWSDOWN
))
400 if (vma
->vm_start
> start
)
402 if (unlikely(grows
& PROT_GROWSUP
)) {
405 if (!(vma
->vm_flags
& VM_GROWSUP
))
409 if (start
> vma
->vm_start
)
412 for (nstart
= start
; ; ) {
413 unsigned long newflags
;
415 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
418 newflags
|= (vma
->vm_flags
& ~(VM_READ
| VM_WRITE
| VM_EXEC
));
420 /* newflags >> 4 shift VM_MAY% in place of VM_% */
421 if ((newflags
& ~(newflags
>> 4)) & (VM_READ
| VM_WRITE
| VM_EXEC
)) {
426 error
= security_file_mprotect(vma
, reqprot
, prot
);
433 error
= mprotect_fixup(vma
, &prev
, nstart
, tmp
, newflags
);
438 if (nstart
< prev
->vm_end
)
439 nstart
= prev
->vm_end
;
444 if (!vma
|| vma
->vm_start
!= nstart
) {
450 up_write(¤t
->mm
->mmap_sem
);
This page took 0.062819 seconds and 6 git commands to generate.