Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _X86_64_PGTABLE_H |
2 | #define _X86_64_PGTABLE_H | |
3 | ||
6df95fd7 | 4 | #include <linux/const.h> |
9d291e78 VG |
5 | #ifndef __ASSEMBLY__ |
6 | ||
1da177e4 LT |
7 | /* |
8 | * This file contains the functions and defines necessary to modify and use | |
9 | * the x86-64 page table tree. | |
10 | */ | |
11 | #include <asm/processor.h> | |
1977f032 | 12 | #include <linux/bitops.h> |
1da177e4 LT |
13 | #include <linux/threads.h> |
14 | #include <asm/pda.h> | |
15 | ||
16 | extern pud_t level3_kernel_pgt[512]; | |
1da177e4 LT |
17 | extern pud_t level3_ident_pgt[512]; |
18 | extern pmd_t level2_kernel_pgt[512]; | |
19 | extern pgd_t init_level4_pgt[]; | |
20 | extern unsigned long __supported_pte_mask; | |
21 | ||
e3ebadd9 | 22 | #define swapper_pg_dir init_level4_pgt |
1da177e4 | 23 | |
1da177e4 LT |
24 | extern void paging_init(void); |
25 | extern void clear_kernel_mapping(unsigned long addr, unsigned long size); | |
26 | ||
1da177e4 LT |
27 | /* |
28 | * ZERO_PAGE is a global shared page that is always zero: used | |
29 | * for zero-mapped memory areas etc.. | |
30 | */ | |
31 | extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)]; | |
e3ebadd9 | 32 | #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) |
1da177e4 | 33 | |
9d291e78 VG |
34 | #endif /* !__ASSEMBLY__ */ |
35 | ||
1da177e4 LT |
36 | /* |
37 | * PGDIR_SHIFT determines what a top-level page table entry can map | |
38 | */ | |
39 | #define PGDIR_SHIFT 39 | |
40 | #define PTRS_PER_PGD 512 | |
41 | ||
42 | /* | |
43 | * 3rd level page | |
44 | */ | |
45 | #define PUD_SHIFT 30 | |
46 | #define PTRS_PER_PUD 512 | |
47 | ||
48 | /* | |
49 | * PMD_SHIFT determines the size of the area a middle-level | |
50 | * page table can map | |
51 | */ | |
52 | #define PMD_SHIFT 21 | |
53 | #define PTRS_PER_PMD 512 | |
54 | ||
55 | /* | |
56 | * entries per page directory level | |
57 | */ | |
58 | #define PTRS_PER_PTE 512 | |
59 | ||
9d291e78 VG |
60 | #ifndef __ASSEMBLY__ |
61 | ||
1da177e4 LT |
62 | #define pte_ERROR(e) \ |
63 | printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e)) | |
64 | #define pmd_ERROR(e) \ | |
65 | printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) | |
66 | #define pud_ERROR(e) \ | |
67 | printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), pud_val(e)) | |
68 | #define pgd_ERROR(e) \ | |
69 | printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) | |
70 | ||
71 | #define pgd_none(x) (!pgd_val(x)) | |
72 | #define pud_none(x) (!pud_val(x)) | |
73 | ||
74 | static inline void set_pte(pte_t *dst, pte_t val) | |
75 | { | |
76 | pte_val(*dst) = pte_val(val); | |
77 | } | |
78 | #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) | |
79 | ||
80 | static inline void set_pmd(pmd_t *dst, pmd_t val) | |
81 | { | |
7a2389b4 | 82 | *dst = val; |
1da177e4 LT |
83 | } |
84 | ||
85 | static inline void set_pud(pud_t *dst, pud_t val) | |
86 | { | |
7a2389b4 | 87 | *dst = val; |
1da177e4 LT |
88 | } |
89 | ||
9c0aa0f9 | 90 | static inline void pud_clear (pud_t *pud) |
1da177e4 LT |
91 | { |
92 | set_pud(pud, __pud(0)); | |
93 | } | |
94 | ||
95 | static inline void set_pgd(pgd_t *dst, pgd_t val) | |
96 | { | |
7a2389b4 | 97 | *dst = val; |
1da177e4 LT |
98 | } |
99 | ||
9c0aa0f9 | 100 | static inline void pgd_clear (pgd_t * pgd) |
1da177e4 LT |
101 | { |
102 | set_pgd(pgd, __pgd(0)); | |
103 | } | |
104 | ||
1da177e4 | 105 | #define ptep_get_and_clear(mm,addr,xp) __pte(xchg(&(xp)->pte, 0)) |
61e06037 | 106 | |
8c65b4a6 TS |
107 | struct mm_struct; |
108 | ||
61e06037 ZA |
109 | static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) |
110 | { | |
111 | pte_t pte; | |
112 | if (full) { | |
113 | pte = *ptep; | |
114 | *ptep = __pte(0); | |
115 | } else { | |
116 | pte = ptep_get_and_clear(mm, addr, ptep); | |
117 | } | |
118 | return pte; | |
119 | } | |
120 | ||
1da177e4 LT |
121 | #define pte_same(a, b) ((a).pte == (b).pte) |
122 | ||
c728252c AV |
123 | #define pte_pgprot(a) (__pgprot((a).pte & ~PHYSICAL_PAGE_MASK)) |
124 | ||
9d291e78 VG |
125 | #endif /* !__ASSEMBLY__ */ |
126 | ||
127 | #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) | |
1da177e4 | 128 | #define PMD_MASK (~(PMD_SIZE-1)) |
9d291e78 | 129 | #define PUD_SIZE (_AC(1,UL) << PUD_SHIFT) |
1da177e4 | 130 | #define PUD_MASK (~(PUD_SIZE-1)) |
9d291e78 | 131 | #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) |
1da177e4 LT |
132 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) |
133 | ||
1da177e4 | 134 | |
63f6564d RD |
135 | #define MAXMEM _AC(0x3fffffffffff, UL) |
136 | #define VMALLOC_START _AC(0xffffc20000000000, UL) | |
137 | #define VMALLOC_END _AC(0xffffe1ffffffffff, UL) | |
0889eba5 | 138 | #define VMEMMAP_START _AC(0xffffe20000000000, UL) |
63f6564d RD |
139 | #define MODULES_VADDR _AC(0xffffffff88000000, UL) |
140 | #define MODULES_END _AC(0xfffffffffff00000, UL) | |
1da177e4 LT |
141 | #define MODULES_LEN (MODULES_END - MODULES_VADDR) |
142 | ||
9d291e78 VG |
143 | #ifndef __ASSEMBLY__ |
144 | ||
eab724e5 JB |
145 | static inline unsigned long pgd_bad(pgd_t pgd) |
146 | { | |
147 | return pgd_val(pgd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); | |
148 | } | |
1da177e4 LT |
149 | |
150 | static inline unsigned long pud_bad(pud_t pud) | |
151 | { | |
eab724e5 JB |
152 | return pud_val(pud) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); |
153 | } | |
154 | ||
155 | static inline unsigned long pmd_bad(pmd_t pmd) | |
156 | { | |
157 | return pmd_val(pmd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); | |
1da177e4 LT |
158 | } |
159 | ||
160 | #define pte_none(x) (!pte_val(x)) | |
161 | #define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) | |
162 | #define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) | |
163 | ||
1c6f7030 | 164 | #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this right? */ |
1da177e4 | 165 | #define pte_page(x) pfn_to_page(pte_pfn(x)) |
6b75aeed | 166 | #define pte_pfn(x) ((pte_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) |
1da177e4 LT |
167 | |
168 | static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) | |
169 | { | |
170 | pte_t pte; | |
171 | pte_val(pte) = (page_nr << PAGE_SHIFT); | |
172 | pte_val(pte) |= pgprot_val(pgprot); | |
173 | pte_val(pte) &= __supported_pte_mask; | |
174 | return pte; | |
175 | } | |
1da177e4 LT |
176 | struct vm_area_struct; |
177 | ||
1da177e4 LT |
178 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) |
179 | { | |
180 | if (!pte_young(*ptep)) | |
181 | return 0; | |
3d1712c9 | 182 | return test_and_clear_bit(_PAGE_BIT_ACCESSED, &ptep->pte); |
1da177e4 LT |
183 | } |
184 | ||
185 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
186 | { | |
3d1712c9 | 187 | clear_bit(_PAGE_BIT_RW, &ptep->pte); |
1da177e4 LT |
188 | } |
189 | ||
190 | /* | |
191 | * Macro to mark a page protection value as "uncacheable". | |
192 | */ | |
193 | #define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) | |
194 | ||
1da177e4 LT |
195 | |
196 | /* | |
197 | * Conversion functions: convert a page and protection to a page entry, | |
198 | * and a page entry and page directory to the page they refer to. | |
199 | */ | |
200 | ||
1da177e4 LT |
201 | /* |
202 | * Level 4 access. | |
203 | */ | |
46a82b2d DM |
204 | #define pgd_page_vaddr(pgd) ((unsigned long) __va((unsigned long)pgd_val(pgd) & PTE_MASK)) |
205 | #define pgd_page(pgd) (pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)) | |
1da177e4 LT |
206 | #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) |
207 | #define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) | |
208 | #define pgd_offset_k(address) (init_level4_pgt + pgd_index(address)) | |
209 | #define pgd_present(pgd) (pgd_val(pgd) & _PAGE_PRESENT) | |
210 | #define mk_kernel_pgd(address) ((pgd_t){ (address) | _KERNPG_TABLE }) | |
211 | ||
212 | /* PUD - Level3 access */ | |
213 | /* to find an entry in a page-table-directory. */ | |
46a82b2d DM |
214 | #define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PHYSICAL_PAGE_MASK)) |
215 | #define pud_page(pud) (pfn_to_page(pud_val(pud) >> PAGE_SHIFT)) | |
1da177e4 | 216 | #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) |
46a82b2d | 217 | #define pud_offset(pgd, address) ((pud_t *) pgd_page_vaddr(*(pgd)) + pud_index(address)) |
1da177e4 LT |
218 | #define pud_present(pud) (pud_val(pud) & _PAGE_PRESENT) |
219 | ||
1da177e4 | 220 | /* PMD - Level 2 access */ |
46a82b2d | 221 | #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK)) |
1da177e4 LT |
222 | #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) |
223 | ||
224 | #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) | |
46a82b2d | 225 | #define pmd_offset(dir, address) ((pmd_t *) pud_page_vaddr(*(dir)) + \ |
1da177e4 LT |
226 | pmd_index(address)) |
227 | #define pmd_none(x) (!pmd_val(x)) | |
228 | #define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) | |
229 | #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) | |
1da177e4 | 230 | #define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot))) |
6b75aeed | 231 | #define pmd_pfn(x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) |
1da177e4 LT |
232 | |
233 | #define pte_to_pgoff(pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) | |
234 | #define pgoff_to_pte(off) ((pte_t) { ((off) << PAGE_SHIFT) | _PAGE_FILE }) | |
235 | #define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT | |
236 | ||
237 | /* PTE - Level 1 access. */ | |
238 | ||
239 | /* page, protection -> pte */ | |
240 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | |
1da177e4 | 241 | |
1da177e4 | 242 | /* Change flags of a PTE */ |
9c0aa0f9 | 243 | static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) |
1da177e4 LT |
244 | { |
245 | pte_val(pte) &= _PAGE_CHG_MASK; | |
246 | pte_val(pte) |= pgprot_val(newprot); | |
247 | pte_val(pte) &= __supported_pte_mask; | |
248 | return pte; | |
249 | } | |
250 | ||
251 | #define pte_index(address) \ | |
1294b118 | 252 | (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) |
46a82b2d | 253 | #define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_vaddr(*(dir)) + \ |
1da177e4 LT |
254 | pte_index(address)) |
255 | ||
256 | /* x86-64 always has all page tables mapped. */ | |
257 | #define pte_offset_map(dir,address) pte_offset_kernel(dir,address) | |
258 | #define pte_offset_map_nested(dir,address) pte_offset_kernel(dir,address) | |
259 | #define pte_unmap(pte) /* NOP */ | |
260 | #define pte_unmap_nested(pte) /* NOP */ | |
261 | ||
262 | #define update_mmu_cache(vma,address,pte) do { } while (0) | |
263 | ||
264 | /* We only update the dirty/accessed state if we set | |
265 | * the dirty bit by hand in the kernel, since the hardware | |
266 | * will do the accessed bit for us, and we don't want to | |
267 | * race with other CPU's that might be updating the dirty | |
268 | * bit at the same time. */ | |
269 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS | |
270 | #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ | |
8dab5241 BH |
271 | ({ \ |
272 | int __changed = !pte_same(*(__ptep), __entry); \ | |
273 | if (__changed && __dirty) { \ | |
274 | set_pte(__ptep, __entry); \ | |
275 | flush_tlb_page(__vma, __address); \ | |
276 | } \ | |
277 | __changed; \ | |
278 | }) | |
1da177e4 LT |
279 | |
280 | /* Encode and de-code a swap entry */ | |
281 | #define __swp_type(x) (((x).val >> 1) & 0x3f) | |
282 | #define __swp_offset(x) ((x).val >> 8) | |
283 | #define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 8) }) | |
284 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) | |
285 | #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) | |
286 | ||
8c914cb7 | 287 | extern spinlock_t pgd_lock; |
2bff7383 | 288 | extern struct list_head pgd_list; |
8c914cb7 | 289 | |
1da177e4 LT |
290 | extern int kern_addr_valid(unsigned long addr); |
291 | ||
19d36ccd AK |
292 | pte_t *lookup_address(unsigned long addr); |
293 | ||
1da177e4 LT |
294 | #define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ |
295 | remap_pfn_range(vma, vaddr, pfn, size, prot) | |
296 | ||
1da177e4 | 297 | #define HAVE_ARCH_UNMAPPED_AREA |
cc503c1b | 298 | #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN |
1da177e4 LT |
299 | |
300 | #define pgtable_cache_init() do { } while (0) | |
da8f153e | 301 | #define check_pgt_cache() do { } while (0) |
1da177e4 LT |
302 | |
303 | #define PAGE_AGP PAGE_KERNEL_NOCACHE | |
304 | #define HAVE_PAGE_AGP 1 | |
305 | ||
306 | /* fs/proc/kcore.c */ | |
307 | #define kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK) | |
308 | #define kc_offset_to_vaddr(o) \ | |
309 | (((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o)) | |
310 | ||
311 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG | |
1da177e4 | 312 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR |
61e06037 | 313 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL |
1da177e4 LT |
314 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
315 | #define __HAVE_ARCH_PTE_SAME | |
316 | #include <asm-generic/pgtable.h> | |
9d291e78 | 317 | #endif /* !__ASSEMBLY__ */ |
1da177e4 LT |
318 | |
319 | #endif /* _X86_64_PGTABLE_H */ |