Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _X86_64_PGTABLE_H |
2 | #define _X86_64_PGTABLE_H | |
3 | ||
6df95fd7 | 4 | #include <linux/const.h> |
9d291e78 VG |
5 | #ifndef __ASSEMBLY__ |
6 | ||
1da177e4 LT |
7 | /* |
8 | * This file contains the functions and defines necessary to modify and use | |
9 | * the x86-64 page table tree. | |
10 | */ | |
11 | #include <asm/processor.h> | |
1977f032 | 12 | #include <linux/bitops.h> |
1da177e4 LT |
13 | #include <linux/threads.h> |
14 | #include <asm/pda.h> | |
15 | ||
16 | extern pud_t level3_kernel_pgt[512]; | |
1da177e4 LT |
17 | extern pud_t level3_ident_pgt[512]; |
18 | extern pmd_t level2_kernel_pgt[512]; | |
19 | extern pgd_t init_level4_pgt[]; | |
1da177e4 | 20 | |
e3ebadd9 | 21 | #define swapper_pg_dir init_level4_pgt |
1da177e4 | 22 | |
1da177e4 LT |
23 | extern void paging_init(void); |
24 | extern void clear_kernel_mapping(unsigned long addr, unsigned long size); | |
25 | ||
1da177e4 LT |
26 | /* |
27 | * ZERO_PAGE is a global shared page that is always zero: used | |
28 | * for zero-mapped memory areas etc.. | |
29 | */ | |
30 | extern unsigned long empty_zero_page[PAGE_SIZE/sizeof(unsigned long)]; | |
e3ebadd9 | 31 | #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) |
1da177e4 | 32 | |
9d291e78 VG |
33 | #endif /* !__ASSEMBLY__ */ |
34 | ||
1da177e4 LT |
35 | /* |
36 | * PGDIR_SHIFT determines what a top-level page table entry can map | |
37 | */ | |
38 | #define PGDIR_SHIFT 39 | |
39 | #define PTRS_PER_PGD 512 | |
40 | ||
41 | /* | |
42 | * 3rd level page | |
43 | */ | |
44 | #define PUD_SHIFT 30 | |
45 | #define PTRS_PER_PUD 512 | |
46 | ||
47 | /* | |
48 | * PMD_SHIFT determines the size of the area a middle-level | |
49 | * page table can map | |
50 | */ | |
51 | #define PMD_SHIFT 21 | |
52 | #define PTRS_PER_PMD 512 | |
53 | ||
54 | /* | |
55 | * entries per page directory level | |
56 | */ | |
57 | #define PTRS_PER_PTE 512 | |
58 | ||
9d291e78 VG |
59 | #ifndef __ASSEMBLY__ |
60 | ||
1da177e4 LT |
61 | #define pte_ERROR(e) \ |
62 | printk("%s:%d: bad pte %p(%016lx).\n", __FILE__, __LINE__, &(e), pte_val(e)) | |
63 | #define pmd_ERROR(e) \ | |
64 | printk("%s:%d: bad pmd %p(%016lx).\n", __FILE__, __LINE__, &(e), pmd_val(e)) | |
65 | #define pud_ERROR(e) \ | |
66 | printk("%s:%d: bad pud %p(%016lx).\n", __FILE__, __LINE__, &(e), pud_val(e)) | |
67 | #define pgd_ERROR(e) \ | |
68 | printk("%s:%d: bad pgd %p(%016lx).\n", __FILE__, __LINE__, &(e), pgd_val(e)) | |
69 | ||
70 | #define pgd_none(x) (!pgd_val(x)) | |
71 | #define pud_none(x) (!pud_val(x)) | |
72 | ||
73 | static inline void set_pte(pte_t *dst, pte_t val) | |
74 | { | |
75 | pte_val(*dst) = pte_val(val); | |
76 | } | |
77 | #define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) | |
78 | ||
79 | static inline void set_pmd(pmd_t *dst, pmd_t val) | |
80 | { | |
7a2389b4 | 81 | *dst = val; |
1da177e4 LT |
82 | } |
83 | ||
84 | static inline void set_pud(pud_t *dst, pud_t val) | |
85 | { | |
7a2389b4 | 86 | *dst = val; |
1da177e4 LT |
87 | } |
88 | ||
9c0aa0f9 | 89 | static inline void pud_clear (pud_t *pud) |
1da177e4 LT |
90 | { |
91 | set_pud(pud, __pud(0)); | |
92 | } | |
93 | ||
94 | static inline void set_pgd(pgd_t *dst, pgd_t val) | |
95 | { | |
7a2389b4 | 96 | *dst = val; |
1da177e4 LT |
97 | } |
98 | ||
9c0aa0f9 | 99 | static inline void pgd_clear (pgd_t * pgd) |
1da177e4 LT |
100 | { |
101 | set_pgd(pgd, __pgd(0)); | |
102 | } | |
103 | ||
1da177e4 | 104 | #define ptep_get_and_clear(mm,addr,xp) __pte(xchg(&(xp)->pte, 0)) |
61e06037 | 105 | |
8c65b4a6 TS |
106 | struct mm_struct; |
107 | ||
61e06037 ZA |
108 | static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) |
109 | { | |
110 | pte_t pte; | |
111 | if (full) { | |
112 | pte = *ptep; | |
113 | *ptep = __pte(0); | |
114 | } else { | |
115 | pte = ptep_get_and_clear(mm, addr, ptep); | |
116 | } | |
117 | return pte; | |
118 | } | |
119 | ||
1da177e4 LT |
120 | #define pte_same(a, b) ((a).pte == (b).pte) |
121 | ||
c728252c AV |
122 | #define pte_pgprot(a) (__pgprot((a).pte & ~PHYSICAL_PAGE_MASK)) |
123 | ||
9d291e78 VG |
124 | #endif /* !__ASSEMBLY__ */ |
125 | ||
126 | #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) | |
1da177e4 | 127 | #define PMD_MASK (~(PMD_SIZE-1)) |
9d291e78 | 128 | #define PUD_SIZE (_AC(1,UL) << PUD_SHIFT) |
1da177e4 | 129 | #define PUD_MASK (~(PUD_SIZE-1)) |
9d291e78 | 130 | #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) |
1da177e4 LT |
131 | #define PGDIR_MASK (~(PGDIR_SIZE-1)) |
132 | ||
1da177e4 | 133 | |
63f6564d RD |
134 | #define MAXMEM _AC(0x3fffffffffff, UL) |
135 | #define VMALLOC_START _AC(0xffffc20000000000, UL) | |
136 | #define VMALLOC_END _AC(0xffffe1ffffffffff, UL) | |
0889eba5 | 137 | #define VMEMMAP_START _AC(0xffffe20000000000, UL) |
63f6564d RD |
138 | #define MODULES_VADDR _AC(0xffffffff88000000, UL) |
139 | #define MODULES_END _AC(0xfffffffffff00000, UL) | |
1da177e4 LT |
140 | #define MODULES_LEN (MODULES_END - MODULES_VADDR) |
141 | ||
9d291e78 VG |
142 | #ifndef __ASSEMBLY__ |
143 | ||
eab724e5 JB |
144 | static inline unsigned long pgd_bad(pgd_t pgd) |
145 | { | |
146 | return pgd_val(pgd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); | |
147 | } | |
1da177e4 LT |
148 | |
149 | static inline unsigned long pud_bad(pud_t pud) | |
150 | { | |
eab724e5 JB |
151 | return pud_val(pud) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); |
152 | } | |
153 | ||
154 | static inline unsigned long pmd_bad(pmd_t pmd) | |
155 | { | |
156 | return pmd_val(pmd) & ~(PTE_MASK | _KERNPG_TABLE | _PAGE_USER); | |
1da177e4 LT |
157 | } |
158 | ||
159 | #define pte_none(x) (!pte_val(x)) | |
160 | #define pte_present(x) (pte_val(x) & (_PAGE_PRESENT | _PAGE_PROTNONE)) | |
161 | #define pte_clear(mm,addr,xp) do { set_pte_at(mm, addr, xp, __pte(0)); } while (0) | |
162 | ||
1c6f7030 | 163 | #define pages_to_mb(x) ((x) >> (20-PAGE_SHIFT)) /* FIXME: is this right? */ |
1da177e4 | 164 | #define pte_page(x) pfn_to_page(pte_pfn(x)) |
6b75aeed | 165 | #define pte_pfn(x) ((pte_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) |
1da177e4 | 166 | |
1da177e4 LT |
167 | struct vm_area_struct; |
168 | ||
1da177e4 LT |
169 | static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) |
170 | { | |
171 | if (!pte_young(*ptep)) | |
172 | return 0; | |
3d1712c9 | 173 | return test_and_clear_bit(_PAGE_BIT_ACCESSED, &ptep->pte); |
1da177e4 LT |
174 | } |
175 | ||
176 | static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
177 | { | |
3d1712c9 | 178 | clear_bit(_PAGE_BIT_RW, &ptep->pte); |
1da177e4 LT |
179 | } |
180 | ||
181 | /* | |
182 | * Macro to mark a page protection value as "uncacheable". | |
183 | */ | |
184 | #define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_PCD | _PAGE_PWT)) | |
185 | ||
1da177e4 LT |
186 | |
187 | /* | |
188 | * Conversion functions: convert a page and protection to a page entry, | |
189 | * and a page entry and page directory to the page they refer to. | |
190 | */ | |
191 | ||
1da177e4 LT |
192 | /* |
193 | * Level 4 access. | |
194 | */ | |
46a82b2d DM |
195 | #define pgd_page_vaddr(pgd) ((unsigned long) __va((unsigned long)pgd_val(pgd) & PTE_MASK)) |
196 | #define pgd_page(pgd) (pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)) | |
1da177e4 LT |
197 | #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) |
198 | #define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) | |
199 | #define pgd_offset_k(address) (init_level4_pgt + pgd_index(address)) | |
200 | #define pgd_present(pgd) (pgd_val(pgd) & _PAGE_PRESENT) | |
201 | #define mk_kernel_pgd(address) ((pgd_t){ (address) | _KERNPG_TABLE }) | |
202 | ||
203 | /* PUD - Level3 access */ | |
204 | /* to find an entry in a page-table-directory. */ | |
46a82b2d DM |
205 | #define pud_page_vaddr(pud) ((unsigned long) __va(pud_val(pud) & PHYSICAL_PAGE_MASK)) |
206 | #define pud_page(pud) (pfn_to_page(pud_val(pud) >> PAGE_SHIFT)) | |
1da177e4 | 207 | #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) |
46a82b2d | 208 | #define pud_offset(pgd, address) ((pud_t *) pgd_page_vaddr(*(pgd)) + pud_index(address)) |
1da177e4 LT |
209 | #define pud_present(pud) (pud_val(pud) & _PAGE_PRESENT) |
210 | ||
1da177e4 | 211 | /* PMD - Level 2 access */ |
46a82b2d | 212 | #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val(pmd) & PTE_MASK)) |
1da177e4 LT |
213 | #define pmd_page(pmd) (pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)) |
214 | ||
215 | #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) | |
46a82b2d | 216 | #define pmd_offset(dir, address) ((pmd_t *) pud_page_vaddr(*(dir)) + \ |
1da177e4 LT |
217 | pmd_index(address)) |
218 | #define pmd_none(x) (!pmd_val(x)) | |
219 | #define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT) | |
220 | #define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0) | |
1da177e4 | 221 | #define pfn_pmd(nr,prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val(prot))) |
6b75aeed | 222 | #define pmd_pfn(x) ((pmd_val(x) & __PHYSICAL_MASK) >> PAGE_SHIFT) |
1da177e4 LT |
223 | |
224 | #define pte_to_pgoff(pte) ((pte_val(pte) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) | |
225 | #define pgoff_to_pte(off) ((pte_t) { ((off) << PAGE_SHIFT) | _PAGE_FILE }) | |
226 | #define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT | |
227 | ||
228 | /* PTE - Level 1 access. */ | |
229 | ||
230 | /* page, protection -> pte */ | |
231 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) | |
1da177e4 | 232 | |
1da177e4 | 233 | #define pte_index(address) \ |
1294b118 | 234 | (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) |
46a82b2d | 235 | #define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_vaddr(*(dir)) + \ |
1da177e4 LT |
236 | pte_index(address)) |
237 | ||
238 | /* x86-64 always has all page tables mapped. */ | |
239 | #define pte_offset_map(dir,address) pte_offset_kernel(dir,address) | |
240 | #define pte_offset_map_nested(dir,address) pte_offset_kernel(dir,address) | |
241 | #define pte_unmap(pte) /* NOP */ | |
242 | #define pte_unmap_nested(pte) /* NOP */ | |
243 | ||
244 | #define update_mmu_cache(vma,address,pte) do { } while (0) | |
245 | ||
246 | /* We only update the dirty/accessed state if we set | |
247 | * the dirty bit by hand in the kernel, since the hardware | |
248 | * will do the accessed bit for us, and we don't want to | |
249 | * race with other CPU's that might be updating the dirty | |
250 | * bit at the same time. */ | |
251 | #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS | |
252 | #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ | |
8dab5241 BH |
253 | ({ \ |
254 | int __changed = !pte_same(*(__ptep), __entry); \ | |
255 | if (__changed && __dirty) { \ | |
256 | set_pte(__ptep, __entry); \ | |
257 | flush_tlb_page(__vma, __address); \ | |
258 | } \ | |
259 | __changed; \ | |
260 | }) | |
1da177e4 LT |
261 | |
262 | /* Encode and de-code a swap entry */ | |
263 | #define __swp_type(x) (((x).val >> 1) & 0x3f) | |
264 | #define __swp_offset(x) ((x).val >> 8) | |
265 | #define __swp_entry(type, offset) ((swp_entry_t) { ((type) << 1) | ((offset) << 8) }) | |
266 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) | |
267 | #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) | |
268 | ||
8c914cb7 | 269 | extern spinlock_t pgd_lock; |
2bff7383 | 270 | extern struct list_head pgd_list; |
8c914cb7 | 271 | |
1da177e4 LT |
272 | extern int kern_addr_valid(unsigned long addr); |
273 | ||
19d36ccd AK |
274 | pte_t *lookup_address(unsigned long addr); |
275 | ||
1da177e4 LT |
276 | #define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ |
277 | remap_pfn_range(vma, vaddr, pfn, size, prot) | |
278 | ||
1da177e4 | 279 | #define HAVE_ARCH_UNMAPPED_AREA |
cc503c1b | 280 | #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN |
1da177e4 LT |
281 | |
282 | #define pgtable_cache_init() do { } while (0) | |
da8f153e | 283 | #define check_pgt_cache() do { } while (0) |
1da177e4 LT |
284 | |
285 | #define PAGE_AGP PAGE_KERNEL_NOCACHE | |
286 | #define HAVE_PAGE_AGP 1 | |
287 | ||
288 | /* fs/proc/kcore.c */ | |
289 | #define kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK) | |
290 | #define kc_offset_to_vaddr(o) \ | |
291 | (((o) & (1UL << (__VIRTUAL_MASK_SHIFT-1))) ? ((o) | (~__VIRTUAL_MASK)) : (o)) | |
292 | ||
293 | #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG | |
1da177e4 | 294 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR |
61e06037 | 295 | #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL |
1da177e4 LT |
296 | #define __HAVE_ARCH_PTEP_SET_WRPROTECT |
297 | #define __HAVE_ARCH_PTE_SAME | |
298 | #include <asm-generic/pgtable.h> | |
9d291e78 | 299 | #endif /* !__ASSEMBLY__ */ |
1da177e4 LT |
300 | |
301 | #endif /* _X86_64_PGTABLE_H */ |