Commit | Line | Data |
---|---|---|
1965aae3 PA |
1 | #ifndef _ASM_X86_PGTABLE_64_H |
2 | #define _ASM_X86_PGTABLE_64_H | |
1da177e4 | 3 | |
6df95fd7 | 4 | #include <linux/const.h> |
9d291e78 VG |
5 | #ifndef __ASSEMBLY__ |
6 | ||
1da177e4 LT |
7 | /* |
8 | * This file contains the functions and defines necessary to modify and use | |
9 | * the x86-64 page table tree. | |
10 | */ | |
11 | #include <asm/processor.h> | |
1977f032 | 12 | #include <linux/bitops.h> |
1da177e4 LT |
13 | #include <linux/threads.h> |
14 | #include <asm/pda.h> | |
15 | ||
16 | extern pud_t level3_kernel_pgt[512]; | |
1da177e4 LT |
17 | extern pud_t level3_ident_pgt[512]; |
18 | extern pmd_t level2_kernel_pgt[512]; | |
084a2a4e JF |
19 | extern pmd_t level2_fixmap_pgt[512]; |
20 | extern pmd_t level2_ident_pgt[512]; | |
1da177e4 | 21 | extern pgd_t init_level4_pgt[]; |
1da177e4 | 22 | |
e3ebadd9 | 23 | #define swapper_pg_dir init_level4_pgt |
1da177e4 | 24 | |
1da177e4 | 25 | extern void paging_init(void); |
1da177e4 | 26 | |
9d291e78 VG |
27 | #endif /* !__ASSEMBLY__ */ |
28 | ||
85958b46 | 29 | #define SHARED_KERNEL_PMD 0 |
e4b71dcf | 30 | |
1da177e4 LT |
31 | /* |
32 | * PGDIR_SHIFT determines what a top-level page table entry can map | |
33 | */ | |
34 | #define PGDIR_SHIFT 39 | |
35 | #define PTRS_PER_PGD 512 | |
36 | ||
37 | /* | |
38 | * 3rd level page | |
39 | */ | |
40 | #define PUD_SHIFT 30 | |
41 | #define PTRS_PER_PUD 512 | |
42 | ||
43 | /* | |
44 | * PMD_SHIFT determines the size of the area a middle-level | |
45 | * page table can map | |
46 | */ | |
47 | #define PMD_SHIFT 21 | |
48 | #define PTRS_PER_PMD 512 | |
49 | ||
50 | /* | |
51 | * entries per page directory level | |
52 | */ | |
53 | #define PTRS_PER_PTE 512 | |
54 | ||
9d291e78 VG |
55 | #ifndef __ASSEMBLY__ |
56 | ||
7f94401e JP |
57 | #define pte_ERROR(e) \ |
58 | printk("%s:%d: bad pte %p(%016lx).\n", \ | |
59 | __FILE__, __LINE__, &(e), pte_val(e)) | |
60 | #define pmd_ERROR(e) \ | |
61 | printk("%s:%d: bad pmd %p(%016lx).\n", \ | |
62 | __FILE__, __LINE__, &(e), pmd_val(e)) | |
63 | #define pud_ERROR(e) \ | |
64 | printk("%s:%d: bad pud %p(%016lx).\n", \ | |
65 | __FILE__, __LINE__, &(e), pud_val(e)) | |
66 | #define pgd_ERROR(e) \ | |
67 | printk("%s:%d: bad pgd %p(%016lx).\n", \ | |
68 | __FILE__, __LINE__, &(e), pgd_val(e)) | |
1da177e4 LT |
69 | |
70 | #define pgd_none(x) (!pgd_val(x)) | |
71 | #define pud_none(x) (!pud_val(x)) | |
72 | ||
4891645e JF |
73 | struct mm_struct; |
74 | ||
0814e0ba EH |
75 | void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte); |
76 | ||
77 | ||
4891645e JF |
78 | static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr, |
79 | pte_t *ptep) | |
1da177e4 | 80 | { |
4891645e JF |
81 | *ptep = native_make_pte(0); |
82 | } | |
1da177e4 | 83 | |
4891645e | 84 | static inline void native_set_pte(pte_t *ptep, pte_t pte) |
1da177e4 | 85 | { |
4891645e JF |
86 | *ptep = pte; |
87 | } | |
1da177e4 | 88 | |
b65e6390 IM |
89 | static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) |
90 | { | |
91 | native_set_pte(ptep, pte); | |
92 | } | |
93 | ||
4891645e | 94 | static inline pte_t native_ptep_get_and_clear(pte_t *xp) |
1da177e4 | 95 | { |
4891645e JF |
96 | #ifdef CONFIG_SMP |
97 | return native_make_pte(xchg(&xp->pte, 0)); | |
98 | #else | |
7f94401e JP |
99 | /* native_local_ptep_get_and_clear, |
100 | but duplicated because of cyclic dependency */ | |
4891645e JF |
101 | pte_t ret = *xp; |
102 | native_pte_clear(NULL, 0, xp); | |
103 | return ret; | |
104 | #endif | |
1da177e4 LT |
105 | } |
106 | ||
4891645e | 107 | static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) |
1da177e4 | 108 | { |
4891645e | 109 | *pmdp = pmd; |
1da177e4 LT |
110 | } |
111 | ||
4891645e | 112 | static inline void native_pmd_clear(pmd_t *pmd) |
1da177e4 | 113 | { |
4891645e JF |
114 | native_set_pmd(pmd, native_make_pmd(0)); |
115 | } | |
1da177e4 | 116 | |
4891645e | 117 | static inline void native_set_pud(pud_t *pudp, pud_t pud) |
1da177e4 | 118 | { |
4891645e | 119 | *pudp = pud; |
1da177e4 LT |
120 | } |
121 | ||
4891645e JF |
122 | static inline void native_pud_clear(pud_t *pud) |
123 | { | |
124 | native_set_pud(pud, native_make_pud(0)); | |
125 | } | |
61e06037 | 126 | |
4891645e JF |
127 | static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) |
128 | { | |
129 | *pgdp = pgd; | |
130 | } | |
8c65b4a6 | 131 | |
7f94401e | 132 | static inline void native_pgd_clear(pgd_t *pgd) |
61e06037 | 133 | { |
4891645e | 134 | native_set_pgd(pgd, native_make_pgd(0)); |
61e06037 ZA |
135 | } |
136 | ||
1da177e4 LT |
137 | #define pte_same(a, b) ((a).pte == (b).pte) |
138 | ||
9d291e78 VG |
139 | #endif /* !__ASSEMBLY__ */ |
140 | ||
7f94401e JP |
141 | #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) |
142 | #define PMD_MASK (~(PMD_SIZE - 1)) | |
143 | #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) | |
144 | #define PUD_MASK (~(PUD_SIZE - 1)) | |
145 | #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) | |
146 | #define PGDIR_MASK (~(PGDIR_SIZE - 1)) | |
1da177e4 | 147 | |
1da177e4 | 148 | |
b6fd6f26 | 149 | #define MAXMEM _AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) |
63f6564d RD |
150 | #define VMALLOC_START _AC(0xffffc20000000000, UL) |
151 | #define VMALLOC_END _AC(0xffffe1ffffffffff, UL) | |
0889eba5 | 152 | #define VMEMMAP_START _AC(0xffffe20000000000, UL) |
85eb69a1 | 153 | #define MODULES_VADDR _AC(0xffffffffa0000000, UL) |
66d4bdf2 | 154 | #define MODULES_END _AC(0xffffffffff000000, UL) |
1da177e4 LT |
155 | #define MODULES_LEN (MODULES_END - MODULES_VADDR) |
156 | ||
9d291e78 VG |
157 | #ifndef __ASSEMBLY__ |
158 | ||
a8375bd8 | 159 | static inline int pgd_bad(pgd_t pgd) |
eab724e5 | 160 | { |
59438c9f | 161 | return (pgd_val(pgd) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE; |
eab724e5 | 162 | } |
1da177e4 | 163 | |
a8375bd8 | 164 | static inline int pud_bad(pud_t pud) |
1da177e4 | 165 | { |
59438c9f | 166 | return (pud_val(pud) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE; |
eab724e5 JB |
167 | } |
168 | ||
a8375bd8 | 169 | static inline int pmd_bad(pmd_t pmd) |
eab724e5 | 170 | { |
59438c9f | 171 | return (pmd_val(pmd) & ~(PTE_PFN_MASK | _PAGE_USER)) != _KERNPG_TABLE; |
1da177e4 LT |
172 | } |
173 | ||
7f94401e JP |
174 | #define pte_none(x) (!pte_val((x))) |
175 | #define pte_present(x) (pte_val((x)) & (_PAGE_PRESENT | _PAGE_PROTNONE)) | |
1da177e4 | 176 | |
7f94401e | 177 | #define pages_to_mb(x) ((x) >> (20 - PAGE_SHIFT)) /* FIXME: is this right? */ |
1da177e4 | 178 | |
1da177e4 LT |
179 | /* |
180 | * Conversion functions: convert a page and protection to a page entry, | |
181 | * and a page entry and page directory to the page they refer to. | |
182 | */ | |
183 | ||
1da177e4 LT |
184 | /* |
185 | * Level 4 access. | |
186 | */ | |
7f94401e | 187 | #define pgd_page_vaddr(pgd) \ |
59438c9f | 188 | ((unsigned long)__va((unsigned long)pgd_val((pgd)) & PTE_PFN_MASK)) |
7f94401e | 189 | #define pgd_page(pgd) (pfn_to_page(pgd_val((pgd)) >> PAGE_SHIFT)) |
1da177e4 | 190 | #define pgd_present(pgd) (pgd_val(pgd) & _PAGE_PRESENT) |
e00fc542 | 191 | static inline int pgd_large(pgd_t pgd) { return 0; } |
e7a9b0b3 | 192 | #define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE) |
1da177e4 LT |
193 | |
194 | /* PUD - Level3 access */ | |
195 | /* to find an entry in a page-table-directory. */ | |
7f94401e JP |
196 | #define pud_page_vaddr(pud) \ |
197 | ((unsigned long)__va(pud_val((pud)) & PHYSICAL_PAGE_MASK)) | |
198 | #define pud_page(pud) (pfn_to_page(pud_val((pud)) >> PAGE_SHIFT)) | |
199 | #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) | |
200 | #define pud_offset(pgd, address) \ | |
201 | ((pud_t *)pgd_page_vaddr(*(pgd)) + pud_index((address))) | |
202 | #define pud_present(pud) (pud_val((pud)) & _PAGE_PRESENT) | |
1da177e4 | 203 | |
61e19a34 AK |
204 | static inline int pud_large(pud_t pte) |
205 | { | |
7f94401e JP |
206 | return (pud_val(pte) & (_PAGE_PSE | _PAGE_PRESENT)) == |
207 | (_PAGE_PSE | _PAGE_PRESENT); | |
61e19a34 AK |
208 | } |
209 | ||
1da177e4 | 210 | /* PMD - Level 2 access */ |
59438c9f | 211 | #define pmd_page_vaddr(pmd) ((unsigned long) __va(pmd_val((pmd)) & PTE_PFN_MASK)) |
7f94401e JP |
212 | #define pmd_page(pmd) (pfn_to_page(pmd_val((pmd)) >> PAGE_SHIFT)) |
213 | ||
214 | #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) | |
215 | #define pmd_offset(dir, address) ((pmd_t *)pud_page_vaddr(*(dir)) + \ | |
216 | pmd_index(address)) | |
217 | #define pmd_none(x) (!pmd_val((x))) | |
218 | #define pmd_present(x) (pmd_val((x)) & _PAGE_PRESENT) | |
219 | #define pfn_pmd(nr, prot) (__pmd(((nr) << PAGE_SHIFT) | pgprot_val((prot)))) | |
220 | #define pmd_pfn(x) ((pmd_val((x)) & __PHYSICAL_MASK) >> PAGE_SHIFT) | |
221 | ||
222 | #define pte_to_pgoff(pte) ((pte_val((pte)) & PHYSICAL_PAGE_MASK) >> PAGE_SHIFT) | |
223 | #define pgoff_to_pte(off) ((pte_t) { .pte = ((off) << PAGE_SHIFT) | \ | |
224 | _PAGE_FILE }) | |
1da177e4 LT |
225 | #define PTE_FILE_MAX_BITS __PHYSICAL_MASK_SHIFT |
226 | ||
227 | /* PTE - Level 1 access. */ | |
228 | ||
229 | /* page, protection -> pte */ | |
7f94401e JP |
230 | #define mk_pte(page, pgprot) pfn_pte(page_to_pfn((page)), (pgprot)) |
231 | ||
232 | #define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) | |
46a82b2d | 233 | #define pte_offset_kernel(dir, address) ((pte_t *) pmd_page_vaddr(*(dir)) + \ |
7f94401e | 234 | pte_index((address))) |
1da177e4 LT |
235 | |
236 | /* x86-64 always has all page tables mapped. */ | |
7f94401e JP |
237 | #define pte_offset_map(dir, address) pte_offset_kernel((dir), (address)) |
238 | #define pte_offset_map_nested(dir, address) pte_offset_kernel((dir), (address)) | |
1da177e4 | 239 | #define pte_unmap(pte) /* NOP */ |
7f94401e | 240 | #define pte_unmap_nested(pte) /* NOP */ |
1da177e4 | 241 | |
7f94401e | 242 | #define update_mmu_cache(vma, address, pte) do { } while (0) |
1da177e4 | 243 | |
00d1c5e0 IM |
244 | extern int direct_gbpages; |
245 | ||
1da177e4 | 246 | /* Encode and de-code a swap entry */ |
1796316a JB |
247 | #if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE |
248 | #define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1) | |
249 | #define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1) | |
250 | #else | |
251 | #define SWP_TYPE_BITS (_PAGE_BIT_PROTNONE - _PAGE_BIT_PRESENT - 1) | |
252 | #define SWP_OFFSET_SHIFT (_PAGE_BIT_FILE + 1) | |
253 | #endif | |
254 | ||
255 | #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) | |
256 | ||
257 | #define __swp_type(x) (((x).val >> (_PAGE_BIT_PRESENT + 1)) \ | |
258 | & ((1U << SWP_TYPE_BITS) - 1)) | |
259 | #define __swp_offset(x) ((x).val >> SWP_OFFSET_SHIFT) | |
260 | #define __swp_entry(type, offset) ((swp_entry_t) { \ | |
261 | ((type) << (_PAGE_BIT_PRESENT + 1)) \ | |
262 | | ((offset) << SWP_OFFSET_SHIFT) }) | |
7f94401e | 263 | #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val((pte)) }) |
c8e5393a | 264 | #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) |
1da177e4 | 265 | |
7f94401e | 266 | extern int kern_addr_valid(unsigned long addr); |
31eedd82 | 267 | extern void cleanup_highmap(void); |
1da177e4 | 268 | |
7f94401e JP |
269 | #define io_remap_pfn_range(vma, vaddr, pfn, size, prot) \ |
270 | remap_pfn_range(vma, vaddr, pfn, size, prot) | |
1da177e4 | 271 | |
1da177e4 | 272 | #define HAVE_ARCH_UNMAPPED_AREA |
cc503c1b | 273 | #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN |
1da177e4 LT |
274 | |
275 | #define pgtable_cache_init() do { } while (0) | |
da8f153e | 276 | #define check_pgt_cache() do { } while (0) |
1da177e4 LT |
277 | |
278 | #define PAGE_AGP PAGE_KERNEL_NOCACHE | |
279 | #define HAVE_PAGE_AGP 1 | |
280 | ||
281 | /* fs/proc/kcore.c */ | |
282 | #define kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK) | |
7f94401e JP |
283 | #define kc_offset_to_vaddr(o) \ |
284 | (((o) & (1UL << (__VIRTUAL_MASK_SHIFT - 1))) \ | |
285 | ? ((o) | ~__VIRTUAL_MASK) \ | |
286 | : (o)) | |
1da177e4 | 287 | |
1da177e4 | 288 | #define __HAVE_ARCH_PTE_SAME |
9d291e78 | 289 | #endif /* !__ASSEMBLY__ */ |
1da177e4 | 290 | |
1965aae3 | 291 | #endif /* _ASM_X86_PGTABLE_64_H */ |