Commit | Line | Data |
---|---|---|
53492b1d GS |
1 | /* |
2 | * IBM System z Huge TLB Page Support for Kernel. | |
3 | * | |
4 | * Copyright IBM Corp. 2008 | |
5 | * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> | |
6 | */ | |
7 | ||
8 | #ifndef _ASM_S390_HUGETLB_H | |
9 | #define _ASM_S390_HUGETLB_H | |
10 | ||
11 | #include <asm/page.h> | |
12 | #include <asm/pgtable.h> | |
13 | ||
14 | ||
15 | #define is_hugepage_only_range(mm, addr, len) 0 | |
16 | #define hugetlb_free_pgd_range free_pgd_range | |
17 | ||
18 | void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, | |
19 | pte_t *ptep, pte_t pte); | |
20 | ||
21 | /* | |
22 | * If the arch doesn't supply something else, assume that hugepage | |
23 | * size aligned regions are ok without further preparation. | |
24 | */ | |
a5516438 AK |
25 | static inline int prepare_hugepage_range(struct file *file, |
26 | unsigned long addr, unsigned long len) | |
53492b1d GS |
27 | { |
28 | if (len & ~HPAGE_MASK) | |
29 | return -EINVAL; | |
30 | if (addr & ~HPAGE_MASK) | |
31 | return -EINVAL; | |
32 | return 0; | |
33 | } | |
34 | ||
35 | #define hugetlb_prefault_arch_hook(mm) do { } while (0) | |
36 | ||
37 | int arch_prepare_hugepage(struct page *page); | |
38 | void arch_release_hugepage(struct page *page); | |
39 | ||
53492b1d GS |
40 | static inline pte_t huge_pte_wrprotect(pte_t pte) |
41 | { | |
42 | pte_val(pte) |= _PAGE_RO; | |
43 | return pte; | |
44 | } | |
45 | ||
46 | static inline int huge_pte_none(pte_t pte) | |
47 | { | |
48 | return (pte_val(pte) & _SEGMENT_ENTRY_INV) && | |
49 | !(pte_val(pte) & _SEGMENT_ENTRY_RO); | |
50 | } | |
51 | ||
52 | static inline pte_t huge_ptep_get(pte_t *ptep) | |
53 | { | |
54 | pte_t pte = *ptep; | |
55 | unsigned long mask; | |
56 | ||
57 | if (!MACHINE_HAS_HPAGE) { | |
58 | ptep = (pte_t *) (pte_val(pte) & _SEGMENT_ENTRY_ORIGIN); | |
59 | if (ptep) { | |
60 | mask = pte_val(pte) & | |
61 | (_SEGMENT_ENTRY_INV | _SEGMENT_ENTRY_RO); | |
62 | pte = pte_mkhuge(*ptep); | |
63 | pte_val(pte) |= mask; | |
64 | } | |
65 | } | |
66 | return pte; | |
67 | } | |
68 | ||
53492b1d GS |
69 | static inline void __pmd_csp(pmd_t *pmdp) |
70 | { | |
71 | register unsigned long reg2 asm("2") = pmd_val(*pmdp); | |
72 | register unsigned long reg3 asm("3") = pmd_val(*pmdp) | | |
73 | _SEGMENT_ENTRY_INV; | |
74 | register unsigned long reg4 asm("4") = ((unsigned long) pmdp) + 5; | |
75 | ||
76 | asm volatile( | |
77 | " csp %1,%3" | |
78 | : "=m" (*pmdp) | |
79 | : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc"); | |
80 | pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY; | |
81 | } | |
82 | ||
83 | static inline void __pmd_idte(unsigned long address, pmd_t *pmdp) | |
84 | { | |
85 | unsigned long sto = (unsigned long) pmdp - | |
86 | pmd_index(address) * sizeof(pmd_t); | |
87 | ||
88 | if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) { | |
89 | asm volatile( | |
90 | " .insn rrf,0xb98e0000,%2,%3,0,0" | |
91 | : "=m" (*pmdp) | |
92 | : "m" (*pmdp), "a" (sto), | |
93 | "a" ((address & HPAGE_MASK)) | |
94 | ); | |
95 | } | |
96 | pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY; | |
97 | } | |
98 | ||
99 | static inline void huge_ptep_invalidate(struct mm_struct *mm, | |
100 | unsigned long address, pte_t *ptep) | |
101 | { | |
102 | pmd_t *pmdp = (pmd_t *) ptep; | |
103 | ||
043d0708 | 104 | if (MACHINE_HAS_IDTE) |
53492b1d | 105 | __pmd_idte(address, pmdp); |
043d0708 MS |
106 | else |
107 | __pmd_csp(pmdp); | |
53492b1d GS |
108 | } |
109 | ||
9016083b GS |
110 | static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, |
111 | unsigned long addr, pte_t *ptep) | |
112 | { | |
113 | pte_t pte = huge_ptep_get(ptep); | |
114 | ||
115 | huge_ptep_invalidate(mm, addr, ptep); | |
116 | return pte; | |
117 | } | |
118 | ||
53492b1d GS |
119 | #define huge_ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \ |
120 | ({ \ | |
121 | int __changed = !pte_same(huge_ptep_get(__ptep), __entry); \ | |
122 | if (__changed) { \ | |
123 | huge_ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \ | |
124 | set_huge_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \ | |
125 | } \ | |
126 | __changed; \ | |
127 | }) | |
128 | ||
129 | #define huge_ptep_set_wrprotect(__mm, __addr, __ptep) \ | |
130 | ({ \ | |
131 | pte_t __pte = huge_ptep_get(__ptep); \ | |
132 | if (pte_write(__pte)) { \ | |
9016083b | 133 | huge_ptep_invalidate(__mm, __addr, __ptep); \ |
53492b1d GS |
134 | set_huge_pte_at(__mm, __addr, __ptep, \ |
135 | huge_pte_wrprotect(__pte)); \ | |
136 | } \ | |
137 | }) | |
138 | ||
139 | static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, | |
140 | unsigned long address, pte_t *ptep) | |
141 | { | |
142 | huge_ptep_invalidate(vma->vm_mm, address, ptep); | |
143 | } | |
144 | ||
145 | #endif /* _ASM_S390_HUGETLB_H */ |