Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef _S390_TLBFLUSH_H |
2 | #define _S390_TLBFLUSH_H | |
3 | ||
1da177e4 | 4 | #include <linux/mm.h> |
53492b1d | 5 | #include <linux/sched.h> |
1da177e4 | 6 | #include <asm/processor.h> |
c1821c2e | 7 | #include <asm/pgalloc.h> |
4ccccc52 | 8 | #include <asm/pgtable.h> |
1da177e4 LT |
9 | |
10 | /* | |
1b948d6c | 11 | * Flush all TLB entries on the local CPU. |
1da177e4 | 12 | */ |
ba8a9229 | 13 | static inline void __tlb_flush_local(void) |
1da177e4 | 14 | { |
ba8a9229 | 15 | asm volatile("ptlb" : : : "memory"); |
1da177e4 | 16 | } |
1da177e4 | 17 | |
ba8a9229 | 18 | /* |
1b948d6c | 19 | * Flush TLB entries for a specific ASCE on all CPUs |
ba8a9229 | 20 | */ |
1b948d6c MS |
21 | static inline void __tlb_flush_idte(unsigned long asce) |
22 | { | |
23 | /* Global TLB flush for the mm */ | |
24 | asm volatile( | |
25 | " .insn rrf,0xb98e0000,0,%0,%1,0" | |
26 | : : "a" (2048), "a" (asce) : "cc"); | |
27 | } | |
28 | ||
29 | /* | |
30 | * Flush TLB entries for a specific ASCE on the local CPU | |
31 | */ | |
32 | static inline void __tlb_flush_idte_local(unsigned long asce) | |
33 | { | |
34 | /* Local TLB flush for the mm */ | |
35 | asm volatile( | |
36 | " .insn rrf,0xb98e0000,0,%0,%1,1" | |
37 | : : "a" (2048), "a" (asce) : "cc"); | |
38 | } | |
39 | ||
40 | #ifdef CONFIG_SMP | |
a806170e HC |
41 | void smp_ptlb_all(void); |
42 | ||
1b948d6c MS |
43 | /* |
44 | * Flush all TLB entries on all CPUs. | |
45 | */ | |
ba8a9229 | 46 | static inline void __tlb_flush_global(void) |
1da177e4 | 47 | { |
4ccccc52 | 48 | unsigned int dummy = 0; |
94c12cc7 | 49 | |
4ccccc52 | 50 | csp(&dummy, 0, 0); |
1da177e4 LT |
51 | } |
52 | ||
1b948d6c MS |
53 | /* |
54 | * Flush TLB entries for a specific mm on all CPUs (in case gmap is used | |
55 | * this implicates multiple ASCEs!). | |
56 | */ | |
374b8f45 MS |
57 | static inline void __tlb_flush_full(struct mm_struct *mm) |
58 | { | |
374b8f45 | 59 | preempt_disable(); |
64f31d58 | 60 | atomic_inc(&mm->context.flush_count); |
1b948d6c MS |
61 | if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { |
62 | /* Local TLB flush */ | |
374b8f45 | 63 | __tlb_flush_local(); |
1b948d6c MS |
64 | } else { |
65 | /* Global TLB flush */ | |
374b8f45 | 66 | __tlb_flush_global(); |
1b948d6c MS |
67 | /* Reset TLB flush mask */ |
68 | if (MACHINE_HAS_TLB_LC) | |
69 | cpumask_copy(mm_cpumask(mm), | |
70 | &mm->context.cpu_attach_mask); | |
71 | } | |
64f31d58 | 72 | atomic_dec(&mm->context.flush_count); |
374b8f45 MS |
73 | preempt_enable(); |
74 | } | |
1b948d6c MS |
75 | |
76 | /* | |
f0454029 DH |
77 | * Flush TLB entries for a specific ASCE on all CPUs. Should never be used |
78 | * when more than one asce (e.g. gmap) ran on this mm. | |
1b948d6c MS |
79 | */ |
80 | static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) | |
81 | { | |
1b948d6c | 82 | preempt_disable(); |
64f31d58 MS |
83 | atomic_inc(&mm->context.flush_count); |
84 | if (MACHINE_HAS_TLB_LC && | |
1b948d6c MS |
85 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { |
86 | __tlb_flush_idte_local(asce); | |
87 | } else { | |
88 | if (MACHINE_HAS_IDTE) | |
89 | __tlb_flush_idte(asce); | |
90 | else | |
91 | __tlb_flush_global(); | |
92 | /* Reset TLB flush mask */ | |
93 | if (MACHINE_HAS_TLB_LC) | |
94 | cpumask_copy(mm_cpumask(mm), | |
95 | &mm->context.cpu_attach_mask); | |
96 | } | |
64f31d58 | 97 | atomic_dec(&mm->context.flush_count); |
1b948d6c MS |
98 | preempt_enable(); |
99 | } | |
100 | ||
101 | static inline void __tlb_flush_kernel(void) | |
102 | { | |
103 | if (MACHINE_HAS_IDTE) | |
723cacbd | 104 | __tlb_flush_idte(init_mm.context.asce); |
1b948d6c MS |
105 | else |
106 | __tlb_flush_global(); | |
107 | } | |
374b8f45 | 108 | #else |
e1c4d014 | 109 | #define __tlb_flush_global() __tlb_flush_local() |
1b948d6c | 110 | #define __tlb_flush_full(mm) __tlb_flush_local() |
374b8f45 | 111 | |
1da177e4 | 112 | /* |
1b948d6c | 113 | * Flush TLB entries for a specific ASCE on all CPUs. |
1da177e4 | 114 | */ |
1b948d6c | 115 | static inline void __tlb_flush_asce(struct mm_struct *mm, unsigned long asce) |
ba8a9229 | 116 | { |
1b948d6c MS |
117 | if (MACHINE_HAS_TLB_LC) |
118 | __tlb_flush_idte_local(asce); | |
119 | else | |
120 | __tlb_flush_local(); | |
ba8a9229 | 121 | } |
1da177e4 | 122 | |
1b948d6c MS |
123 | static inline void __tlb_flush_kernel(void) |
124 | { | |
125 | if (MACHINE_HAS_TLB_LC) | |
723cacbd | 126 | __tlb_flush_idte_local(init_mm.context.asce); |
1b948d6c MS |
127 | else |
128 | __tlb_flush_local(); | |
129 | } | |
130 | #endif | |
131 | ||
ba8a9229 | 132 | static inline void __tlb_flush_mm(struct mm_struct * mm) |
1da177e4 | 133 | { |
ba8a9229 MS |
134 | /* |
135 | * If the machine has IDTE we prefer to do a per mm flush | |
136 | * on all cpus instead of doing a local flush if the mm | |
137 | * only ran on the local cpu. | |
138 | */ | |
e5992f2e | 139 | if (MACHINE_HAS_IDTE && list_empty(&mm->context.gmap_list)) |
723cacbd | 140 | __tlb_flush_asce(mm, mm->context.asce); |
043d0708 MS |
141 | else |
142 | __tlb_flush_full(mm); | |
1da177e4 LT |
143 | } |
144 | ||
5c474a1e | 145 | static inline void __tlb_flush_mm_lazy(struct mm_struct * mm) |
1da177e4 | 146 | { |
050eef36 | 147 | if (mm->context.flush_mm) { |
ba8a9229 | 148 | __tlb_flush_mm(mm); |
050eef36 MS |
149 | mm->context.flush_mm = 0; |
150 | } | |
1da177e4 LT |
151 | } |
152 | ||
ba8a9229 MS |
153 | /* |
154 | * TLB flushing: | |
155 | * flush_tlb() - flushes the current mm struct TLBs | |
156 | * flush_tlb_all() - flushes all processes TLBs | |
157 | * flush_tlb_mm(mm) - flushes the specified mm context TLB's | |
158 | * flush_tlb_page(vma, vmaddr) - flushes one page | |
159 | * flush_tlb_range(vma, start, end) - flushes a range of pages | |
160 | * flush_tlb_kernel_range(start, end) - flushes a range of kernel pages | |
161 | */ | |
1da177e4 | 162 | |
ba8a9229 MS |
163 | /* |
164 | * flush_tlb_mm goes together with ptep_set_wrprotect for the | |
165 | * copy_page_range operation and flush_tlb_range is related to | |
166 | * ptep_get_and_clear for change_protection. ptep_set_wrprotect and | |
167 | * ptep_get_and_clear do not flush the TLBs directly if the mm has | |
168 | * only one user. At the end of the update the flush_tlb_mm and | |
169 | * flush_tlb_range functions need to do the flush. | |
170 | */ | |
171 | #define flush_tlb() do { } while (0) | |
172 | #define flush_tlb_all() do { } while (0) | |
ba8a9229 | 173 | #define flush_tlb_page(vma, addr) do { } while (0) |
8ffd74a0 MS |
174 | |
175 | static inline void flush_tlb_mm(struct mm_struct *mm) | |
176 | { | |
5c474a1e | 177 | __tlb_flush_mm_lazy(mm); |
8ffd74a0 MS |
178 | } |
179 | ||
180 | static inline void flush_tlb_range(struct vm_area_struct *vma, | |
181 | unsigned long start, unsigned long end) | |
182 | { | |
5c474a1e | 183 | __tlb_flush_mm_lazy(vma->vm_mm); |
8ffd74a0 MS |
184 | } |
185 | ||
186 | static inline void flush_tlb_kernel_range(unsigned long start, | |
187 | unsigned long end) | |
188 | { | |
1b948d6c | 189 | __tlb_flush_kernel(); |
8ffd74a0 | 190 | } |
1da177e4 | 191 | |
1da177e4 | 192 | #endif /* _S390_TLBFLUSH_H */ |