Commit | Line | Data |
---|---|---|
18468d93 RA |
1 | /* |
2 | * include/linux/balloon_compaction.h | |
3 | * | |
4 | * Common interface definitions for making balloon pages movable by compaction. | |
5 | * | |
6 | * Despite being perfectly possible to perform ballooned pages migration, they | |
7 | * make a special corner case to compaction scans because balloon pages are not | |
8 | * enlisted at any LRU list like the other pages we do compact / migrate. | |
9 | * | |
10 | * As the page isolation scanning step a compaction thread does is a lockless | |
11 | * procedure (from a page standpoint), it might bring some racy situations while | |
12 | * performing balloon page compaction. In order to sort out these racy scenarios | |
13 | * and safely perform balloon's page compaction and migration we must, always, | |
14 | * ensure following these three simple rules: | |
15 | * | |
16 | * i. when updating a balloon's page ->mapping element, strictly do it under | |
17 | * the following lock order, independently of the far superior | |
18 | * locking scheme (lru_lock, balloon_lock): | |
19 | * +-page_lock(page); | |
20 | * +--spin_lock_irq(&b_dev_info->pages_lock); | |
21 | * ... page->mapping updates here ... | |
22 | * | |
23 | * ii. before isolating or dequeueing a balloon page from the balloon device | |
24 | * pages list, the page reference counter must be raised by one and the | |
25 | * extra refcount must be dropped when the page is enqueued back into | |
26 | * the balloon device page list, thus a balloon page keeps its reference | |
27 | * counter raised only while it is under our special handling; | |
28 | * | |
29 | * iii. after the lockless scan step have selected a potential balloon page for | |
30 | * isolation, re-test the page->mapping flags and the page ref counter | |
31 | * under the proper page lock, to ensure isolating a valid balloon page | |
32 | * (not yet isolated, nor under release procedure) | |
33 | * | |
34 | * The functions provided by this interface are placed to help on coping with | |
35 | * the aforementioned balloon page corner case, as well as to ensure the simple | |
36 | * set of exposed rules are satisfied while we are dealing with balloon pages | |
37 | * compaction / migration. | |
38 | * | |
39 | * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com> | |
40 | */ | |
41 | #ifndef _LINUX_BALLOON_COMPACTION_H | |
42 | #define _LINUX_BALLOON_COMPACTION_H | |
43 | #include <linux/pagemap.h> | |
44 | #include <linux/page-flags.h> | |
45 | #include <linux/migrate.h> | |
46 | #include <linux/gfp.h> | |
47 | #include <linux/err.h> | |
48 | ||
49 | /* | |
50 | * Balloon device information descriptor. | |
51 | * This struct is used to allow the common balloon compaction interface | |
52 | * procedures to find the proper balloon device holding memory pages they'll | |
53 | * have to cope for page compaction / migration, as well as it serves the | |
54 | * balloon driver as a page book-keeper for its registered balloon devices. | |
55 | */ | |
56 | struct balloon_dev_info { | |
57 | void *balloon_device; /* balloon device descriptor */ | |
58 | struct address_space *mapping; /* balloon special page->mapping */ | |
59 | unsigned long isolated_pages; /* # of isolated pages for migration */ | |
60 | spinlock_t pages_lock; /* Protection to pages list */ | |
61 | struct list_head pages; /* Pages enqueued & handled to Host */ | |
62 | }; | |
63 | ||
64 | extern struct page *balloon_page_enqueue(struct balloon_dev_info *b_dev_info); | |
65 | extern struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info); | |
66 | extern struct balloon_dev_info *balloon_devinfo_alloc( | |
67 | void *balloon_dev_descriptor); | |
68 | ||
69 | static inline void balloon_devinfo_free(struct balloon_dev_info *b_dev_info) | |
70 | { | |
71 | kfree(b_dev_info); | |
72 | } | |
73 | ||
74 | /* | |
75 | * balloon_page_free - release a balloon page back to the page free lists | |
76 | * @page: ballooned page to be set free | |
77 | * | |
78 | * This function must be used to properly set free an isolated/dequeued balloon | |
79 | * page at the end of a sucessful page migration, or at the balloon driver's | |
80 | * page release procedure. | |
81 | */ | |
82 | static inline void balloon_page_free(struct page *page) | |
83 | { | |
84 | /* | |
85 | * Balloon pages always get an extra refcount before being isolated | |
86 | * and before being dequeued to help on sorting out fortuite colisions | |
87 | * between a thread attempting to isolate and another thread attempting | |
88 | * to release the very same balloon page. | |
89 | * | |
90 | * Before we handle the page back to Buddy, lets drop its extra refcnt. | |
91 | */ | |
92 | put_page(page); | |
93 | __free_page(page); | |
94 | } | |
95 | ||
96 | #ifdef CONFIG_BALLOON_COMPACTION | |
97 | extern bool balloon_page_isolate(struct page *page); | |
98 | extern void balloon_page_putback(struct page *page); | |
99 | extern int balloon_page_migrate(struct page *newpage, | |
100 | struct page *page, enum migrate_mode mode); | |
101 | extern struct address_space | |
102 | *balloon_mapping_alloc(struct balloon_dev_info *b_dev_info, | |
103 | const struct address_space_operations *a_ops); | |
104 | ||
105 | static inline void balloon_mapping_free(struct address_space *balloon_mapping) | |
106 | { | |
107 | kfree(balloon_mapping); | |
108 | } | |
109 | ||
110 | /* | |
111 | * page_flags_cleared - helper to perform balloon @page ->flags tests. | |
112 | * | |
113 | * As balloon pages are obtained from buddy and we do not play with page->flags | |
114 | * at driver level (exception made when we get the page lock for compaction), | |
115 | * we can safely identify a ballooned page by checking if the | |
116 | * PAGE_FLAGS_CHECK_AT_PREP page->flags are all cleared. This approach also | |
117 | * helps us skip ballooned pages that are locked for compaction or release, thus | |
118 | * mitigating their racy check at balloon_page_movable() | |
119 | */ | |
120 | static inline bool page_flags_cleared(struct page *page) | |
121 | { | |
122 | return !(page->flags & PAGE_FLAGS_CHECK_AT_PREP); | |
123 | } | |
124 | ||
125 | /* | |
126 | * __is_movable_balloon_page - helper to perform @page mapping->flags tests | |
127 | */ | |
128 | static inline bool __is_movable_balloon_page(struct page *page) | |
129 | { | |
130 | struct address_space *mapping = page->mapping; | |
131 | return mapping_balloon(mapping); | |
132 | } | |
133 | ||
134 | /* | |
135 | * balloon_page_movable - test page->mapping->flags to identify balloon pages | |
136 | * that can be moved by compaction/migration. | |
137 | * | |
138 | * This function is used at core compaction's page isolation scheme, therefore | |
139 | * most pages exposed to it are not enlisted as balloon pages and so, to avoid | |
140 | * undesired side effects like racing against __free_pages(), we cannot afford | |
141 | * holding the page locked while testing page->mapping->flags here. | |
142 | * | |
143 | * As we might return false positives in the case of a balloon page being just | |
144 | * released under us, the page->mapping->flags need to be re-tested later, | |
145 | * under the proper page lock, at the functions that will be coping with the | |
146 | * balloon page case. | |
147 | */ | |
148 | static inline bool balloon_page_movable(struct page *page) | |
149 | { | |
150 | /* | |
151 | * Before dereferencing and testing mapping->flags, let's make sure | |
152 | * this is not a page that uses ->mapping in a different way | |
153 | */ | |
154 | if (page_flags_cleared(page) && !page_mapped(page) && | |
155 | page_count(page) == 1) | |
156 | return __is_movable_balloon_page(page); | |
157 | ||
158 | return false; | |
159 | } | |
160 | ||
117aad1e RA |
161 | /* |
162 | * isolated_balloon_page - identify an isolated balloon page on private | |
163 | * compaction/migration page lists. | |
164 | * | |
165 | * After a compaction thread isolates a balloon page for migration, it raises | |
166 | * the page refcount to prevent concurrent compaction threads from re-isolating | |
167 | * the same page. For that reason putback_movable_pages(), or other routines | |
168 | * that need to identify isolated balloon pages on private pagelists, cannot | |
169 | * rely on balloon_page_movable() to accomplish the task. | |
170 | */ | |
171 | static inline bool isolated_balloon_page(struct page *page) | |
172 | { | |
173 | /* Already isolated balloon pages, by default, have a raised refcount */ | |
174 | if (page_flags_cleared(page) && !page_mapped(page) && | |
175 | page_count(page) >= 2) | |
176 | return __is_movable_balloon_page(page); | |
177 | ||
178 | return false; | |
179 | } | |
180 | ||
18468d93 RA |
181 | /* |
182 | * balloon_page_insert - insert a page into the balloon's page list and make | |
183 | * the page->mapping assignment accordingly. | |
184 | * @page : page to be assigned as a 'balloon page' | |
185 | * @mapping : allocated special 'balloon_mapping' | |
186 | * @head : balloon's device page list head | |
187 | * | |
188 | * Caller must ensure the page is locked and the spin_lock protecting balloon | |
189 | * pages list is held before inserting a page into the balloon device. | |
190 | */ | |
191 | static inline void balloon_page_insert(struct page *page, | |
192 | struct address_space *mapping, | |
193 | struct list_head *head) | |
194 | { | |
195 | page->mapping = mapping; | |
196 | list_add(&page->lru, head); | |
197 | } | |
198 | ||
199 | /* | |
200 | * balloon_page_delete - delete a page from balloon's page list and clear | |
201 | * the page->mapping assignement accordingly. | |
202 | * @page : page to be released from balloon's page list | |
203 | * | |
204 | * Caller must ensure the page is locked and the spin_lock protecting balloon | |
205 | * pages list is held before deleting a page from the balloon device. | |
206 | */ | |
207 | static inline void balloon_page_delete(struct page *page) | |
208 | { | |
209 | page->mapping = NULL; | |
210 | list_del(&page->lru); | |
211 | } | |
212 | ||
213 | /* | |
214 | * balloon_page_device - get the b_dev_info descriptor for the balloon device | |
215 | * that enqueues the given page. | |
216 | */ | |
217 | static inline struct balloon_dev_info *balloon_page_device(struct page *page) | |
218 | { | |
219 | struct address_space *mapping = page->mapping; | |
220 | if (likely(mapping)) | |
221 | return mapping->private_data; | |
222 | ||
223 | return NULL; | |
224 | } | |
225 | ||
226 | static inline gfp_t balloon_mapping_gfp_mask(void) | |
227 | { | |
228 | return GFP_HIGHUSER_MOVABLE; | |
229 | } | |
230 | ||
231 | static inline bool balloon_compaction_check(void) | |
232 | { | |
233 | return true; | |
234 | } | |
235 | ||
236 | #else /* !CONFIG_BALLOON_COMPACTION */ | |
237 | ||
238 | static inline void *balloon_mapping_alloc(void *balloon_device, | |
239 | const struct address_space_operations *a_ops) | |
240 | { | |
241 | return ERR_PTR(-EOPNOTSUPP); | |
242 | } | |
243 | ||
244 | static inline void balloon_mapping_free(struct address_space *balloon_mapping) | |
245 | { | |
246 | return; | |
247 | } | |
248 | ||
249 | static inline void balloon_page_insert(struct page *page, | |
250 | struct address_space *mapping, | |
251 | struct list_head *head) | |
252 | { | |
253 | list_add(&page->lru, head); | |
254 | } | |
255 | ||
256 | static inline void balloon_page_delete(struct page *page) | |
257 | { | |
258 | list_del(&page->lru); | |
259 | } | |
260 | ||
261 | static inline bool balloon_page_movable(struct page *page) | |
262 | { | |
263 | return false; | |
264 | } | |
265 | ||
117aad1e RA |
266 | static inline bool isolated_balloon_page(struct page *page) |
267 | { | |
268 | return false; | |
269 | } | |
270 | ||
18468d93 RA |
271 | static inline bool balloon_page_isolate(struct page *page) |
272 | { | |
273 | return false; | |
274 | } | |
275 | ||
276 | static inline void balloon_page_putback(struct page *page) | |
277 | { | |
278 | return; | |
279 | } | |
280 | ||
281 | static inline int balloon_page_migrate(struct page *newpage, | |
282 | struct page *page, enum migrate_mode mode) | |
283 | { | |
284 | return 0; | |
285 | } | |
286 | ||
287 | static inline gfp_t balloon_mapping_gfp_mask(void) | |
288 | { | |
289 | return GFP_HIGHUSER; | |
290 | } | |
291 | ||
292 | static inline bool balloon_compaction_check(void) | |
293 | { | |
294 | return false; | |
295 | } | |
296 | #endif /* CONFIG_BALLOON_COMPACTION */ | |
297 | #endif /* _LINUX_BALLOON_COMPACTION_H */ |