Commit | Line | Data |
---|---|---|
63f9600f JSR |
1 | /* |
2 | * MTRR (Memory Type Range Register) cleanup | |
3 | * | |
4 | * Copyright (C) 2009 Yinghai Lu | |
5 | * | |
6 | * This library is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU Library General Public | |
8 | * License as published by the Free Software Foundation; either | |
9 | * version 2 of the License, or (at your option) any later version. | |
10 | * | |
11 | * This library is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | * Library General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU Library General Public | |
17 | * License along with this library; if not, write to the Free | |
18 | * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | |
19 | */ | |
0d890355 YL |
20 | #include <linux/module.h> |
21 | #include <linux/init.h> | |
22 | #include <linux/pci.h> | |
23 | #include <linux/smp.h> | |
24 | #include <linux/cpu.h> | |
63f9600f JSR |
25 | #include <linux/mutex.h> |
26 | #include <linux/uaccess.h> | |
27 | #include <linux/kvm_para.h> | |
27811d8c | 28 | #include <linux/range.h> |
0d890355 | 29 | |
63f9600f | 30 | #include <asm/processor.h> |
0d890355 YL |
31 | #include <asm/e820.h> |
32 | #include <asm/mtrr.h> | |
0d890355 | 33 | #include <asm/msr.h> |
63f9600f | 34 | |
0d890355 YL |
35 | #include "mtrr.h" |
36 | ||
e3d0e692 IM |
37 | struct var_mtrr_range_state { |
38 | unsigned long base_pfn; | |
39 | unsigned long size_pfn; | |
40 | mtrr_type type; | |
41 | }; | |
42 | ||
43 | struct var_mtrr_state { | |
44 | unsigned long range_startk; | |
45 | unsigned long range_sizek; | |
46 | unsigned long chunk_sizek; | |
47 | unsigned long gran_sizek; | |
48 | unsigned int reg; | |
49 | }; | |
50 | ||
51 | /* Should be related to MTRR_VAR_RANGES nums */ | |
52 | #define RANGE_NUM 256 | |
53 | ||
27811d8c | 54 | static struct range __initdata range[RANGE_NUM]; |
e3d0e692 IM |
55 | static int __initdata nr_range; |
56 | ||
57 | static struct var_mtrr_range_state __initdata range_state[RANGE_NUM]; | |
58 | ||
59 | static int __initdata debug_print; | |
1b74dde7 | 60 | #define Dprintk(x...) do { if (debug_print) pr_debug(x); } while (0) |
e3d0e692 | 61 | |
1b74dde7 | 62 | #define BIOS_BUG_MSG \ |
63f9600f | 63 | "WARNING: BIOS bug: VAR MTRR %d contains strange UC entry under 1M, check with your system vendor!\n" |
0d890355 YL |
64 | |
65 | static int __init | |
27811d8c | 66 | x86_get_mtrr_mem_range(struct range *range, int nr_range, |
0d890355 YL |
67 | unsigned long extra_remove_base, |
68 | unsigned long extra_remove_size) | |
69 | { | |
4e16c888 | 70 | unsigned long base, size; |
0d890355 | 71 | mtrr_type type; |
4e16c888 | 72 | int i; |
0d890355 YL |
73 | |
74 | for (i = 0; i < num_var_ranges; i++) { | |
75 | type = range_state[i].type; | |
76 | if (type != MTRR_TYPE_WRBACK) | |
77 | continue; | |
78 | base = range_state[i].base_pfn; | |
79 | size = range_state[i].size_pfn; | |
27811d8c | 80 | nr_range = add_range_with_merge(range, RANGE_NUM, nr_range, |
e9a0064a | 81 | base, base + size); |
0d890355 YL |
82 | } |
83 | if (debug_print) { | |
1b74dde7 | 84 | pr_debug("After WB checking\n"); |
0d890355 | 85 | for (i = 0; i < nr_range; i++) |
1b74dde7 | 86 | pr_debug("MTRR MAP PFN: %016llx - %016llx\n", |
e9a0064a | 87 | range[i].start, range[i].end); |
0d890355 YL |
88 | } |
89 | ||
63f9600f | 90 | /* Take out UC ranges: */ |
0d890355 YL |
91 | for (i = 0; i < num_var_ranges; i++) { |
92 | type = range_state[i].type; | |
93 | if (type != MTRR_TYPE_UNCACHABLE && | |
94 | type != MTRR_TYPE_WRPROT) | |
95 | continue; | |
96 | size = range_state[i].size_pfn; | |
97 | if (!size) | |
98 | continue; | |
99 | base = range_state[i].base_pfn; | |
f0348c43 | 100 | if (base < (1<<(20-PAGE_SHIFT)) && mtrr_state.have_fixed && |
9b3aca62 TK |
101 | (mtrr_state.enabled & MTRR_STATE_MTRR_ENABLED) && |
102 | (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) { | |
f0348c43 | 103 | /* Var MTRR contains UC entry below 1M? Skip it: */ |
1b74dde7 | 104 | pr_warn(BIOS_BUG_MSG, i); |
f0348c43 YL |
105 | if (base + size <= (1<<(20-PAGE_SHIFT))) |
106 | continue; | |
107 | size -= (1<<(20-PAGE_SHIFT)) - base; | |
108 | base = 1<<(20-PAGE_SHIFT); | |
109 | } | |
e9a0064a | 110 | subtract_range(range, RANGE_NUM, base, base + size); |
0d890355 YL |
111 | } |
112 | if (extra_remove_size) | |
27811d8c | 113 | subtract_range(range, RANGE_NUM, extra_remove_base, |
e9a0064a | 114 | extra_remove_base + extra_remove_size); |
0d890355 | 115 | |
0d890355 | 116 | if (debug_print) { |
1b74dde7 | 117 | pr_debug("After UC checking\n"); |
5bf65b9b YL |
118 | for (i = 0; i < RANGE_NUM; i++) { |
119 | if (!range[i].end) | |
120 | continue; | |
1b74dde7 | 121 | pr_debug("MTRR MAP PFN: %016llx - %016llx\n", |
e9a0064a | 122 | range[i].start, range[i].end); |
5bf65b9b | 123 | } |
0d890355 YL |
124 | } |
125 | ||
126 | /* sort the ranges */ | |
5bf65b9b | 127 | nr_range = clean_sort_range(range, RANGE_NUM); |
0d890355 | 128 | if (debug_print) { |
1b74dde7 | 129 | pr_debug("After sorting\n"); |
0d890355 | 130 | for (i = 0; i < nr_range; i++) |
1b74dde7 | 131 | pr_debug("MTRR MAP PFN: %016llx - %016llx\n", |
e9a0064a | 132 | range[i].start, range[i].end); |
0d890355 YL |
133 | } |
134 | ||
0d890355 YL |
135 | return nr_range; |
136 | } | |
137 | ||
0d890355 YL |
138 | #ifdef CONFIG_MTRR_SANITIZER |
139 | ||
27811d8c | 140 | static unsigned long __init sum_ranges(struct range *range, int nr_range) |
0d890355 | 141 | { |
63f9600f | 142 | unsigned long sum = 0; |
0d890355 YL |
143 | int i; |
144 | ||
0d890355 | 145 | for (i = 0; i < nr_range; i++) |
e9a0064a | 146 | sum += range[i].end - range[i].start; |
0d890355 YL |
147 | |
148 | return sum; | |
149 | } | |
150 | ||
151 | static int enable_mtrr_cleanup __initdata = | |
152 | CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT; | |
153 | ||
154 | static int __init disable_mtrr_cleanup_setup(char *str) | |
155 | { | |
156 | enable_mtrr_cleanup = 0; | |
157 | return 0; | |
158 | } | |
159 | early_param("disable_mtrr_cleanup", disable_mtrr_cleanup_setup); | |
160 | ||
161 | static int __init enable_mtrr_cleanup_setup(char *str) | |
162 | { | |
163 | enable_mtrr_cleanup = 1; | |
164 | return 0; | |
165 | } | |
166 | early_param("enable_mtrr_cleanup", enable_mtrr_cleanup_setup); | |
167 | ||
168 | static int __init mtrr_cleanup_debug_setup(char *str) | |
169 | { | |
170 | debug_print = 1; | |
171 | return 0; | |
172 | } | |
173 | early_param("mtrr_cleanup_debug", mtrr_cleanup_debug_setup); | |
174 | ||
0d890355 YL |
175 | static void __init |
176 | set_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, | |
63f9600f | 177 | unsigned char type, unsigned int address_bits) |
0d890355 YL |
178 | { |
179 | u32 base_lo, base_hi, mask_lo, mask_hi; | |
180 | u64 base, mask; | |
181 | ||
182 | if (!sizek) { | |
183 | fill_mtrr_var_range(reg, 0, 0, 0, 0); | |
184 | return; | |
185 | } | |
186 | ||
187 | mask = (1ULL << address_bits) - 1; | |
188 | mask &= ~((((u64)sizek) << 10) - 1); | |
189 | ||
63f9600f | 190 | base = ((u64)basek) << 10; |
0d890355 YL |
191 | |
192 | base |= type; | |
193 | mask |= 0x800; | |
194 | ||
195 | base_lo = base & ((1ULL<<32) - 1); | |
196 | base_hi = base >> 32; | |
197 | ||
198 | mask_lo = mask & ((1ULL<<32) - 1); | |
199 | mask_hi = mask >> 32; | |
200 | ||
201 | fill_mtrr_var_range(reg, base_lo, base_hi, mask_lo, mask_hi); | |
202 | } | |
203 | ||
204 | static void __init | |
205 | save_var_mtrr(unsigned int reg, unsigned long basek, unsigned long sizek, | |
63f9600f | 206 | unsigned char type) |
0d890355 YL |
207 | { |
208 | range_state[reg].base_pfn = basek >> (PAGE_SHIFT - 10); | |
209 | range_state[reg].size_pfn = sizek >> (PAGE_SHIFT - 10); | |
210 | range_state[reg].type = type; | |
211 | } | |
212 | ||
63f9600f | 213 | static void __init set_var_mtrr_all(unsigned int address_bits) |
0d890355 YL |
214 | { |
215 | unsigned long basek, sizek; | |
216 | unsigned char type; | |
217 | unsigned int reg; | |
218 | ||
219 | for (reg = 0; reg < num_var_ranges; reg++) { | |
220 | basek = range_state[reg].base_pfn << (PAGE_SHIFT - 10); | |
221 | sizek = range_state[reg].size_pfn << (PAGE_SHIFT - 10); | |
222 | type = range_state[reg].type; | |
223 | ||
224 | set_var_mtrr(reg, basek, sizek, type, address_bits); | |
225 | } | |
226 | } | |
227 | ||
228 | static unsigned long to_size_factor(unsigned long sizek, char *factorp) | |
229 | { | |
0d890355 | 230 | unsigned long base = sizek; |
63f9600f | 231 | char factor; |
0d890355 YL |
232 | |
233 | if (base & ((1<<10) - 1)) { | |
63f9600f | 234 | /* Not MB-aligned: */ |
0d890355 YL |
235 | factor = 'K'; |
236 | } else if (base & ((1<<20) - 1)) { | |
237 | factor = 'M'; | |
238 | base >>= 10; | |
239 | } else { | |
240 | factor = 'G'; | |
241 | base >>= 20; | |
242 | } | |
243 | ||
244 | *factorp = factor; | |
245 | ||
246 | return base; | |
247 | } | |
248 | ||
249 | static unsigned int __init | |
250 | range_to_mtrr(unsigned int reg, unsigned long range_startk, | |
251 | unsigned long range_sizek, unsigned char type) | |
252 | { | |
253 | if (!range_sizek || (reg >= num_var_ranges)) | |
254 | return reg; | |
255 | ||
256 | while (range_sizek) { | |
257 | unsigned long max_align, align; | |
258 | unsigned long sizek; | |
259 | ||
63f9600f | 260 | /* Compute the maximum size with which we can make a range: */ |
0d890355 | 261 | if (range_startk) |
1ba9a294 | 262 | max_align = __ffs(range_startk); |
0d890355 | 263 | else |
1ba9a294 | 264 | max_align = BITS_PER_LONG - 1; |
63f9600f | 265 | |
1ba9a294 | 266 | align = __fls(range_sizek); |
0d890355 YL |
267 | if (align > max_align) |
268 | align = max_align; | |
269 | ||
2da06af8 | 270 | sizek = 1UL << align; |
0d890355 YL |
271 | if (debug_print) { |
272 | char start_factor = 'K', size_factor = 'K'; | |
273 | unsigned long start_base, size_base; | |
274 | ||
63f9600f JSR |
275 | start_base = to_size_factor(range_startk, &start_factor); |
276 | size_base = to_size_factor(sizek, &size_factor); | |
0d890355 | 277 | |
63f9600f | 278 | Dprintk("Setting variable MTRR %d, " |
0d890355 YL |
279 | "base: %ld%cB, range: %ld%cB, type %s\n", |
280 | reg, start_base, start_factor, | |
281 | size_base, size_factor, | |
282 | (type == MTRR_TYPE_UNCACHABLE) ? "UC" : | |
283 | ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other") | |
284 | ); | |
285 | } | |
286 | save_var_mtrr(reg++, range_startk, sizek, type); | |
287 | range_startk += sizek; | |
288 | range_sizek -= sizek; | |
289 | if (reg >= num_var_ranges) | |
290 | break; | |
291 | } | |
292 | return reg; | |
293 | } | |
294 | ||
295 | static unsigned __init | |
296 | range_to_mtrr_with_hole(struct var_mtrr_state *state, unsigned long basek, | |
297 | unsigned long sizek) | |
298 | { | |
299 | unsigned long hole_basek, hole_sizek; | |
300 | unsigned long second_basek, second_sizek; | |
301 | unsigned long range0_basek, range0_sizek; | |
302 | unsigned long range_basek, range_sizek; | |
303 | unsigned long chunk_sizek; | |
304 | unsigned long gran_sizek; | |
305 | ||
306 | hole_basek = 0; | |
307 | hole_sizek = 0; | |
308 | second_basek = 0; | |
309 | second_sizek = 0; | |
310 | chunk_sizek = state->chunk_sizek; | |
311 | gran_sizek = state->gran_sizek; | |
312 | ||
63f9600f | 313 | /* Align with gran size, prevent small block used up MTRRs: */ |
0d890355 YL |
314 | range_basek = ALIGN(state->range_startk, gran_sizek); |
315 | if ((range_basek > basek) && basek) | |
316 | return second_sizek; | |
63f9600f | 317 | |
0d890355 YL |
318 | state->range_sizek -= (range_basek - state->range_startk); |
319 | range_sizek = ALIGN(state->range_sizek, gran_sizek); | |
320 | ||
321 | while (range_sizek > state->range_sizek) { | |
322 | range_sizek -= gran_sizek; | |
323 | if (!range_sizek) | |
324 | return 0; | |
325 | } | |
326 | state->range_sizek = range_sizek; | |
327 | ||
63f9600f | 328 | /* Try to append some small hole: */ |
0d890355 YL |
329 | range0_basek = state->range_startk; |
330 | range0_sizek = ALIGN(state->range_sizek, chunk_sizek); | |
331 | ||
63f9600f | 332 | /* No increase: */ |
0d890355 | 333 | if (range0_sizek == state->range_sizek) { |
63f9600f JSR |
334 | Dprintk("rangeX: %016lx - %016lx\n", |
335 | range0_basek<<10, | |
336 | (range0_basek + state->range_sizek)<<10); | |
0d890355 YL |
337 | state->reg = range_to_mtrr(state->reg, range0_basek, |
338 | state->range_sizek, MTRR_TYPE_WRBACK); | |
339 | return 0; | |
340 | } | |
341 | ||
63f9600f | 342 | /* Only cut back when it is not the last: */ |
0d890355 YL |
343 | if (sizek) { |
344 | while (range0_basek + range0_sizek > (basek + sizek)) { | |
345 | if (range0_sizek >= chunk_sizek) | |
346 | range0_sizek -= chunk_sizek; | |
347 | else | |
348 | range0_sizek = 0; | |
349 | ||
350 | if (!range0_sizek) | |
351 | break; | |
352 | } | |
353 | } | |
354 | ||
355 | second_try: | |
356 | range_basek = range0_basek + range0_sizek; | |
357 | ||
63f9600f | 358 | /* One hole in the middle: */ |
0d890355 YL |
359 | if (range_basek > basek && range_basek <= (basek + sizek)) |
360 | second_sizek = range_basek - basek; | |
361 | ||
362 | if (range0_sizek > state->range_sizek) { | |
363 | ||
63f9600f | 364 | /* One hole in middle or at the end: */ |
0d890355 YL |
365 | hole_sizek = range0_sizek - state->range_sizek - second_sizek; |
366 | ||
63f9600f | 367 | /* Hole size should be less than half of range0 size: */ |
0d890355 YL |
368 | if (hole_sizek >= (range0_sizek >> 1) && |
369 | range0_sizek >= chunk_sizek) { | |
370 | range0_sizek -= chunk_sizek; | |
371 | second_sizek = 0; | |
372 | hole_sizek = 0; | |
373 | ||
374 | goto second_try; | |
375 | } | |
376 | } | |
377 | ||
378 | if (range0_sizek) { | |
63f9600f JSR |
379 | Dprintk("range0: %016lx - %016lx\n", |
380 | range0_basek<<10, | |
381 | (range0_basek + range0_sizek)<<10); | |
0d890355 YL |
382 | state->reg = range_to_mtrr(state->reg, range0_basek, |
383 | range0_sizek, MTRR_TYPE_WRBACK); | |
384 | } | |
385 | ||
386 | if (range0_sizek < state->range_sizek) { | |
63f9600f | 387 | /* Need to handle left over range: */ |
0d890355 YL |
388 | range_sizek = state->range_sizek - range0_sizek; |
389 | ||
63f9600f JSR |
390 | Dprintk("range: %016lx - %016lx\n", |
391 | range_basek<<10, | |
392 | (range_basek + range_sizek)<<10); | |
393 | ||
0d890355 YL |
394 | state->reg = range_to_mtrr(state->reg, range_basek, |
395 | range_sizek, MTRR_TYPE_WRBACK); | |
396 | } | |
397 | ||
398 | if (hole_sizek) { | |
399 | hole_basek = range_basek - hole_sizek - second_sizek; | |
63f9600f JSR |
400 | Dprintk("hole: %016lx - %016lx\n", |
401 | hole_basek<<10, | |
402 | (hole_basek + hole_sizek)<<10); | |
0d890355 YL |
403 | state->reg = range_to_mtrr(state->reg, hole_basek, |
404 | hole_sizek, MTRR_TYPE_UNCACHABLE); | |
405 | } | |
406 | ||
407 | return second_sizek; | |
408 | } | |
409 | ||
410 | static void __init | |
411 | set_var_mtrr_range(struct var_mtrr_state *state, unsigned long base_pfn, | |
412 | unsigned long size_pfn) | |
413 | { | |
414 | unsigned long basek, sizek; | |
415 | unsigned long second_sizek = 0; | |
416 | ||
417 | if (state->reg >= num_var_ranges) | |
418 | return; | |
419 | ||
420 | basek = base_pfn << (PAGE_SHIFT - 10); | |
421 | sizek = size_pfn << (PAGE_SHIFT - 10); | |
422 | ||
63f9600f | 423 | /* See if I can merge with the last range: */ |
0d890355 YL |
424 | if ((basek <= 1024) || |
425 | (state->range_startk + state->range_sizek == basek)) { | |
426 | unsigned long endk = basek + sizek; | |
427 | state->range_sizek = endk - state->range_startk; | |
428 | return; | |
429 | } | |
63f9600f | 430 | /* Write the range mtrrs: */ |
0d890355 YL |
431 | if (state->range_sizek != 0) |
432 | second_sizek = range_to_mtrr_with_hole(state, basek, sizek); | |
433 | ||
63f9600f | 434 | /* Allocate an msr: */ |
0d890355 YL |
435 | state->range_startk = basek + second_sizek; |
436 | state->range_sizek = sizek - second_sizek; | |
437 | } | |
438 | ||
63f9600f | 439 | /* Mininum size of mtrr block that can take hole: */ |
0d890355 YL |
440 | static u64 mtrr_chunk_size __initdata = (256ULL<<20); |
441 | ||
442 | static int __init parse_mtrr_chunk_size_opt(char *p) | |
443 | { | |
444 | if (!p) | |
445 | return -EINVAL; | |
446 | mtrr_chunk_size = memparse(p, &p); | |
447 | return 0; | |
448 | } | |
449 | early_param("mtrr_chunk_size", parse_mtrr_chunk_size_opt); | |
450 | ||
63f9600f | 451 | /* Granularity of mtrr of block: */ |
0d890355 YL |
452 | static u64 mtrr_gran_size __initdata; |
453 | ||
454 | static int __init parse_mtrr_gran_size_opt(char *p) | |
455 | { | |
456 | if (!p) | |
457 | return -EINVAL; | |
458 | mtrr_gran_size = memparse(p, &p); | |
459 | return 0; | |
460 | } | |
461 | early_param("mtrr_gran_size", parse_mtrr_gran_size_opt); | |
462 | ||
63f9600f | 463 | static unsigned long nr_mtrr_spare_reg __initdata = |
0d890355 YL |
464 | CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT; |
465 | ||
466 | static int __init parse_mtrr_spare_reg(char *arg) | |
467 | { | |
468 | if (arg) | |
469 | nr_mtrr_spare_reg = simple_strtoul(arg, NULL, 0); | |
470 | return 0; | |
471 | } | |
0d890355 YL |
472 | early_param("mtrr_spare_reg_nr", parse_mtrr_spare_reg); |
473 | ||
474 | static int __init | |
27811d8c | 475 | x86_setup_var_mtrrs(struct range *range, int nr_range, |
0d890355 YL |
476 | u64 chunk_size, u64 gran_size) |
477 | { | |
478 | struct var_mtrr_state var_state; | |
0d890355 | 479 | int num_reg; |
63f9600f | 480 | int i; |
0d890355 YL |
481 | |
482 | var_state.range_startk = 0; | |
483 | var_state.range_sizek = 0; | |
484 | var_state.reg = 0; | |
485 | var_state.chunk_sizek = chunk_size >> 10; | |
486 | var_state.gran_sizek = gran_size >> 10; | |
487 | ||
488 | memset(range_state, 0, sizeof(range_state)); | |
489 | ||
63f9600f JSR |
490 | /* Write the range: */ |
491 | for (i = 0; i < nr_range; i++) { | |
0d890355 | 492 | set_var_mtrr_range(&var_state, range[i].start, |
e9a0064a | 493 | range[i].end - range[i].start); |
63f9600f | 494 | } |
0d890355 | 495 | |
63f9600f | 496 | /* Write the last range: */ |
0d890355 YL |
497 | if (var_state.range_sizek != 0) |
498 | range_to_mtrr_with_hole(&var_state, 0, 0); | |
499 | ||
500 | num_reg = var_state.reg; | |
63f9600f | 501 | /* Clear out the extra MTRR's: */ |
0d890355 YL |
502 | while (var_state.reg < num_var_ranges) { |
503 | save_var_mtrr(var_state.reg, 0, 0, 0); | |
504 | var_state.reg++; | |
505 | } | |
506 | ||
507 | return num_reg; | |
508 | } | |
509 | ||
510 | struct mtrr_cleanup_result { | |
63f9600f JSR |
511 | unsigned long gran_sizek; |
512 | unsigned long chunk_sizek; | |
513 | unsigned long lose_cover_sizek; | |
514 | unsigned int num_reg; | |
515 | int bad; | |
0d890355 YL |
516 | }; |
517 | ||
518 | /* | |
519 | * gran_size: 64K, 128K, 256K, 512K, 1M, 2M, ..., 2G | |
520 | * chunk size: gran_size, ..., 2G | |
521 | * so we need (1+16)*8 | |
522 | */ | |
523 | #define NUM_RESULT 136 | |
524 | #define PSHIFT (PAGE_SHIFT - 10) | |
525 | ||
526 | static struct mtrr_cleanup_result __initdata result[NUM_RESULT]; | |
527 | static unsigned long __initdata min_loss_pfn[RANGE_NUM]; | |
528 | ||
529 | static void __init print_out_mtrr_range_state(void) | |
530 | { | |
0d890355 YL |
531 | char start_factor = 'K', size_factor = 'K'; |
532 | unsigned long start_base, size_base; | |
533 | mtrr_type type; | |
63f9600f | 534 | int i; |
0d890355 YL |
535 | |
536 | for (i = 0; i < num_var_ranges; i++) { | |
537 | ||
538 | size_base = range_state[i].size_pfn << (PAGE_SHIFT - 10); | |
539 | if (!size_base) | |
540 | continue; | |
541 | ||
542 | size_base = to_size_factor(size_base, &size_factor), | |
543 | start_base = range_state[i].base_pfn << (PAGE_SHIFT - 10); | |
544 | start_base = to_size_factor(start_base, &start_factor), | |
545 | type = range_state[i].type; | |
546 | ||
1b74dde7 | 547 | pr_debug("reg %d, base: %ld%cB, range: %ld%cB, type %s\n", |
0d890355 YL |
548 | i, start_base, start_factor, |
549 | size_base, size_factor, | |
550 | (type == MTRR_TYPE_UNCACHABLE) ? "UC" : | |
551 | ((type == MTRR_TYPE_WRPROT) ? "WP" : | |
552 | ((type == MTRR_TYPE_WRBACK) ? "WB" : "Other")) | |
553 | ); | |
554 | } | |
555 | } | |
556 | ||
557 | static int __init mtrr_need_cleanup(void) | |
558 | { | |
559 | int i; | |
560 | mtrr_type type; | |
561 | unsigned long size; | |
63f9600f | 562 | /* Extra one for all 0: */ |
0d890355 YL |
563 | int num[MTRR_NUM_TYPES + 1]; |
564 | ||
63f9600f | 565 | /* Check entries number: */ |
0d890355 YL |
566 | memset(num, 0, sizeof(num)); |
567 | for (i = 0; i < num_var_ranges; i++) { | |
568 | type = range_state[i].type; | |
569 | size = range_state[i].size_pfn; | |
570 | if (type >= MTRR_NUM_TYPES) | |
571 | continue; | |
572 | if (!size) | |
573 | type = MTRR_NUM_TYPES; | |
0d890355 YL |
574 | num[type]++; |
575 | } | |
576 | ||
63f9600f | 577 | /* Check if we got UC entries: */ |
0d890355 YL |
578 | if (!num[MTRR_TYPE_UNCACHABLE]) |
579 | return 0; | |
580 | ||
63f9600f | 581 | /* Check if we only had WB and UC */ |
0d890355 | 582 | if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != |
63f9600f | 583 | num_var_ranges - num[MTRR_NUM_TYPES]) |
0d890355 YL |
584 | return 0; |
585 | ||
586 | return 1; | |
587 | } | |
588 | ||
589 | static unsigned long __initdata range_sums; | |
63f9600f JSR |
590 | |
591 | static void __init | |
592 | mtrr_calc_range_state(u64 chunk_size, u64 gran_size, | |
593 | unsigned long x_remove_base, | |
594 | unsigned long x_remove_size, int i) | |
0d890355 | 595 | { |
c332813b RV |
596 | /* |
597 | * range_new should really be an automatic variable, but | |
598 | * putting 4096 bytes on the stack is frowned upon, to put it | |
599 | * mildly. It is safe to make it a static __initdata variable, | |
600 | * since mtrr_calc_range_state is only called during init and | |
601 | * there's no way it will call itself recursively. | |
602 | */ | |
603 | static struct range range_new[RANGE_NUM] __initdata; | |
0d890355 | 604 | unsigned long range_sums_new; |
c332813b | 605 | int nr_range_new; |
63f9600f | 606 | int num_reg; |
0d890355 | 607 | |
63f9600f JSR |
608 | /* Convert ranges to var ranges state: */ |
609 | num_reg = x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); | |
0d890355 | 610 | |
63f9600f | 611 | /* We got new setting in range_state, check it: */ |
0d890355 YL |
612 | memset(range_new, 0, sizeof(range_new)); |
613 | nr_range_new = x86_get_mtrr_mem_range(range_new, 0, | |
63f9600f | 614 | x_remove_base, x_remove_size); |
0d890355 YL |
615 | range_sums_new = sum_ranges(range_new, nr_range_new); |
616 | ||
617 | result[i].chunk_sizek = chunk_size >> 10; | |
618 | result[i].gran_sizek = gran_size >> 10; | |
619 | result[i].num_reg = num_reg; | |
63f9600f | 620 | |
0d890355 | 621 | if (range_sums < range_sums_new) { |
63f9600f | 622 | result[i].lose_cover_sizek = (range_sums_new - range_sums) << PSHIFT; |
0d890355 | 623 | result[i].bad = 1; |
63f9600f JSR |
624 | } else { |
625 | result[i].lose_cover_sizek = (range_sums - range_sums_new) << PSHIFT; | |
626 | } | |
0d890355 | 627 | |
63f9600f | 628 | /* Double check it: */ |
0d890355 | 629 | if (!result[i].bad && !result[i].lose_cover_sizek) { |
63f9600f JSR |
630 | if (nr_range_new != nr_range || memcmp(range, range_new, sizeof(range))) |
631 | result[i].bad = 1; | |
0d890355 YL |
632 | } |
633 | ||
63f9600f JSR |
634 | if (!result[i].bad && (range_sums - range_sums_new < min_loss_pfn[num_reg])) |
635 | min_loss_pfn[num_reg] = range_sums - range_sums_new; | |
0d890355 YL |
636 | } |
637 | ||
638 | static void __init mtrr_print_out_one_result(int i) | |
639 | { | |
0d890355 | 640 | unsigned long gran_base, chunk_base, lose_base; |
63f9600f | 641 | char gran_factor, chunk_factor, lose_factor; |
0d890355 | 642 | |
b2691085 JP |
643 | gran_base = to_size_factor(result[i].gran_sizek, &gran_factor); |
644 | chunk_base = to_size_factor(result[i].chunk_sizek, &chunk_factor); | |
645 | lose_base = to_size_factor(result[i].lose_cover_sizek, &lose_factor); | |
63f9600f JSR |
646 | |
647 | pr_info("%sgran_size: %ld%c \tchunk_size: %ld%c \t", | |
648 | result[i].bad ? "*BAD*" : " ", | |
649 | gran_base, gran_factor, chunk_base, chunk_factor); | |
650 | pr_cont("num_reg: %d \tlose cover RAM: %s%ld%c\n", | |
651 | result[i].num_reg, result[i].bad ? "-" : "", | |
652 | lose_base, lose_factor); | |
0d890355 YL |
653 | } |
654 | ||
655 | static int __init mtrr_search_optimal_index(void) | |
656 | { | |
0d890355 YL |
657 | int num_reg_good; |
658 | int index_good; | |
63f9600f | 659 | int i; |
0d890355 YL |
660 | |
661 | if (nr_mtrr_spare_reg >= num_var_ranges) | |
662 | nr_mtrr_spare_reg = num_var_ranges - 1; | |
63f9600f | 663 | |
0d890355 YL |
664 | num_reg_good = -1; |
665 | for (i = num_var_ranges - nr_mtrr_spare_reg; i > 0; i--) { | |
666 | if (!min_loss_pfn[i]) | |
667 | num_reg_good = i; | |
668 | } | |
669 | ||
670 | index_good = -1; | |
671 | if (num_reg_good != -1) { | |
672 | for (i = 0; i < NUM_RESULT; i++) { | |
673 | if (!result[i].bad && | |
674 | result[i].num_reg == num_reg_good && | |
675 | !result[i].lose_cover_sizek) { | |
676 | index_good = i; | |
677 | break; | |
678 | } | |
679 | } | |
680 | } | |
681 | ||
682 | return index_good; | |
683 | } | |
684 | ||
0d890355 YL |
685 | int __init mtrr_cleanup(unsigned address_bits) |
686 | { | |
63f9600f | 687 | unsigned long x_remove_base, x_remove_size; |
0d890355 | 688 | unsigned long base, size, def, dummy; |
0d890355 | 689 | u64 chunk_size, gran_size; |
63f9600f | 690 | mtrr_type type; |
0d890355 YL |
691 | int index_good; |
692 | int i; | |
693 | ||
694 | if (!is_cpu(INTEL) || enable_mtrr_cleanup < 1) | |
695 | return 0; | |
63f9600f | 696 | |
52650257 | 697 | rdmsr(MSR_MTRRdefType, def, dummy); |
0d890355 YL |
698 | def &= 0xff; |
699 | if (def != MTRR_TYPE_UNCACHABLE) | |
700 | return 0; | |
701 | ||
63f9600f | 702 | /* Get it and store it aside: */ |
0d890355 YL |
703 | memset(range_state, 0, sizeof(range_state)); |
704 | for (i = 0; i < num_var_ranges; i++) { | |
705 | mtrr_if->get(i, &base, &size, &type); | |
706 | range_state[i].base_pfn = base; | |
707 | range_state[i].size_pfn = size; | |
708 | range_state[i].type = type; | |
709 | } | |
710 | ||
63f9600f | 711 | /* Check if we need handle it and can handle it: */ |
0d890355 YL |
712 | if (!mtrr_need_cleanup()) |
713 | return 0; | |
714 | ||
63f9600f | 715 | /* Print original var MTRRs at first, for debugging: */ |
1b74dde7 | 716 | pr_debug("original variable MTRRs\n"); |
0d890355 YL |
717 | print_out_mtrr_range_state(); |
718 | ||
719 | memset(range, 0, sizeof(range)); | |
63f9600f JSR |
720 | x_remove_size = 0; |
721 | x_remove_base = 1 << (32 - PAGE_SHIFT); | |
0d890355 | 722 | if (mtrr_tom2) |
63f9600f JSR |
723 | x_remove_size = (mtrr_tom2 >> PAGE_SHIFT) - x_remove_base; |
724 | ||
0d890355 | 725 | /* |
63f9600f JSR |
726 | * [0, 1M) should always be covered by var mtrr with WB |
727 | * and fixed mtrrs should take effect before var mtrr for it: | |
0d890355 | 728 | */ |
d8d386c1 | 729 | nr_range = add_range_with_merge(range, RANGE_NUM, 0, 0, |
e9a0064a | 730 | 1ULL<<(20 - PAGE_SHIFT)); |
d8d386c1 YL |
731 | /* add from var mtrr at last */ |
732 | nr_range = x86_get_mtrr_mem_range(range, nr_range, | |
733 | x_remove_base, x_remove_size); | |
0d890355 YL |
734 | |
735 | range_sums = sum_ranges(range, nr_range); | |
1b74dde7 | 736 | pr_info("total RAM covered: %ldM\n", |
0d890355 YL |
737 | range_sums >> (20 - PAGE_SHIFT)); |
738 | ||
739 | if (mtrr_chunk_size && mtrr_gran_size) { | |
740 | i = 0; | |
741 | mtrr_calc_range_state(mtrr_chunk_size, mtrr_gran_size, | |
63f9600f | 742 | x_remove_base, x_remove_size, i); |
0d890355 YL |
743 | |
744 | mtrr_print_out_one_result(i); | |
745 | ||
746 | if (!result[i].bad) { | |
747 | set_var_mtrr_all(address_bits); | |
1b74dde7 | 748 | pr_debug("New variable MTRRs\n"); |
0d890355 YL |
749 | print_out_mtrr_range_state(); |
750 | return 1; | |
751 | } | |
1b74dde7 | 752 | pr_info("invalid mtrr_gran_size or mtrr_chunk_size, will find optimal one\n"); |
0d890355 YL |
753 | } |
754 | ||
755 | i = 0; | |
756 | memset(min_loss_pfn, 0xff, sizeof(min_loss_pfn)); | |
757 | memset(result, 0, sizeof(result)); | |
758 | for (gran_size = (1ULL<<16); gran_size < (1ULL<<32); gran_size <<= 1) { | |
759 | ||
760 | for (chunk_size = gran_size; chunk_size < (1ULL<<32); | |
761 | chunk_size <<= 1) { | |
762 | ||
763 | if (i >= NUM_RESULT) | |
764 | continue; | |
765 | ||
766 | mtrr_calc_range_state(chunk_size, gran_size, | |
63f9600f | 767 | x_remove_base, x_remove_size, i); |
0d890355 YL |
768 | if (debug_print) { |
769 | mtrr_print_out_one_result(i); | |
1b74dde7 | 770 | pr_info("\n"); |
0d890355 YL |
771 | } |
772 | ||
773 | i++; | |
774 | } | |
775 | } | |
776 | ||
63f9600f | 777 | /* Try to find the optimal index: */ |
0d890355 YL |
778 | index_good = mtrr_search_optimal_index(); |
779 | ||
780 | if (index_good != -1) { | |
1b74dde7 | 781 | pr_info("Found optimal setting for mtrr clean up\n"); |
0d890355 YL |
782 | i = index_good; |
783 | mtrr_print_out_one_result(i); | |
784 | ||
63f9600f | 785 | /* Convert ranges to var ranges state: */ |
0d890355 YL |
786 | chunk_size = result[i].chunk_sizek; |
787 | chunk_size <<= 10; | |
788 | gran_size = result[i].gran_sizek; | |
789 | gran_size <<= 10; | |
790 | x86_setup_var_mtrrs(range, nr_range, chunk_size, gran_size); | |
791 | set_var_mtrr_all(address_bits); | |
1b74dde7 | 792 | pr_debug("New variable MTRRs\n"); |
0d890355 YL |
793 | print_out_mtrr_range_state(); |
794 | return 1; | |
795 | } else { | |
796 | /* print out all */ | |
797 | for (i = 0; i < NUM_RESULT; i++) | |
798 | mtrr_print_out_one_result(i); | |
799 | } | |
800 | ||
1b74dde7 CY |
801 | pr_info("mtrr_cleanup: can not find optimal value\n"); |
802 | pr_info("please specify mtrr_gran_size/mtrr_chunk_size\n"); | |
0d890355 YL |
803 | |
804 | return 0; | |
805 | } | |
806 | #else | |
807 | int __init mtrr_cleanup(unsigned address_bits) | |
808 | { | |
809 | return 0; | |
810 | } | |
811 | #endif | |
812 | ||
813 | static int disable_mtrr_trim; | |
814 | ||
815 | static int __init disable_mtrr_trim_setup(char *str) | |
816 | { | |
817 | disable_mtrr_trim = 1; | |
818 | return 0; | |
819 | } | |
820 | early_param("disable_mtrr_trim", disable_mtrr_trim_setup); | |
821 | ||
822 | /* | |
823 | * Newer AMD K8s and later CPUs have a special magic MSR way to force WB | |
824 | * for memory >4GB. Check for that here. | |
825 | * Note this won't check if the MTRRs < 4GB where the magic bit doesn't | |
826 | * apply to are wrong, but so far we don't know of any such case in the wild. | |
827 | */ | |
63f9600f JSR |
828 | #define Tom2Enabled (1U << 21) |
829 | #define Tom2ForceMemTypeWB (1U << 22) | |
0d890355 YL |
830 | |
831 | int __init amd_special_default_mtrr(void) | |
832 | { | |
833 | u32 l, h; | |
834 | ||
835 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) | |
836 | return 0; | |
3fdbf004 | 837 | if (boot_cpu_data.x86 < 0xf) |
0d890355 | 838 | return 0; |
63f9600f | 839 | /* In case some hypervisor doesn't pass SYSCFG through: */ |
0d890355 YL |
840 | if (rdmsr_safe(MSR_K8_SYSCFG, &l, &h) < 0) |
841 | return 0; | |
842 | /* | |
843 | * Memory between 4GB and top of mem is forced WB by this magic bit. | |
844 | * Reserved before K8RevF, but should be zero there. | |
845 | */ | |
846 | if ((l & (Tom2Enabled | Tom2ForceMemTypeWB)) == | |
847 | (Tom2Enabled | Tom2ForceMemTypeWB)) | |
848 | return 1; | |
849 | return 0; | |
850 | } | |
851 | ||
63f9600f JSR |
852 | static u64 __init |
853 | real_trim_memory(unsigned long start_pfn, unsigned long limit_pfn) | |
0d890355 YL |
854 | { |
855 | u64 trim_start, trim_size; | |
63f9600f | 856 | |
0d890355 YL |
857 | trim_start = start_pfn; |
858 | trim_start <<= PAGE_SHIFT; | |
63f9600f | 859 | |
0d890355 YL |
860 | trim_size = limit_pfn; |
861 | trim_size <<= PAGE_SHIFT; | |
862 | trim_size -= trim_start; | |
863 | ||
63f9600f | 864 | return e820_update_range(trim_start, trim_size, E820_RAM, E820_RESERVED); |
0d890355 | 865 | } |
63f9600f | 866 | |
0d890355 YL |
867 | /** |
868 | * mtrr_trim_uncached_memory - trim RAM not covered by MTRRs | |
869 | * @end_pfn: ending page frame number | |
870 | * | |
871 | * Some buggy BIOSes don't setup the MTRRs properly for systems with certain | |
872 | * memory configurations. This routine checks that the highest MTRR matches | |
873 | * the end of memory, to make sure the MTRRs having a write back type cover | |
63f9600f | 874 | * all of the memory the kernel is intending to use. If not, it'll trim any |
0d890355 YL |
875 | * memory off the end by adjusting end_pfn, removing it from the kernel's |
876 | * allocation pools, warning the user with an obnoxious message. | |
877 | */ | |
878 | int __init mtrr_trim_uncached_memory(unsigned long end_pfn) | |
879 | { | |
880 | unsigned long i, base, size, highest_pfn = 0, def, dummy; | |
881 | mtrr_type type; | |
882 | u64 total_trim_size; | |
0d890355 YL |
883 | /* extra one for all 0 */ |
884 | int num[MTRR_NUM_TYPES + 1]; | |
63f9600f | 885 | |
0d890355 YL |
886 | /* |
887 | * Make sure we only trim uncachable memory on machines that | |
888 | * support the Intel MTRR architecture: | |
889 | */ | |
890 | if (!is_cpu(INTEL) || disable_mtrr_trim) | |
891 | return 0; | |
63f9600f | 892 | |
52650257 | 893 | rdmsr(MSR_MTRRdefType, def, dummy); |
0d890355 YL |
894 | def &= 0xff; |
895 | if (def != MTRR_TYPE_UNCACHABLE) | |
896 | return 0; | |
897 | ||
63f9600f | 898 | /* Get it and store it aside: */ |
0d890355 YL |
899 | memset(range_state, 0, sizeof(range_state)); |
900 | for (i = 0; i < num_var_ranges; i++) { | |
901 | mtrr_if->get(i, &base, &size, &type); | |
902 | range_state[i].base_pfn = base; | |
903 | range_state[i].size_pfn = size; | |
904 | range_state[i].type = type; | |
905 | } | |
906 | ||
63f9600f | 907 | /* Find highest cached pfn: */ |
0d890355 YL |
908 | for (i = 0; i < num_var_ranges; i++) { |
909 | type = range_state[i].type; | |
910 | if (type != MTRR_TYPE_WRBACK) | |
911 | continue; | |
912 | base = range_state[i].base_pfn; | |
913 | size = range_state[i].size_pfn; | |
914 | if (highest_pfn < base + size) | |
915 | highest_pfn = base + size; | |
916 | } | |
917 | ||
63f9600f | 918 | /* kvm/qemu doesn't have mtrr set right, don't trim them all: */ |
0d890355 | 919 | if (!highest_pfn) { |
1b74dde7 | 920 | pr_info("CPU MTRRs all blank - virtualized system.\n"); |
0d890355 YL |
921 | return 0; |
922 | } | |
923 | ||
63f9600f | 924 | /* Check entries number: */ |
0d890355 YL |
925 | memset(num, 0, sizeof(num)); |
926 | for (i = 0; i < num_var_ranges; i++) { | |
927 | type = range_state[i].type; | |
928 | if (type >= MTRR_NUM_TYPES) | |
929 | continue; | |
930 | size = range_state[i].size_pfn; | |
931 | if (!size) | |
932 | type = MTRR_NUM_TYPES; | |
933 | num[type]++; | |
934 | } | |
935 | ||
63f9600f | 936 | /* No entry for WB? */ |
0d890355 YL |
937 | if (!num[MTRR_TYPE_WRBACK]) |
938 | return 0; | |
939 | ||
63f9600f | 940 | /* Check if we only had WB and UC: */ |
0d890355 YL |
941 | if (num[MTRR_TYPE_WRBACK] + num[MTRR_TYPE_UNCACHABLE] != |
942 | num_var_ranges - num[MTRR_NUM_TYPES]) | |
943 | return 0; | |
944 | ||
945 | memset(range, 0, sizeof(range)); | |
946 | nr_range = 0; | |
947 | if (mtrr_tom2) { | |
948 | range[nr_range].start = (1ULL<<(32 - PAGE_SHIFT)); | |
e9a0064a YL |
949 | range[nr_range].end = mtrr_tom2 >> PAGE_SHIFT; |
950 | if (highest_pfn < range[nr_range].end) | |
951 | highest_pfn = range[nr_range].end; | |
0d890355 YL |
952 | nr_range++; |
953 | } | |
954 | nr_range = x86_get_mtrr_mem_range(range, nr_range, 0, 0); | |
955 | ||
63f9600f | 956 | /* Check the head: */ |
0d890355 | 957 | total_trim_size = 0; |
0d890355 YL |
958 | if (range[0].start) |
959 | total_trim_size += real_trim_memory(0, range[0].start); | |
63f9600f JSR |
960 | |
961 | /* Check the holes: */ | |
0d890355 | 962 | for (i = 0; i < nr_range - 1; i++) { |
e9a0064a YL |
963 | if (range[i].end < range[i+1].start) |
964 | total_trim_size += real_trim_memory(range[i].end, | |
0d890355 YL |
965 | range[i+1].start); |
966 | } | |
63f9600f JSR |
967 | |
968 | /* Check the top: */ | |
0d890355 | 969 | i = nr_range - 1; |
e9a0064a YL |
970 | if (range[i].end < end_pfn) |
971 | total_trim_size += real_trim_memory(range[i].end, | |
0d890355 YL |
972 | end_pfn); |
973 | ||
974 | if (total_trim_size) { | |
1b74dde7 CY |
975 | pr_warn("WARNING: BIOS bug: CPU MTRRs don't cover all of memory, losing %lluMB of RAM.\n", |
976 | total_trim_size >> 20); | |
0d890355 YL |
977 | |
978 | if (!changed_by_mtrr_cleanup) | |
979 | WARN_ON(1); | |
980 | ||
63f9600f | 981 | pr_info("update e820 for mtrr\n"); |
0d890355 YL |
982 | update_e820(); |
983 | ||
984 | return 1; | |
985 | } | |
986 | ||
987 | return 0; | |
988 | } |