Commit | Line | Data |
---|---|---|
7c8c6b97 PM |
1 | /* |
2 | * Procedures for maintaining information about logical memory blocks. | |
3 | * | |
4 | * Peter Bergner, IBM Corp. June 2001. | |
5 | * Copyright (C) 2001 Peter Bergner. | |
d9b2b2a2 | 6 | * |
7c8c6b97 PM |
7 | * This program is free software; you can redistribute it and/or |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
12 | ||
7c8c6b97 PM |
13 | #include <linux/kernel.h> |
14 | #include <linux/init.h> | |
15 | #include <linux/bitops.h> | |
d9b2b2a2 | 16 | #include <linux/lmb.h> |
7c8c6b97 | 17 | |
3b9331da ME |
18 | #define LMB_ALLOC_ANYWHERE 0 |
19 | ||
eb481899 ME |
20 | struct lmb lmb; |
21 | ||
faa6cfde DM |
22 | static int lmb_debug; |
23 | ||
24 | static int __init early_lmb(char *p) | |
25 | { | |
26 | if (p && strstr(p, "debug")) | |
27 | lmb_debug = 1; | |
28 | return 0; | |
29 | } | |
30 | early_param("lmb", early_lmb); | |
31 | ||
7c8c6b97 PM |
32 | void lmb_dump_all(void) |
33 | { | |
7c8c6b97 PM |
34 | unsigned long i; |
35 | ||
faa6cfde DM |
36 | if (!lmb_debug) |
37 | return; | |
38 | ||
39 | pr_info("lmb_dump_all:\n"); | |
40 | pr_info(" memory.cnt = 0x%lx\n", lmb.memory.cnt); | |
41 | pr_info(" memory.size = 0x%llx\n", | |
e5f27095 | 42 | (unsigned long long)lmb.memory.size); |
7c8c6b97 | 43 | for (i=0; i < lmb.memory.cnt ;i++) { |
faa6cfde | 44 | pr_info(" memory.region[0x%lx].base = 0x%llx\n", |
e5f27095 | 45 | i, (unsigned long long)lmb.memory.region[i].base); |
faa6cfde | 46 | pr_info(" .size = 0x%llx\n", |
e5f27095 | 47 | (unsigned long long)lmb.memory.region[i].size); |
7c8c6b97 PM |
48 | } |
49 | ||
faa6cfde DM |
50 | pr_info(" reserved.cnt = 0x%lx\n", lmb.reserved.cnt); |
51 | pr_info(" reserved.size = 0x%lx\n", lmb.reserved.size); | |
7c8c6b97 | 52 | for (i=0; i < lmb.reserved.cnt ;i++) { |
faa6cfde | 53 | pr_info(" reserved.region[0x%lx].base = 0x%llx\n", |
e5f27095 | 54 | i, (unsigned long long)lmb.reserved.region[i].base); |
faa6cfde | 55 | pr_info(" .size = 0x%llx\n", |
e5f27095 | 56 | (unsigned long long)lmb.reserved.region[i].size); |
7c8c6b97 | 57 | } |
7c8c6b97 PM |
58 | } |
59 | ||
98d5c21c BP |
60 | static unsigned long lmb_addrs_overlap(u64 base1, u64 size1, u64 base2, |
61 | u64 size2) | |
7c8c6b97 | 62 | { |
300613e5 | 63 | return ((base1 < (base2 + size2)) && (base2 < (base1 + size1))); |
7c8c6b97 PM |
64 | } |
65 | ||
98d5c21c | 66 | static long lmb_addrs_adjacent(u64 base1, u64 size1, u64 base2, u64 size2) |
7c8c6b97 PM |
67 | { |
68 | if (base2 == base1 + size1) | |
69 | return 1; | |
70 | else if (base1 == base2 + size2) | |
71 | return -1; | |
72 | ||
73 | return 0; | |
74 | } | |
75 | ||
98d5c21c | 76 | static long lmb_regions_adjacent(struct lmb_region *rgn, |
7c8c6b97 PM |
77 | unsigned long r1, unsigned long r2) |
78 | { | |
e5f27095 BB |
79 | u64 base1 = rgn->region[r1].base; |
80 | u64 size1 = rgn->region[r1].size; | |
81 | u64 base2 = rgn->region[r2].base; | |
82 | u64 size2 = rgn->region[r2].size; | |
7c8c6b97 PM |
83 | |
84 | return lmb_addrs_adjacent(base1, size1, base2, size2); | |
85 | } | |
86 | ||
98d5c21c | 87 | static void lmb_remove_region(struct lmb_region *rgn, unsigned long r) |
7c8c6b97 PM |
88 | { |
89 | unsigned long i; | |
90 | ||
2babf5c2 ME |
91 | for (i = r; i < rgn->cnt - 1; i++) { |
92 | rgn->region[i].base = rgn->region[i + 1].base; | |
93 | rgn->region[i].size = rgn->region[i + 1].size; | |
7c8c6b97 PM |
94 | } |
95 | rgn->cnt--; | |
96 | } | |
97 | ||
2babf5c2 | 98 | /* Assumption: base addr of region 1 < base addr of region 2 */ |
98d5c21c | 99 | static void lmb_coalesce_regions(struct lmb_region *rgn, |
2babf5c2 ME |
100 | unsigned long r1, unsigned long r2) |
101 | { | |
102 | rgn->region[r1].size += rgn->region[r2].size; | |
103 | lmb_remove_region(rgn, r2); | |
104 | } | |
105 | ||
7c8c6b97 PM |
106 | void __init lmb_init(void) |
107 | { | |
108 | /* Create a dummy zero size LMB which will get coalesced away later. | |
109 | * This simplifies the lmb_add() code below... | |
110 | */ | |
111 | lmb.memory.region[0].base = 0; | |
112 | lmb.memory.region[0].size = 0; | |
113 | lmb.memory.cnt = 1; | |
114 | ||
115 | /* Ditto. */ | |
116 | lmb.reserved.region[0].base = 0; | |
117 | lmb.reserved.region[0].size = 0; | |
118 | lmb.reserved.cnt = 1; | |
119 | } | |
120 | ||
7c8c6b97 PM |
121 | void __init lmb_analyze(void) |
122 | { | |
123 | int i; | |
124 | ||
125 | lmb.memory.size = 0; | |
126 | ||
127 | for (i = 0; i < lmb.memory.cnt; i++) | |
128 | lmb.memory.size += lmb.memory.region[i].size; | |
129 | } | |
130 | ||
98d5c21c | 131 | static long lmb_add_region(struct lmb_region *rgn, u64 base, u64 size) |
7c8c6b97 | 132 | { |
56d6d1a7 MA |
133 | unsigned long coalesced = 0; |
134 | long adjacent, i; | |
7c8c6b97 | 135 | |
27e6672b KG |
136 | if ((rgn->cnt == 1) && (rgn->region[0].size == 0)) { |
137 | rgn->region[0].base = base; | |
138 | rgn->region[0].size = size; | |
139 | return 0; | |
140 | } | |
141 | ||
7c8c6b97 | 142 | /* First try and coalesce this LMB with another. */ |
300613e5 | 143 | for (i = 0; i < rgn->cnt; i++) { |
e5f27095 BB |
144 | u64 rgnbase = rgn->region[i].base; |
145 | u64 rgnsize = rgn->region[i].size; | |
7c8c6b97 | 146 | |
eb6de286 DG |
147 | if ((rgnbase == base) && (rgnsize == size)) |
148 | /* Already have this region, so we're done */ | |
149 | return 0; | |
150 | ||
300613e5 PM |
151 | adjacent = lmb_addrs_adjacent(base, size, rgnbase, rgnsize); |
152 | if (adjacent > 0) { | |
7c8c6b97 PM |
153 | rgn->region[i].base -= size; |
154 | rgn->region[i].size += size; | |
155 | coalesced++; | |
156 | break; | |
300613e5 | 157 | } else if (adjacent < 0) { |
7c8c6b97 PM |
158 | rgn->region[i].size += size; |
159 | coalesced++; | |
160 | break; | |
161 | } | |
162 | } | |
163 | ||
300613e5 | 164 | if ((i < rgn->cnt - 1) && lmb_regions_adjacent(rgn, i, i+1)) { |
7c8c6b97 PM |
165 | lmb_coalesce_regions(rgn, i, i+1); |
166 | coalesced++; | |
167 | } | |
168 | ||
169 | if (coalesced) | |
170 | return coalesced; | |
171 | if (rgn->cnt >= MAX_LMB_REGIONS) | |
172 | return -1; | |
173 | ||
174 | /* Couldn't coalesce the LMB, so add it to the sorted table. */ | |
300613e5 | 175 | for (i = rgn->cnt - 1; i >= 0; i--) { |
7c8c6b97 PM |
176 | if (base < rgn->region[i].base) { |
177 | rgn->region[i+1].base = rgn->region[i].base; | |
178 | rgn->region[i+1].size = rgn->region[i].size; | |
179 | } else { | |
180 | rgn->region[i+1].base = base; | |
181 | rgn->region[i+1].size = size; | |
182 | break; | |
183 | } | |
184 | } | |
74b20dad KG |
185 | |
186 | if (base < rgn->region[0].base) { | |
187 | rgn->region[0].base = base; | |
188 | rgn->region[0].size = size; | |
189 | } | |
7c8c6b97 PM |
190 | rgn->cnt++; |
191 | ||
192 | return 0; | |
193 | } | |
194 | ||
98d5c21c | 195 | long lmb_add(u64 base, u64 size) |
7c8c6b97 | 196 | { |
300613e5 | 197 | struct lmb_region *_rgn = &lmb.memory; |
7c8c6b97 PM |
198 | |
199 | /* On pSeries LPAR systems, the first LMB is our RMO region. */ | |
200 | if (base == 0) | |
201 | lmb.rmo_size = size; | |
202 | ||
203 | return lmb_add_region(_rgn, base, size); | |
204 | ||
205 | } | |
206 | ||
98d5c21c BP |
207 | long lmb_remove(u64 base, u64 size) |
208 | { | |
209 | struct lmb_region *rgn = &(lmb.memory); | |
210 | u64 rgnbegin, rgnend; | |
211 | u64 end = base + size; | |
212 | int i; | |
213 | ||
214 | rgnbegin = rgnend = 0; /* supress gcc warnings */ | |
215 | ||
216 | /* Find the region where (base, size) belongs to */ | |
217 | for (i=0; i < rgn->cnt; i++) { | |
218 | rgnbegin = rgn->region[i].base; | |
219 | rgnend = rgnbegin + rgn->region[i].size; | |
220 | ||
221 | if ((rgnbegin <= base) && (end <= rgnend)) | |
222 | break; | |
223 | } | |
224 | ||
225 | /* Didn't find the region */ | |
226 | if (i == rgn->cnt) | |
227 | return -1; | |
228 | ||
229 | /* Check to see if we are removing entire region */ | |
230 | if ((rgnbegin == base) && (rgnend == end)) { | |
231 | lmb_remove_region(rgn, i); | |
232 | return 0; | |
233 | } | |
234 | ||
235 | /* Check to see if region is matching at the front */ | |
236 | if (rgnbegin == base) { | |
237 | rgn->region[i].base = end; | |
238 | rgn->region[i].size -= size; | |
239 | return 0; | |
240 | } | |
241 | ||
242 | /* Check to see if the region is matching at the end */ | |
243 | if (rgnend == end) { | |
244 | rgn->region[i].size -= size; | |
245 | return 0; | |
246 | } | |
247 | ||
248 | /* | |
249 | * We need to split the entry - adjust the current one to the | |
250 | * beginging of the hole and add the region after hole. | |
251 | */ | |
252 | rgn->region[i].size = base - rgn->region[i].base; | |
253 | return lmb_add_region(rgn, end, rgnend - end); | |
254 | } | |
255 | ||
e5f27095 | 256 | long __init lmb_reserve(u64 base, u64 size) |
7c8c6b97 | 257 | { |
300613e5 | 258 | struct lmb_region *_rgn = &lmb.reserved; |
7c8c6b97 | 259 | |
8c20fafa ME |
260 | BUG_ON(0 == size); |
261 | ||
7c8c6b97 PM |
262 | return lmb_add_region(_rgn, base, size); |
263 | } | |
264 | ||
300613e5 | 265 | long __init lmb_overlaps_region(struct lmb_region *rgn, u64 base, u64 size) |
7c8c6b97 PM |
266 | { |
267 | unsigned long i; | |
268 | ||
300613e5 | 269 | for (i = 0; i < rgn->cnt; i++) { |
e5f27095 BB |
270 | u64 rgnbase = rgn->region[i].base; |
271 | u64 rgnsize = rgn->region[i].size; | |
300613e5 | 272 | if (lmb_addrs_overlap(base, size, rgnbase, rgnsize)) |
7c8c6b97 | 273 | break; |
7c8c6b97 PM |
274 | } |
275 | ||
276 | return (i < rgn->cnt) ? i : -1; | |
277 | } | |
278 | ||
c50f68c8 DM |
279 | static u64 lmb_align_down(u64 addr, u64 size) |
280 | { | |
281 | return addr & ~(size - 1); | |
282 | } | |
283 | ||
284 | static u64 lmb_align_up(u64 addr, u64 size) | |
285 | { | |
286 | return (addr + (size - 1)) & ~(size - 1); | |
287 | } | |
288 | ||
289 | static u64 __init lmb_alloc_nid_unreserved(u64 start, u64 end, | |
290 | u64 size, u64 align) | |
291 | { | |
d9024df0 | 292 | u64 base, res_base; |
c50f68c8 DM |
293 | long j; |
294 | ||
295 | base = lmb_align_down((end - size), align); | |
d9024df0 PM |
296 | while (start <= base) { |
297 | j = lmb_overlaps_region(&lmb.reserved, base, size); | |
298 | if (j < 0) { | |
299 | /* this area isn't reserved, take it */ | |
4978db5b | 300 | if (lmb_add_region(&lmb.reserved, base, size) < 0) |
d9024df0 PM |
301 | base = ~(u64)0; |
302 | return base; | |
303 | } | |
304 | res_base = lmb.reserved.region[j].base; | |
305 | if (res_base < size) | |
306 | break; | |
307 | base = lmb_align_down(res_base - size, align); | |
c50f68c8 DM |
308 | } |
309 | ||
310 | return ~(u64)0; | |
311 | } | |
312 | ||
313 | static u64 __init lmb_alloc_nid_region(struct lmb_property *mp, | |
314 | u64 (*nid_range)(u64, u64, int *), | |
315 | u64 size, u64 align, int nid) | |
316 | { | |
317 | u64 start, end; | |
318 | ||
319 | start = mp->base; | |
320 | end = start + mp->size; | |
321 | ||
322 | start = lmb_align_up(start, align); | |
323 | while (start < end) { | |
324 | u64 this_end; | |
325 | int this_nid; | |
326 | ||
327 | this_end = nid_range(start, end, &this_nid); | |
328 | if (this_nid == nid) { | |
329 | u64 ret = lmb_alloc_nid_unreserved(start, this_end, | |
330 | size, align); | |
331 | if (ret != ~(u64)0) | |
332 | return ret; | |
333 | } | |
334 | start = this_end; | |
335 | } | |
336 | ||
337 | return ~(u64)0; | |
338 | } | |
339 | ||
340 | u64 __init lmb_alloc_nid(u64 size, u64 align, int nid, | |
341 | u64 (*nid_range)(u64 start, u64 end, int *nid)) | |
342 | { | |
343 | struct lmb_region *mem = &lmb.memory; | |
344 | int i; | |
345 | ||
4978db5b DM |
346 | BUG_ON(0 == size); |
347 | ||
348 | size = lmb_align_up(size, align); | |
349 | ||
c50f68c8 DM |
350 | for (i = 0; i < mem->cnt; i++) { |
351 | u64 ret = lmb_alloc_nid_region(&mem->region[i], | |
352 | nid_range, | |
353 | size, align, nid); | |
354 | if (ret != ~(u64)0) | |
355 | return ret; | |
356 | } | |
357 | ||
358 | return lmb_alloc(size, align); | |
359 | } | |
360 | ||
e5f27095 | 361 | u64 __init lmb_alloc(u64 size, u64 align) |
7c8c6b97 PM |
362 | { |
363 | return lmb_alloc_base(size, align, LMB_ALLOC_ANYWHERE); | |
364 | } | |
365 | ||
e5f27095 | 366 | u64 __init lmb_alloc_base(u64 size, u64 align, u64 max_addr) |
d7a5b2ff | 367 | { |
e5f27095 | 368 | u64 alloc; |
d7a5b2ff ME |
369 | |
370 | alloc = __lmb_alloc_base(size, align, max_addr); | |
371 | ||
2c276603 | 372 | if (alloc == 0) |
e5f27095 BB |
373 | panic("ERROR: Failed to allocate 0x%llx bytes below 0x%llx.\n", |
374 | (unsigned long long) size, (unsigned long long) max_addr); | |
d7a5b2ff ME |
375 | |
376 | return alloc; | |
377 | } | |
378 | ||
e5f27095 | 379 | u64 __init __lmb_alloc_base(u64 size, u64 align, u64 max_addr) |
7c8c6b97 PM |
380 | { |
381 | long i, j; | |
e5f27095 | 382 | u64 base = 0; |
d9024df0 | 383 | u64 res_base; |
7c8c6b97 | 384 | |
8c20fafa ME |
385 | BUG_ON(0 == size); |
386 | ||
4978db5b DM |
387 | size = lmb_align_up(size, align); |
388 | ||
d9b2b2a2 | 389 | /* On some platforms, make sure we allocate lowmem */ |
d9024df0 | 390 | /* Note that LMB_REAL_LIMIT may be LMB_ALLOC_ANYWHERE */ |
7c8c6b97 | 391 | if (max_addr == LMB_ALLOC_ANYWHERE) |
d9b2b2a2 DM |
392 | max_addr = LMB_REAL_LIMIT; |
393 | ||
300613e5 | 394 | for (i = lmb.memory.cnt - 1; i >= 0; i--) { |
e5f27095 BB |
395 | u64 lmbbase = lmb.memory.region[i].base; |
396 | u64 lmbsize = lmb.memory.region[i].size; | |
7c8c6b97 | 397 | |
d9024df0 PM |
398 | if (lmbsize < size) |
399 | continue; | |
7c8c6b97 | 400 | if (max_addr == LMB_ALLOC_ANYWHERE) |
d9b2b2a2 | 401 | base = lmb_align_down(lmbbase + lmbsize - size, align); |
7c8c6b97 PM |
402 | else if (lmbbase < max_addr) { |
403 | base = min(lmbbase + lmbsize, max_addr); | |
d9b2b2a2 | 404 | base = lmb_align_down(base - size, align); |
7c8c6b97 PM |
405 | } else |
406 | continue; | |
407 | ||
d9024df0 | 408 | while (base && lmbbase <= base) { |
300613e5 | 409 | j = lmb_overlaps_region(&lmb.reserved, base, size); |
d9024df0 PM |
410 | if (j < 0) { |
411 | /* this area isn't reserved, take it */ | |
4978db5b | 412 | if (lmb_add_region(&lmb.reserved, base, size) < 0) |
d9024df0 PM |
413 | return 0; |
414 | return base; | |
415 | } | |
416 | res_base = lmb.reserved.region[j].base; | |
417 | if (res_base < size) | |
300613e5 | 418 | break; |
d9024df0 | 419 | base = lmb_align_down(res_base - size, align); |
300613e5 | 420 | } |
7c8c6b97 | 421 | } |
d9024df0 | 422 | return 0; |
7c8c6b97 PM |
423 | } |
424 | ||
425 | /* You must call lmb_analyze() before this. */ | |
e5f27095 | 426 | u64 __init lmb_phys_mem_size(void) |
7c8c6b97 PM |
427 | { |
428 | return lmb.memory.size; | |
429 | } | |
430 | ||
e5f27095 | 431 | u64 __init lmb_end_of_DRAM(void) |
7c8c6b97 PM |
432 | { |
433 | int idx = lmb.memory.cnt - 1; | |
434 | ||
435 | return (lmb.memory.region[idx].base + lmb.memory.region[idx].size); | |
436 | } | |
437 | ||
2babf5c2 | 438 | /* You must call lmb_analyze() after this. */ |
e5f27095 | 439 | void __init lmb_enforce_memory_limit(u64 memory_limit) |
7c8c6b97 | 440 | { |
e5f27095 BB |
441 | unsigned long i; |
442 | u64 limit; | |
2babf5c2 | 443 | struct lmb_property *p; |
7c8c6b97 | 444 | |
300613e5 | 445 | if (!memory_limit) |
7c8c6b97 PM |
446 | return; |
447 | ||
2babf5c2 | 448 | /* Truncate the lmb regions to satisfy the memory limit. */ |
7c8c6b97 PM |
449 | limit = memory_limit; |
450 | for (i = 0; i < lmb.memory.cnt; i++) { | |
451 | if (limit > lmb.memory.region[i].size) { | |
452 | limit -= lmb.memory.region[i].size; | |
453 | continue; | |
454 | } | |
455 | ||
456 | lmb.memory.region[i].size = limit; | |
457 | lmb.memory.cnt = i + 1; | |
458 | break; | |
459 | } | |
2babf5c2 | 460 | |
30f30e13 ME |
461 | if (lmb.memory.region[0].size < lmb.rmo_size) |
462 | lmb.rmo_size = lmb.memory.region[0].size; | |
2babf5c2 ME |
463 | |
464 | /* And truncate any reserves above the limit also. */ | |
465 | for (i = 0; i < lmb.reserved.cnt; i++) { | |
466 | p = &lmb.reserved.region[i]; | |
467 | ||
468 | if (p->base > memory_limit) | |
469 | p->size = 0; | |
470 | else if ((p->base + p->size) > memory_limit) | |
471 | p->size = memory_limit - p->base; | |
472 | ||
473 | if (p->size == 0) { | |
474 | lmb_remove_region(&lmb.reserved, i); | |
475 | i--; | |
476 | } | |
477 | } | |
7c8c6b97 | 478 | } |
f98eeb4e | 479 | |
e5f27095 | 480 | int __init lmb_is_reserved(u64 addr) |
f98eeb4e KG |
481 | { |
482 | int i; | |
483 | ||
484 | for (i = 0; i < lmb.reserved.cnt; i++) { | |
e5f27095 BB |
485 | u64 upper = lmb.reserved.region[i].base + |
486 | lmb.reserved.region[i].size - 1; | |
f98eeb4e KG |
487 | if ((addr >= lmb.reserved.region[i].base) && (addr <= upper)) |
488 | return 1; | |
489 | } | |
490 | return 0; | |
491 | } | |
9d88a2eb BP |
492 | |
493 | /* | |
494 | * Given a <base, len>, find which memory regions belong to this range. | |
495 | * Adjust the request and return a contiguous chunk. | |
496 | */ | |
497 | int lmb_find(struct lmb_property *res) | |
498 | { | |
499 | int i; | |
500 | u64 rstart, rend; | |
501 | ||
502 | rstart = res->base; | |
503 | rend = rstart + res->size - 1; | |
504 | ||
505 | for (i = 0; i < lmb.memory.cnt; i++) { | |
506 | u64 start = lmb.memory.region[i].base; | |
507 | u64 end = start + lmb.memory.region[i].size - 1; | |
508 | ||
509 | if (start > rend) | |
510 | return -1; | |
511 | ||
512 | if ((end >= rstart) && (start < rend)) { | |
513 | /* adjust the request */ | |
514 | if (rstart < start) | |
515 | rstart = start; | |
516 | if (rend > end) | |
517 | rend = end; | |
518 | res->base = rstart; | |
519 | res->size = rend - rstart + 1; | |
520 | return 0; | |
521 | } | |
522 | } | |
523 | return -1; | |
524 | } |