Commit | Line | Data |
---|---|---|
c672528a CC |
1 | /* |
2 | * Copyright (C) 2012 Google, Inc. | |
3 | * | |
4 | * This software is licensed under the terms of the GNU General Public | |
5 | * License version 2, as published by the Free Software Foundation, and | |
6 | * may be copied, distributed, and modified under those terms. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | */ | |
14 | ||
404a6043 CC |
15 | #include <linux/device.h> |
16 | #include <linux/err.h> | |
c672528a CC |
17 | #include <linux/errno.h> |
18 | #include <linux/kernel.h> | |
19 | #include <linux/init.h> | |
20 | #include <linux/io.h> | |
404a6043 CC |
21 | #include <linux/list.h> |
22 | #include <linux/memblock.h> | |
9cc05ad9 | 23 | #include <linux/rslib.h> |
c672528a | 24 | #include <linux/slab.h> |
404a6043 | 25 | #include <linux/vmalloc.h> |
cddb8751 | 26 | #include <linux/pstore_ram.h> |
24c3d2f3 | 27 | #include <asm/page.h> |
c672528a | 28 | |
c672528a CC |
29 | struct persistent_ram_buffer { |
30 | uint32_t sig; | |
808d0387 CC |
31 | atomic_t start; |
32 | atomic_t size; | |
c672528a CC |
33 | uint8_t data[0]; |
34 | }; | |
35 | ||
36 | #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ | |
37 | ||
404a6043 | 38 | static __initdata LIST_HEAD(persistent_ram_list); |
c672528a | 39 | |
808d0387 CC |
40 | static inline size_t buffer_size(struct persistent_ram_zone *prz) |
41 | { | |
42 | return atomic_read(&prz->buffer->size); | |
43 | } | |
44 | ||
45 | static inline size_t buffer_start(struct persistent_ram_zone *prz) | |
46 | { | |
47 | return atomic_read(&prz->buffer->start); | |
48 | } | |
49 | ||
50 | /* increase and wrap the start pointer, returning the old value */ | |
51 | static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) | |
52 | { | |
53 | int old; | |
54 | int new; | |
55 | ||
56 | do { | |
57 | old = atomic_read(&prz->buffer->start); | |
58 | new = old + a; | |
59 | while (unlikely(new > prz->buffer_size)) | |
60 | new -= prz->buffer_size; | |
61 | } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old); | |
62 | ||
63 | return old; | |
64 | } | |
65 | ||
66 | /* increase the size counter until it hits the max size */ | |
67 | static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a) | |
68 | { | |
69 | size_t old; | |
70 | size_t new; | |
71 | ||
72 | if (atomic_read(&prz->buffer->size) == prz->buffer_size) | |
73 | return; | |
74 | ||
75 | do { | |
76 | old = atomic_read(&prz->buffer->size); | |
77 | new = old + a; | |
78 | if (new > prz->buffer_size) | |
79 | new = prz->buffer_size; | |
80 | } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); | |
81 | } | |
82 | ||
a15d0b36 | 83 | static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, |
c672528a CC |
84 | uint8_t *data, size_t len, uint8_t *ecc) |
85 | { | |
86 | int i; | |
9cc05ad9 CC |
87 | uint16_t par[prz->ecc_size]; |
88 | ||
c672528a CC |
89 | /* Initialize the parity buffer */ |
90 | memset(par, 0, sizeof(par)); | |
91 | encode_rs8(prz->rs_decoder, data, len, par, 0); | |
9cc05ad9 | 92 | for (i = 0; i < prz->ecc_size; i++) |
c672528a CC |
93 | ecc[i] = par[i]; |
94 | } | |
95 | ||
96 | static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, | |
97 | void *data, size_t len, uint8_t *ecc) | |
98 | { | |
99 | int i; | |
9cc05ad9 CC |
100 | uint16_t par[prz->ecc_size]; |
101 | ||
102 | for (i = 0; i < prz->ecc_size; i++) | |
c672528a CC |
103 | par[i] = ecc[i]; |
104 | return decode_rs8(prz->rs_decoder, data, par, len, | |
105 | NULL, 0, NULL, 0, NULL); | |
106 | } | |
c672528a | 107 | |
a15d0b36 | 108 | static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, |
808d0387 | 109 | unsigned int start, unsigned int count) |
c672528a CC |
110 | { |
111 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
112 | uint8_t *buffer_end = buffer->data + prz->buffer_size; |
113 | uint8_t *block; | |
114 | uint8_t *par; | |
9cc05ad9 CC |
115 | int ecc_block_size = prz->ecc_block_size; |
116 | int ecc_size = prz->ecc_size; | |
117 | int size = prz->ecc_block_size; | |
118 | ||
119 | if (!prz->ecc) | |
120 | return; | |
121 | ||
808d0387 CC |
122 | block = buffer->data + (start & ~(ecc_block_size - 1)); |
123 | par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size; | |
124 | ||
c672528a | 125 | do { |
9cc05ad9 | 126 | if (block + ecc_block_size > buffer_end) |
c672528a CC |
127 | size = buffer_end - block; |
128 | persistent_ram_encode_rs8(prz, block, size, par); | |
9cc05ad9 CC |
129 | block += ecc_block_size; |
130 | par += ecc_size; | |
808d0387 | 131 | } while (block < buffer->data + start + count); |
c672528a CC |
132 | } |
133 | ||
9cc05ad9 | 134 | static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) |
c672528a | 135 | { |
c672528a CC |
136 | struct persistent_ram_buffer *buffer = prz->buffer; |
137 | ||
9cc05ad9 CC |
138 | if (!prz->ecc) |
139 | return; | |
140 | ||
c672528a CC |
141 | persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), |
142 | prz->par_header); | |
c672528a CC |
143 | } |
144 | ||
9cc05ad9 | 145 | static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) |
c672528a CC |
146 | { |
147 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
148 | uint8_t *block; |
149 | uint8_t *par; | |
150 | ||
9cc05ad9 CC |
151 | if (!prz->ecc) |
152 | return; | |
153 | ||
c672528a CC |
154 | block = buffer->data; |
155 | par = prz->par_buffer; | |
808d0387 | 156 | while (block < buffer->data + buffer_size(prz)) { |
c672528a | 157 | int numerr; |
9cc05ad9 | 158 | int size = prz->ecc_block_size; |
c672528a CC |
159 | if (block + size > buffer->data + prz->buffer_size) |
160 | size = buffer->data + prz->buffer_size - block; | |
161 | numerr = persistent_ram_decode_rs8(prz, block, size, par); | |
162 | if (numerr > 0) { | |
9cc05ad9 | 163 | pr_devel("persistent_ram: error in block %p, %d\n", |
c672528a | 164 | block, numerr); |
c672528a CC |
165 | prz->corrected_bytes += numerr; |
166 | } else if (numerr < 0) { | |
9cc05ad9 | 167 | pr_devel("persistent_ram: uncorrectable error in block %p\n", |
c672528a | 168 | block); |
c672528a CC |
169 | prz->bad_blocks++; |
170 | } | |
9cc05ad9 CC |
171 | block += prz->ecc_block_size; |
172 | par += prz->ecc_size; | |
173 | } | |
174 | } | |
175 | ||
176 | static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, | |
177 | size_t buffer_size) | |
178 | { | |
179 | int numerr; | |
180 | struct persistent_ram_buffer *buffer = prz->buffer; | |
181 | int ecc_blocks; | |
182 | ||
183 | if (!prz->ecc) | |
184 | return 0; | |
185 | ||
186 | prz->ecc_block_size = 128; | |
187 | prz->ecc_size = 16; | |
188 | prz->ecc_symsize = 8; | |
189 | prz->ecc_poly = 0x11d; | |
190 | ||
191 | ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size); | |
192 | prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size; | |
193 | ||
194 | if (prz->buffer_size > buffer_size) { | |
195 | pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n", | |
196 | buffer_size, prz->buffer_size); | |
197 | return -EINVAL; | |
198 | } | |
199 | ||
200 | prz->par_buffer = buffer->data + prz->buffer_size; | |
201 | prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size; | |
202 | ||
203 | /* | |
204 | * first consecutive root is 0 | |
205 | * primitive element to generate roots = 1 | |
206 | */ | |
207 | prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1, | |
208 | prz->ecc_size); | |
209 | if (prz->rs_decoder == NULL) { | |
210 | pr_info("persistent_ram: init_rs failed\n"); | |
211 | return -EINVAL; | |
c672528a | 212 | } |
9cc05ad9 CC |
213 | |
214 | prz->corrected_bytes = 0; | |
215 | prz->bad_blocks = 0; | |
216 | ||
217 | numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), | |
218 | prz->par_header); | |
219 | if (numerr > 0) { | |
220 | pr_info("persistent_ram: error in header, %d\n", numerr); | |
221 | prz->corrected_bytes += numerr; | |
222 | } else if (numerr < 0) { | |
223 | pr_info("persistent_ram: uncorrectable error in header\n"); | |
224 | prz->bad_blocks++; | |
225 | } | |
226 | ||
227 | return 0; | |
228 | } | |
229 | ||
230 | ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, | |
231 | char *str, size_t len) | |
232 | { | |
233 | ssize_t ret; | |
234 | ||
235 | if (prz->corrected_bytes || prz->bad_blocks) | |
236 | ret = snprintf(str, len, "" | |
237 | "\n%d Corrected bytes, %d unrecoverable blocks\n", | |
238 | prz->corrected_bytes, prz->bad_blocks); | |
239 | else | |
240 | ret = snprintf(str, len, "\nNo errors detected\n"); | |
241 | ||
242 | return ret; | |
243 | } | |
244 | ||
a15d0b36 | 245 | static void notrace persistent_ram_update(struct persistent_ram_zone *prz, |
808d0387 | 246 | const void *s, unsigned int start, unsigned int count) |
9cc05ad9 CC |
247 | { |
248 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
249 | memcpy(buffer->data + start, s, count); |
250 | persistent_ram_update_ecc(prz, start, count); | |
9cc05ad9 CC |
251 | } |
252 | ||
201e4aca | 253 | void persistent_ram_save_old(struct persistent_ram_zone *prz) |
9cc05ad9 CC |
254 | { |
255 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
256 | size_t size = buffer_size(prz); |
257 | size_t start = buffer_start(prz); | |
9cc05ad9 | 258 | |
201e4aca AV |
259 | if (!size) |
260 | return; | |
c672528a | 261 | |
201e4aca AV |
262 | if (!prz->old_log) { |
263 | persistent_ram_ecc_old(prz); | |
264 | prz->old_log = kmalloc(size, GFP_KERNEL); | |
265 | } | |
266 | if (!prz->old_log) { | |
c672528a CC |
267 | pr_err("persistent_ram: failed to allocate buffer\n"); |
268 | return; | |
269 | } | |
270 | ||
808d0387 CC |
271 | prz->old_log_size = size; |
272 | memcpy(prz->old_log, &buffer->data[start], size - start); | |
273 | memcpy(prz->old_log + size - start, &buffer->data[0], start); | |
c672528a CC |
274 | } |
275 | ||
a15d0b36 | 276 | int notrace persistent_ram_write(struct persistent_ram_zone *prz, |
c672528a CC |
277 | const void *s, unsigned int count) |
278 | { | |
279 | int rem; | |
280 | int c = count; | |
808d0387 | 281 | size_t start; |
c672528a | 282 | |
808d0387 | 283 | if (unlikely(c > prz->buffer_size)) { |
c672528a CC |
284 | s += c - prz->buffer_size; |
285 | c = prz->buffer_size; | |
286 | } | |
808d0387 | 287 | |
484dd30e | 288 | buffer_size_add(prz, c); |
808d0387 CC |
289 | |
290 | start = buffer_start_add(prz, c); | |
291 | ||
292 | rem = prz->buffer_size - start; | |
293 | if (unlikely(rem < c)) { | |
294 | persistent_ram_update(prz, s, start, rem); | |
c672528a CC |
295 | s += rem; |
296 | c -= rem; | |
808d0387 | 297 | start = 0; |
c672528a | 298 | } |
808d0387 | 299 | persistent_ram_update(prz, s, start, c); |
c672528a | 300 | |
9cc05ad9 | 301 | persistent_ram_update_header_ecc(prz); |
c672528a CC |
302 | |
303 | return count; | |
304 | } | |
305 | ||
c672528a CC |
306 | size_t persistent_ram_old_size(struct persistent_ram_zone *prz) |
307 | { | |
308 | return prz->old_log_size; | |
309 | } | |
310 | ||
311 | void *persistent_ram_old(struct persistent_ram_zone *prz) | |
312 | { | |
313 | return prz->old_log; | |
314 | } | |
315 | ||
316 | void persistent_ram_free_old(struct persistent_ram_zone *prz) | |
317 | { | |
318 | kfree(prz->old_log); | |
319 | prz->old_log = NULL; | |
320 | prz->old_log_size = 0; | |
321 | } | |
322 | ||
fce39793 AV |
323 | void persistent_ram_zap(struct persistent_ram_zone *prz) |
324 | { | |
325 | atomic_set(&prz->buffer->start, 0); | |
326 | atomic_set(&prz->buffer->size, 0); | |
327 | persistent_ram_update_header_ecc(prz); | |
328 | } | |
329 | ||
2b1321e4 | 330 | static void *persistent_ram_vmap(phys_addr_t start, size_t size) |
c672528a | 331 | { |
404a6043 CC |
332 | struct page **pages; |
333 | phys_addr_t page_start; | |
334 | unsigned int page_count; | |
335 | pgprot_t prot; | |
336 | unsigned int i; | |
2b1321e4 | 337 | void *vaddr; |
404a6043 CC |
338 | |
339 | page_start = start - offset_in_page(start); | |
340 | page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); | |
341 | ||
342 | prot = pgprot_noncached(PAGE_KERNEL); | |
343 | ||
344 | pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL); | |
345 | if (!pages) { | |
346 | pr_err("%s: Failed to allocate array for %u pages\n", __func__, | |
347 | page_count); | |
2b1321e4 | 348 | return NULL; |
404a6043 CC |
349 | } |
350 | ||
351 | for (i = 0; i < page_count; i++) { | |
352 | phys_addr_t addr = page_start + i * PAGE_SIZE; | |
353 | pages[i] = pfn_to_page(addr >> PAGE_SHIFT); | |
354 | } | |
2b1321e4 | 355 | vaddr = vmap(pages, page_count, VM_MAP, prot); |
404a6043 | 356 | kfree(pages); |
2b1321e4 AV |
357 | |
358 | return vaddr; | |
359 | } | |
360 | ||
24c3d2f3 AV |
361 | static void *persistent_ram_iomap(phys_addr_t start, size_t size) |
362 | { | |
363 | if (!request_mem_region(start, size, "persistent_ram")) { | |
364 | pr_err("request mem region (0x%llx@0x%llx) failed\n", | |
365 | (unsigned long long)size, (unsigned long long)start); | |
366 | return NULL; | |
367 | } | |
368 | ||
369 | return ioremap(start, size); | |
370 | } | |
371 | ||
2b1321e4 AV |
372 | static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, |
373 | struct persistent_ram_zone *prz) | |
374 | { | |
d3b48769 AV |
375 | prz->paddr = start; |
376 | prz->size = size; | |
377 | ||
24c3d2f3 AV |
378 | if (pfn_valid(start >> PAGE_SHIFT)) |
379 | prz->vaddr = persistent_ram_vmap(start, size); | |
380 | else | |
381 | prz->vaddr = persistent_ram_iomap(start, size); | |
382 | ||
404a6043 | 383 | if (!prz->vaddr) { |
2b1321e4 AV |
384 | pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, |
385 | (unsigned long long)size, (unsigned long long)start); | |
404a6043 CC |
386 | return -ENOMEM; |
387 | } | |
388 | ||
389 | prz->buffer = prz->vaddr + offset_in_page(start); | |
390 | prz->buffer_size = size - sizeof(struct persistent_ram_buffer); | |
391 | ||
392 | return 0; | |
393 | } | |
394 | ||
bb4206f2 | 395 | static int __init persistent_ram_post_init(struct persistent_ram_zone *prz, bool ecc) |
404a6043 | 396 | { |
bb4206f2 | 397 | int ret; |
c672528a | 398 | |
9cc05ad9 | 399 | prz->ecc = ecc; |
bb4206f2 | 400 | |
404a6043 | 401 | ret = persistent_ram_init_ecc(prz, prz->buffer_size); |
9cc05ad9 | 402 | if (ret) |
bb4206f2 | 403 | return ret; |
c672528a | 404 | |
404a6043 | 405 | if (prz->buffer->sig == PERSISTENT_RAM_SIG) { |
808d0387 CC |
406 | if (buffer_size(prz) > prz->buffer_size || |
407 | buffer_start(prz) > buffer_size(prz)) | |
408 | pr_info("persistent_ram: found existing invalid buffer," | |
f56d711b | 409 | " size %zu, start %zu\n", |
808d0387 | 410 | buffer_size(prz), buffer_start(prz)); |
c672528a | 411 | else { |
808d0387 | 412 | pr_info("persistent_ram: found existing buffer," |
f56d711b | 413 | " size %zu, start %zu\n", |
808d0387 | 414 | buffer_size(prz), buffer_start(prz)); |
c672528a | 415 | persistent_ram_save_old(prz); |
25b63da6 | 416 | return 0; |
c672528a CC |
417 | } |
418 | } else { | |
808d0387 CC |
419 | pr_info("persistent_ram: no valid data in buffer" |
420 | " (sig = 0x%08x)\n", prz->buffer->sig); | |
c672528a CC |
421 | } |
422 | ||
404a6043 | 423 | prz->buffer->sig = PERSISTENT_RAM_SIG; |
fce39793 | 424 | persistent_ram_zap(prz); |
c672528a | 425 | |
bb4206f2 AV |
426 | return 0; |
427 | } | |
428 | ||
d3b48769 AV |
429 | void persistent_ram_free(struct persistent_ram_zone *prz) |
430 | { | |
431 | if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { | |
432 | vunmap(prz->vaddr); | |
433 | } else { | |
434 | iounmap(prz->vaddr); | |
435 | release_mem_region(prz->paddr, prz->size); | |
436 | } | |
437 | persistent_ram_free_old(prz); | |
438 | kfree(prz); | |
439 | } | |
440 | ||
8cf5aff8 AV |
441 | struct persistent_ram_zone * __init persistent_ram_new(phys_addr_t start, |
442 | size_t size, | |
443 | bool ecc) | |
444 | { | |
445 | struct persistent_ram_zone *prz; | |
446 | int ret = -ENOMEM; | |
447 | ||
448 | prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); | |
449 | if (!prz) { | |
450 | pr_err("persistent_ram: failed to allocate persistent ram zone\n"); | |
451 | goto err; | |
452 | } | |
453 | ||
454 | ret = persistent_ram_buffer_map(start, size, prz); | |
455 | if (ret) | |
456 | goto err; | |
457 | ||
458 | persistent_ram_post_init(prz, ecc); | |
8cf5aff8 AV |
459 | |
460 | return prz; | |
461 | err: | |
462 | kfree(prz); | |
463 | return ERR_PTR(ret); | |
464 | } | |
465 | ||
7dd8e9be AV |
466 | #ifndef MODULE |
467 | static int __init persistent_ram_buffer_init(const char *name, | |
468 | struct persistent_ram_zone *prz) | |
469 | { | |
470 | int i; | |
471 | struct persistent_ram *ram; | |
472 | struct persistent_ram_descriptor *desc; | |
473 | phys_addr_t start; | |
474 | ||
475 | list_for_each_entry(ram, &persistent_ram_list, node) { | |
476 | start = ram->start; | |
477 | for (i = 0; i < ram->num_descs; i++) { | |
478 | desc = &ram->descs[i]; | |
479 | if (!strcmp(desc->name, name)) | |
480 | return persistent_ram_buffer_map(start, | |
481 | desc->size, prz); | |
482 | start += desc->size; | |
483 | } | |
484 | } | |
485 | ||
486 | return -EINVAL; | |
487 | } | |
488 | ||
bb4206f2 AV |
489 | static __init |
490 | struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc) | |
491 | { | |
492 | struct persistent_ram_zone *prz; | |
493 | int ret = -ENOMEM; | |
494 | ||
495 | prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); | |
496 | if (!prz) { | |
497 | pr_err("persistent_ram: failed to allocate persistent ram zone\n"); | |
498 | goto err; | |
499 | } | |
500 | ||
501 | ret = persistent_ram_buffer_init(dev_name(dev), prz); | |
502 | if (ret) { | |
503 | pr_err("persistent_ram: failed to initialize buffer\n"); | |
504 | goto err; | |
505 | } | |
506 | ||
507 | persistent_ram_post_init(prz, ecc); | |
508 | ||
404a6043 | 509 | return prz; |
474a8988 JJ |
510 | err: |
511 | kfree(prz); | |
512 | return ERR_PTR(ret); | |
404a6043 | 513 | } |
c672528a | 514 | |
404a6043 CC |
515 | struct persistent_ram_zone * __init |
516 | persistent_ram_init_ringbuffer(struct device *dev, bool ecc) | |
517 | { | |
518 | return __persistent_ram_init(dev, ecc); | |
c672528a CC |
519 | } |
520 | ||
404a6043 | 521 | int __init persistent_ram_early_init(struct persistent_ram *ram) |
c672528a | 522 | { |
404a6043 CC |
523 | int ret; |
524 | ||
525 | ret = memblock_reserve(ram->start, ram->size); | |
526 | if (ret) { | |
527 | pr_err("Failed to reserve persistent memory from %08lx-%08lx\n", | |
528 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
529 | return ret; | |
530 | } | |
531 | ||
532 | list_add_tail(&ram->node, &persistent_ram_list); | |
533 | ||
534 | pr_info("Initialized persistent memory from %08lx-%08lx\n", | |
535 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
536 | ||
537 | return 0; | |
c672528a | 538 | } |
7dd8e9be | 539 | #endif |