Commit | Line | Data |
---|---|---|
c672528a CC |
1 | /* |
2 | * Copyright (C) 2012 Google, Inc. | |
3 | * | |
4 | * This software is licensed under the terms of the GNU General Public | |
5 | * License version 2, as published by the Free Software Foundation, and | |
6 | * may be copied, distributed, and modified under those terms. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | */ | |
14 | ||
404a6043 CC |
15 | #include <linux/device.h> |
16 | #include <linux/err.h> | |
c672528a CC |
17 | #include <linux/errno.h> |
18 | #include <linux/kernel.h> | |
19 | #include <linux/init.h> | |
20 | #include <linux/io.h> | |
404a6043 CC |
21 | #include <linux/list.h> |
22 | #include <linux/memblock.h> | |
9cc05ad9 | 23 | #include <linux/rslib.h> |
c672528a | 24 | #include <linux/slab.h> |
404a6043 | 25 | #include <linux/vmalloc.h> |
c672528a CC |
26 | #include "persistent_ram.h" |
27 | ||
c672528a CC |
28 | struct persistent_ram_buffer { |
29 | uint32_t sig; | |
808d0387 CC |
30 | atomic_t start; |
31 | atomic_t size; | |
c672528a CC |
32 | uint8_t data[0]; |
33 | }; | |
34 | ||
35 | #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ | |
36 | ||
404a6043 | 37 | static __initdata LIST_HEAD(persistent_ram_list); |
c672528a | 38 | |
808d0387 CC |
39 | static inline size_t buffer_size(struct persistent_ram_zone *prz) |
40 | { | |
41 | return atomic_read(&prz->buffer->size); | |
42 | } | |
43 | ||
44 | static inline size_t buffer_start(struct persistent_ram_zone *prz) | |
45 | { | |
46 | return atomic_read(&prz->buffer->start); | |
47 | } | |
48 | ||
49 | /* increase and wrap the start pointer, returning the old value */ | |
50 | static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) | |
51 | { | |
52 | int old; | |
53 | int new; | |
54 | ||
55 | do { | |
56 | old = atomic_read(&prz->buffer->start); | |
57 | new = old + a; | |
58 | while (unlikely(new > prz->buffer_size)) | |
59 | new -= prz->buffer_size; | |
60 | } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old); | |
61 | ||
62 | return old; | |
63 | } | |
64 | ||
65 | /* increase the size counter until it hits the max size */ | |
66 | static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a) | |
67 | { | |
68 | size_t old; | |
69 | size_t new; | |
70 | ||
71 | if (atomic_read(&prz->buffer->size) == prz->buffer_size) | |
72 | return; | |
73 | ||
74 | do { | |
75 | old = atomic_read(&prz->buffer->size); | |
76 | new = old + a; | |
77 | if (new > prz->buffer_size) | |
78 | new = prz->buffer_size; | |
79 | } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); | |
80 | } | |
81 | ||
82 | /* increase the size counter, retuning an error if it hits the max size */ | |
83 | static inline ssize_t buffer_size_add_clamp(struct persistent_ram_zone *prz, | |
84 | size_t a) | |
85 | { | |
86 | size_t old; | |
87 | size_t new; | |
88 | ||
89 | do { | |
90 | old = atomic_read(&prz->buffer->size); | |
91 | new = old + a; | |
92 | if (new > prz->buffer_size) | |
93 | return -ENOMEM; | |
94 | } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); | |
95 | ||
96 | return 0; | |
97 | } | |
98 | ||
a15d0b36 | 99 | static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, |
c672528a CC |
100 | uint8_t *data, size_t len, uint8_t *ecc) |
101 | { | |
102 | int i; | |
9cc05ad9 CC |
103 | uint16_t par[prz->ecc_size]; |
104 | ||
c672528a CC |
105 | /* Initialize the parity buffer */ |
106 | memset(par, 0, sizeof(par)); | |
107 | encode_rs8(prz->rs_decoder, data, len, par, 0); | |
9cc05ad9 | 108 | for (i = 0; i < prz->ecc_size; i++) |
c672528a CC |
109 | ecc[i] = par[i]; |
110 | } | |
111 | ||
112 | static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, | |
113 | void *data, size_t len, uint8_t *ecc) | |
114 | { | |
115 | int i; | |
9cc05ad9 CC |
116 | uint16_t par[prz->ecc_size]; |
117 | ||
118 | for (i = 0; i < prz->ecc_size; i++) | |
c672528a CC |
119 | par[i] = ecc[i]; |
120 | return decode_rs8(prz->rs_decoder, data, par, len, | |
121 | NULL, 0, NULL, 0, NULL); | |
122 | } | |
c672528a | 123 | |
a15d0b36 | 124 | static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, |
808d0387 | 125 | unsigned int start, unsigned int count) |
c672528a CC |
126 | { |
127 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
128 | uint8_t *buffer_end = buffer->data + prz->buffer_size; |
129 | uint8_t *block; | |
130 | uint8_t *par; | |
9cc05ad9 CC |
131 | int ecc_block_size = prz->ecc_block_size; |
132 | int ecc_size = prz->ecc_size; | |
133 | int size = prz->ecc_block_size; | |
134 | ||
135 | if (!prz->ecc) | |
136 | return; | |
137 | ||
808d0387 CC |
138 | block = buffer->data + (start & ~(ecc_block_size - 1)); |
139 | par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size; | |
140 | ||
c672528a | 141 | do { |
9cc05ad9 | 142 | if (block + ecc_block_size > buffer_end) |
c672528a CC |
143 | size = buffer_end - block; |
144 | persistent_ram_encode_rs8(prz, block, size, par); | |
9cc05ad9 CC |
145 | block += ecc_block_size; |
146 | par += ecc_size; | |
808d0387 | 147 | } while (block < buffer->data + start + count); |
c672528a CC |
148 | } |
149 | ||
9cc05ad9 | 150 | static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) |
c672528a | 151 | { |
c672528a CC |
152 | struct persistent_ram_buffer *buffer = prz->buffer; |
153 | ||
9cc05ad9 CC |
154 | if (!prz->ecc) |
155 | return; | |
156 | ||
c672528a CC |
157 | persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), |
158 | prz->par_header); | |
c672528a CC |
159 | } |
160 | ||
9cc05ad9 | 161 | static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) |
c672528a CC |
162 | { |
163 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
164 | uint8_t *block; |
165 | uint8_t *par; | |
166 | ||
9cc05ad9 CC |
167 | if (!prz->ecc) |
168 | return; | |
169 | ||
c672528a CC |
170 | block = buffer->data; |
171 | par = prz->par_buffer; | |
808d0387 | 172 | while (block < buffer->data + buffer_size(prz)) { |
c672528a | 173 | int numerr; |
9cc05ad9 | 174 | int size = prz->ecc_block_size; |
c672528a CC |
175 | if (block + size > buffer->data + prz->buffer_size) |
176 | size = buffer->data + prz->buffer_size - block; | |
177 | numerr = persistent_ram_decode_rs8(prz, block, size, par); | |
178 | if (numerr > 0) { | |
9cc05ad9 | 179 | pr_devel("persistent_ram: error in block %p, %d\n", |
c672528a | 180 | block, numerr); |
c672528a CC |
181 | prz->corrected_bytes += numerr; |
182 | } else if (numerr < 0) { | |
9cc05ad9 | 183 | pr_devel("persistent_ram: uncorrectable error in block %p\n", |
c672528a | 184 | block); |
c672528a CC |
185 | prz->bad_blocks++; |
186 | } | |
9cc05ad9 CC |
187 | block += prz->ecc_block_size; |
188 | par += prz->ecc_size; | |
189 | } | |
190 | } | |
191 | ||
192 | static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, | |
193 | size_t buffer_size) | |
194 | { | |
195 | int numerr; | |
196 | struct persistent_ram_buffer *buffer = prz->buffer; | |
197 | int ecc_blocks; | |
198 | ||
199 | if (!prz->ecc) | |
200 | return 0; | |
201 | ||
202 | prz->ecc_block_size = 128; | |
203 | prz->ecc_size = 16; | |
204 | prz->ecc_symsize = 8; | |
205 | prz->ecc_poly = 0x11d; | |
206 | ||
207 | ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size); | |
208 | prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size; | |
209 | ||
210 | if (prz->buffer_size > buffer_size) { | |
211 | pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n", | |
212 | buffer_size, prz->buffer_size); | |
213 | return -EINVAL; | |
214 | } | |
215 | ||
216 | prz->par_buffer = buffer->data + prz->buffer_size; | |
217 | prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size; | |
218 | ||
219 | /* | |
220 | * first consecutive root is 0 | |
221 | * primitive element to generate roots = 1 | |
222 | */ | |
223 | prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1, | |
224 | prz->ecc_size); | |
225 | if (prz->rs_decoder == NULL) { | |
226 | pr_info("persistent_ram: init_rs failed\n"); | |
227 | return -EINVAL; | |
c672528a | 228 | } |
9cc05ad9 CC |
229 | |
230 | prz->corrected_bytes = 0; | |
231 | prz->bad_blocks = 0; | |
232 | ||
233 | numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), | |
234 | prz->par_header); | |
235 | if (numerr > 0) { | |
236 | pr_info("persistent_ram: error in header, %d\n", numerr); | |
237 | prz->corrected_bytes += numerr; | |
238 | } else if (numerr < 0) { | |
239 | pr_info("persistent_ram: uncorrectable error in header\n"); | |
240 | prz->bad_blocks++; | |
241 | } | |
242 | ||
243 | return 0; | |
244 | } | |
245 | ||
246 | ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, | |
247 | char *str, size_t len) | |
248 | { | |
249 | ssize_t ret; | |
250 | ||
251 | if (prz->corrected_bytes || prz->bad_blocks) | |
252 | ret = snprintf(str, len, "" | |
253 | "\n%d Corrected bytes, %d unrecoverable blocks\n", | |
254 | prz->corrected_bytes, prz->bad_blocks); | |
255 | else | |
256 | ret = snprintf(str, len, "\nNo errors detected\n"); | |
257 | ||
258 | return ret; | |
259 | } | |
260 | ||
a15d0b36 | 261 | static void notrace persistent_ram_update(struct persistent_ram_zone *prz, |
808d0387 | 262 | const void *s, unsigned int start, unsigned int count) |
9cc05ad9 CC |
263 | { |
264 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
265 | memcpy(buffer->data + start, s, count); |
266 | persistent_ram_update_ecc(prz, start, count); | |
9cc05ad9 CC |
267 | } |
268 | ||
269 | static void __init | |
270 | persistent_ram_save_old(struct persistent_ram_zone *prz) | |
271 | { | |
272 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
273 | size_t size = buffer_size(prz); |
274 | size_t start = buffer_start(prz); | |
9cc05ad9 CC |
275 | char *dest; |
276 | ||
277 | persistent_ram_ecc_old(prz); | |
c672528a | 278 | |
808d0387 | 279 | dest = kmalloc(size, GFP_KERNEL); |
c672528a CC |
280 | if (dest == NULL) { |
281 | pr_err("persistent_ram: failed to allocate buffer\n"); | |
282 | return; | |
283 | } | |
284 | ||
285 | prz->old_log = dest; | |
808d0387 CC |
286 | prz->old_log_size = size; |
287 | memcpy(prz->old_log, &buffer->data[start], size - start); | |
288 | memcpy(prz->old_log + size - start, &buffer->data[0], start); | |
c672528a CC |
289 | } |
290 | ||
a15d0b36 | 291 | int notrace persistent_ram_write(struct persistent_ram_zone *prz, |
c672528a CC |
292 | const void *s, unsigned int count) |
293 | { | |
294 | int rem; | |
295 | int c = count; | |
808d0387 | 296 | size_t start; |
c672528a | 297 | |
808d0387 | 298 | if (unlikely(c > prz->buffer_size)) { |
c672528a CC |
299 | s += c - prz->buffer_size; |
300 | c = prz->buffer_size; | |
301 | } | |
808d0387 CC |
302 | |
303 | buffer_size_add_clamp(prz, c); | |
304 | ||
305 | start = buffer_start_add(prz, c); | |
306 | ||
307 | rem = prz->buffer_size - start; | |
308 | if (unlikely(rem < c)) { | |
309 | persistent_ram_update(prz, s, start, rem); | |
c672528a CC |
310 | s += rem; |
311 | c -= rem; | |
808d0387 | 312 | start = 0; |
c672528a | 313 | } |
808d0387 | 314 | persistent_ram_update(prz, s, start, c); |
c672528a | 315 | |
9cc05ad9 | 316 | persistent_ram_update_header_ecc(prz); |
c672528a CC |
317 | |
318 | return count; | |
319 | } | |
320 | ||
c672528a CC |
321 | size_t persistent_ram_old_size(struct persistent_ram_zone *prz) |
322 | { | |
323 | return prz->old_log_size; | |
324 | } | |
325 | ||
326 | void *persistent_ram_old(struct persistent_ram_zone *prz) | |
327 | { | |
328 | return prz->old_log; | |
329 | } | |
330 | ||
331 | void persistent_ram_free_old(struct persistent_ram_zone *prz) | |
332 | { | |
333 | kfree(prz->old_log); | |
334 | prz->old_log = NULL; | |
335 | prz->old_log_size = 0; | |
336 | } | |
337 | ||
404a6043 CC |
338 | static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, |
339 | struct persistent_ram_zone *prz) | |
c672528a | 340 | { |
404a6043 CC |
341 | struct page **pages; |
342 | phys_addr_t page_start; | |
343 | unsigned int page_count; | |
344 | pgprot_t prot; | |
345 | unsigned int i; | |
346 | ||
347 | page_start = start - offset_in_page(start); | |
348 | page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); | |
349 | ||
350 | prot = pgprot_noncached(PAGE_KERNEL); | |
351 | ||
352 | pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL); | |
353 | if (!pages) { | |
354 | pr_err("%s: Failed to allocate array for %u pages\n", __func__, | |
355 | page_count); | |
356 | return -ENOMEM; | |
357 | } | |
358 | ||
359 | for (i = 0; i < page_count; i++) { | |
360 | phys_addr_t addr = page_start + i * PAGE_SIZE; | |
361 | pages[i] = pfn_to_page(addr >> PAGE_SHIFT); | |
362 | } | |
363 | prz->vaddr = vmap(pages, page_count, VM_MAP, prot); | |
364 | kfree(pages); | |
365 | if (!prz->vaddr) { | |
366 | pr_err("%s: Failed to map %u pages\n", __func__, page_count); | |
367 | return -ENOMEM; | |
368 | } | |
369 | ||
370 | prz->buffer = prz->vaddr + offset_in_page(start); | |
371 | prz->buffer_size = size - sizeof(struct persistent_ram_buffer); | |
372 | ||
373 | return 0; | |
374 | } | |
375 | ||
376 | static int __init persistent_ram_buffer_init(const char *name, | |
377 | struct persistent_ram_zone *prz) | |
378 | { | |
379 | int i; | |
380 | struct persistent_ram *ram; | |
381 | struct persistent_ram_descriptor *desc; | |
382 | phys_addr_t start; | |
383 | ||
384 | list_for_each_entry(ram, &persistent_ram_list, node) { | |
385 | start = ram->start; | |
386 | for (i = 0; i < ram->num_descs; i++) { | |
387 | desc = &ram->descs[i]; | |
388 | if (!strcmp(desc->name, name)) | |
389 | return persistent_ram_buffer_map(start, | |
390 | desc->size, prz); | |
391 | start += desc->size; | |
392 | } | |
393 | } | |
394 | ||
395 | return -EINVAL; | |
396 | } | |
397 | ||
398 | static __init | |
399 | struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc) | |
400 | { | |
401 | struct persistent_ram_zone *prz; | |
474a8988 | 402 | int ret = -ENOMEM; |
c672528a | 403 | |
404a6043 CC |
404 | prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); |
405 | if (!prz) { | |
406 | pr_err("persistent_ram: failed to allocate persistent ram zone\n"); | |
474a8988 | 407 | goto err; |
404a6043 | 408 | } |
c672528a | 409 | |
404a6043 CC |
410 | ret = persistent_ram_buffer_init(dev_name(dev), prz); |
411 | if (ret) { | |
412 | pr_err("persistent_ram: failed to initialize buffer\n"); | |
474a8988 | 413 | goto err; |
c672528a CC |
414 | } |
415 | ||
9cc05ad9 | 416 | prz->ecc = ecc; |
404a6043 | 417 | ret = persistent_ram_init_ecc(prz, prz->buffer_size); |
9cc05ad9 | 418 | if (ret) |
474a8988 | 419 | goto err; |
c672528a | 420 | |
404a6043 | 421 | if (prz->buffer->sig == PERSISTENT_RAM_SIG) { |
808d0387 CC |
422 | if (buffer_size(prz) > prz->buffer_size || |
423 | buffer_start(prz) > buffer_size(prz)) | |
424 | pr_info("persistent_ram: found existing invalid buffer," | |
f56d711b | 425 | " size %zu, start %zu\n", |
808d0387 | 426 | buffer_size(prz), buffer_start(prz)); |
c672528a | 427 | else { |
808d0387 | 428 | pr_info("persistent_ram: found existing buffer," |
f56d711b | 429 | " size %zu, start %zu\n", |
808d0387 | 430 | buffer_size(prz), buffer_start(prz)); |
c672528a CC |
431 | persistent_ram_save_old(prz); |
432 | } | |
433 | } else { | |
808d0387 CC |
434 | pr_info("persistent_ram: no valid data in buffer" |
435 | " (sig = 0x%08x)\n", prz->buffer->sig); | |
c672528a CC |
436 | } |
437 | ||
404a6043 | 438 | prz->buffer->sig = PERSISTENT_RAM_SIG; |
808d0387 CC |
439 | atomic_set(&prz->buffer->start, 0); |
440 | atomic_set(&prz->buffer->size, 0); | |
c672528a | 441 | |
404a6043 | 442 | return prz; |
474a8988 JJ |
443 | err: |
444 | kfree(prz); | |
445 | return ERR_PTR(ret); | |
404a6043 | 446 | } |
c672528a | 447 | |
404a6043 CC |
448 | struct persistent_ram_zone * __init |
449 | persistent_ram_init_ringbuffer(struct device *dev, bool ecc) | |
450 | { | |
451 | return __persistent_ram_init(dev, ecc); | |
c672528a CC |
452 | } |
453 | ||
404a6043 | 454 | int __init persistent_ram_early_init(struct persistent_ram *ram) |
c672528a | 455 | { |
404a6043 CC |
456 | int ret; |
457 | ||
458 | ret = memblock_reserve(ram->start, ram->size); | |
459 | if (ret) { | |
460 | pr_err("Failed to reserve persistent memory from %08lx-%08lx\n", | |
461 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
462 | return ret; | |
463 | } | |
464 | ||
465 | list_add_tail(&ram->node, &persistent_ram_list); | |
466 | ||
467 | pr_info("Initialized persistent memory from %08lx-%08lx\n", | |
468 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
469 | ||
470 | return 0; | |
c672528a | 471 | } |