Commit | Line | Data |
---|---|---|
c672528a CC |
1 | /* |
2 | * Copyright (C) 2012 Google, Inc. | |
3 | * | |
4 | * This software is licensed under the terms of the GNU General Public | |
5 | * License version 2, as published by the Free Software Foundation, and | |
6 | * may be copied, distributed, and modified under those terms. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
11 | * GNU General Public License for more details. | |
12 | * | |
13 | */ | |
14 | ||
404a6043 CC |
15 | #include <linux/device.h> |
16 | #include <linux/err.h> | |
c672528a CC |
17 | #include <linux/errno.h> |
18 | #include <linux/kernel.h> | |
19 | #include <linux/init.h> | |
20 | #include <linux/io.h> | |
404a6043 CC |
21 | #include <linux/list.h> |
22 | #include <linux/memblock.h> | |
9cc05ad9 | 23 | #include <linux/rslib.h> |
c672528a | 24 | #include <linux/slab.h> |
404a6043 | 25 | #include <linux/vmalloc.h> |
c672528a CC |
26 | #include "persistent_ram.h" |
27 | ||
c672528a CC |
28 | struct persistent_ram_buffer { |
29 | uint32_t sig; | |
808d0387 CC |
30 | atomic_t start; |
31 | atomic_t size; | |
c672528a CC |
32 | uint8_t data[0]; |
33 | }; | |
34 | ||
35 | #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ | |
36 | ||
404a6043 | 37 | static __initdata LIST_HEAD(persistent_ram_list); |
c672528a | 38 | |
808d0387 CC |
39 | static inline size_t buffer_size(struct persistent_ram_zone *prz) |
40 | { | |
41 | return atomic_read(&prz->buffer->size); | |
42 | } | |
43 | ||
44 | static inline size_t buffer_start(struct persistent_ram_zone *prz) | |
45 | { | |
46 | return atomic_read(&prz->buffer->start); | |
47 | } | |
48 | ||
49 | /* increase and wrap the start pointer, returning the old value */ | |
50 | static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) | |
51 | { | |
52 | int old; | |
53 | int new; | |
54 | ||
55 | do { | |
56 | old = atomic_read(&prz->buffer->start); | |
57 | new = old + a; | |
58 | while (unlikely(new > prz->buffer_size)) | |
59 | new -= prz->buffer_size; | |
60 | } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old); | |
61 | ||
62 | return old; | |
63 | } | |
64 | ||
65 | /* increase the size counter until it hits the max size */ | |
66 | static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a) | |
67 | { | |
68 | size_t old; | |
69 | size_t new; | |
70 | ||
71 | if (atomic_read(&prz->buffer->size) == prz->buffer_size) | |
72 | return; | |
73 | ||
74 | do { | |
75 | old = atomic_read(&prz->buffer->size); | |
76 | new = old + a; | |
77 | if (new > prz->buffer_size) | |
78 | new = prz->buffer_size; | |
79 | } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old); | |
80 | } | |
81 | ||
a15d0b36 | 82 | static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, |
c672528a CC |
83 | uint8_t *data, size_t len, uint8_t *ecc) |
84 | { | |
85 | int i; | |
9cc05ad9 CC |
86 | uint16_t par[prz->ecc_size]; |
87 | ||
c672528a CC |
88 | /* Initialize the parity buffer */ |
89 | memset(par, 0, sizeof(par)); | |
90 | encode_rs8(prz->rs_decoder, data, len, par, 0); | |
9cc05ad9 | 91 | for (i = 0; i < prz->ecc_size; i++) |
c672528a CC |
92 | ecc[i] = par[i]; |
93 | } | |
94 | ||
95 | static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, | |
96 | void *data, size_t len, uint8_t *ecc) | |
97 | { | |
98 | int i; | |
9cc05ad9 CC |
99 | uint16_t par[prz->ecc_size]; |
100 | ||
101 | for (i = 0; i < prz->ecc_size; i++) | |
c672528a CC |
102 | par[i] = ecc[i]; |
103 | return decode_rs8(prz->rs_decoder, data, par, len, | |
104 | NULL, 0, NULL, 0, NULL); | |
105 | } | |
c672528a | 106 | |
a15d0b36 | 107 | static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, |
808d0387 | 108 | unsigned int start, unsigned int count) |
c672528a CC |
109 | { |
110 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
111 | uint8_t *buffer_end = buffer->data + prz->buffer_size; |
112 | uint8_t *block; | |
113 | uint8_t *par; | |
9cc05ad9 CC |
114 | int ecc_block_size = prz->ecc_block_size; |
115 | int ecc_size = prz->ecc_size; | |
116 | int size = prz->ecc_block_size; | |
117 | ||
118 | if (!prz->ecc) | |
119 | return; | |
120 | ||
808d0387 CC |
121 | block = buffer->data + (start & ~(ecc_block_size - 1)); |
122 | par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size; | |
123 | ||
c672528a | 124 | do { |
9cc05ad9 | 125 | if (block + ecc_block_size > buffer_end) |
c672528a CC |
126 | size = buffer_end - block; |
127 | persistent_ram_encode_rs8(prz, block, size, par); | |
9cc05ad9 CC |
128 | block += ecc_block_size; |
129 | par += ecc_size; | |
808d0387 | 130 | } while (block < buffer->data + start + count); |
c672528a CC |
131 | } |
132 | ||
9cc05ad9 | 133 | static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) |
c672528a | 134 | { |
c672528a CC |
135 | struct persistent_ram_buffer *buffer = prz->buffer; |
136 | ||
9cc05ad9 CC |
137 | if (!prz->ecc) |
138 | return; | |
139 | ||
c672528a CC |
140 | persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), |
141 | prz->par_header); | |
c672528a CC |
142 | } |
143 | ||
9cc05ad9 | 144 | static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) |
c672528a CC |
145 | { |
146 | struct persistent_ram_buffer *buffer = prz->buffer; | |
c672528a CC |
147 | uint8_t *block; |
148 | uint8_t *par; | |
149 | ||
9cc05ad9 CC |
150 | if (!prz->ecc) |
151 | return; | |
152 | ||
c672528a CC |
153 | block = buffer->data; |
154 | par = prz->par_buffer; | |
808d0387 | 155 | while (block < buffer->data + buffer_size(prz)) { |
c672528a | 156 | int numerr; |
9cc05ad9 | 157 | int size = prz->ecc_block_size; |
c672528a CC |
158 | if (block + size > buffer->data + prz->buffer_size) |
159 | size = buffer->data + prz->buffer_size - block; | |
160 | numerr = persistent_ram_decode_rs8(prz, block, size, par); | |
161 | if (numerr > 0) { | |
9cc05ad9 | 162 | pr_devel("persistent_ram: error in block %p, %d\n", |
c672528a | 163 | block, numerr); |
c672528a CC |
164 | prz->corrected_bytes += numerr; |
165 | } else if (numerr < 0) { | |
9cc05ad9 | 166 | pr_devel("persistent_ram: uncorrectable error in block %p\n", |
c672528a | 167 | block); |
c672528a CC |
168 | prz->bad_blocks++; |
169 | } | |
9cc05ad9 CC |
170 | block += prz->ecc_block_size; |
171 | par += prz->ecc_size; | |
172 | } | |
173 | } | |
174 | ||
175 | static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, | |
176 | size_t buffer_size) | |
177 | { | |
178 | int numerr; | |
179 | struct persistent_ram_buffer *buffer = prz->buffer; | |
180 | int ecc_blocks; | |
181 | ||
182 | if (!prz->ecc) | |
183 | return 0; | |
184 | ||
185 | prz->ecc_block_size = 128; | |
186 | prz->ecc_size = 16; | |
187 | prz->ecc_symsize = 8; | |
188 | prz->ecc_poly = 0x11d; | |
189 | ||
190 | ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size); | |
191 | prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size; | |
192 | ||
193 | if (prz->buffer_size > buffer_size) { | |
194 | pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n", | |
195 | buffer_size, prz->buffer_size); | |
196 | return -EINVAL; | |
197 | } | |
198 | ||
199 | prz->par_buffer = buffer->data + prz->buffer_size; | |
200 | prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size; | |
201 | ||
202 | /* | |
203 | * first consecutive root is 0 | |
204 | * primitive element to generate roots = 1 | |
205 | */ | |
206 | prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1, | |
207 | prz->ecc_size); | |
208 | if (prz->rs_decoder == NULL) { | |
209 | pr_info("persistent_ram: init_rs failed\n"); | |
210 | return -EINVAL; | |
c672528a | 211 | } |
9cc05ad9 CC |
212 | |
213 | prz->corrected_bytes = 0; | |
214 | prz->bad_blocks = 0; | |
215 | ||
216 | numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), | |
217 | prz->par_header); | |
218 | if (numerr > 0) { | |
219 | pr_info("persistent_ram: error in header, %d\n", numerr); | |
220 | prz->corrected_bytes += numerr; | |
221 | } else if (numerr < 0) { | |
222 | pr_info("persistent_ram: uncorrectable error in header\n"); | |
223 | prz->bad_blocks++; | |
224 | } | |
225 | ||
226 | return 0; | |
227 | } | |
228 | ||
229 | ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, | |
230 | char *str, size_t len) | |
231 | { | |
232 | ssize_t ret; | |
233 | ||
234 | if (prz->corrected_bytes || prz->bad_blocks) | |
235 | ret = snprintf(str, len, "" | |
236 | "\n%d Corrected bytes, %d unrecoverable blocks\n", | |
237 | prz->corrected_bytes, prz->bad_blocks); | |
238 | else | |
239 | ret = snprintf(str, len, "\nNo errors detected\n"); | |
240 | ||
241 | return ret; | |
242 | } | |
243 | ||
a15d0b36 | 244 | static void notrace persistent_ram_update(struct persistent_ram_zone *prz, |
808d0387 | 245 | const void *s, unsigned int start, unsigned int count) |
9cc05ad9 CC |
246 | { |
247 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
248 | memcpy(buffer->data + start, s, count); |
249 | persistent_ram_update_ecc(prz, start, count); | |
9cc05ad9 CC |
250 | } |
251 | ||
252 | static void __init | |
253 | persistent_ram_save_old(struct persistent_ram_zone *prz) | |
254 | { | |
255 | struct persistent_ram_buffer *buffer = prz->buffer; | |
808d0387 CC |
256 | size_t size = buffer_size(prz); |
257 | size_t start = buffer_start(prz); | |
9cc05ad9 CC |
258 | char *dest; |
259 | ||
260 | persistent_ram_ecc_old(prz); | |
c672528a | 261 | |
808d0387 | 262 | dest = kmalloc(size, GFP_KERNEL); |
c672528a CC |
263 | if (dest == NULL) { |
264 | pr_err("persistent_ram: failed to allocate buffer\n"); | |
265 | return; | |
266 | } | |
267 | ||
268 | prz->old_log = dest; | |
808d0387 CC |
269 | prz->old_log_size = size; |
270 | memcpy(prz->old_log, &buffer->data[start], size - start); | |
271 | memcpy(prz->old_log + size - start, &buffer->data[0], start); | |
c672528a CC |
272 | } |
273 | ||
a15d0b36 | 274 | int notrace persistent_ram_write(struct persistent_ram_zone *prz, |
c672528a CC |
275 | const void *s, unsigned int count) |
276 | { | |
277 | int rem; | |
278 | int c = count; | |
808d0387 | 279 | size_t start; |
c672528a | 280 | |
808d0387 | 281 | if (unlikely(c > prz->buffer_size)) { |
c672528a CC |
282 | s += c - prz->buffer_size; |
283 | c = prz->buffer_size; | |
284 | } | |
808d0387 | 285 | |
484dd30e | 286 | buffer_size_add(prz, c); |
808d0387 CC |
287 | |
288 | start = buffer_start_add(prz, c); | |
289 | ||
290 | rem = prz->buffer_size - start; | |
291 | if (unlikely(rem < c)) { | |
292 | persistent_ram_update(prz, s, start, rem); | |
c672528a CC |
293 | s += rem; |
294 | c -= rem; | |
808d0387 | 295 | start = 0; |
c672528a | 296 | } |
808d0387 | 297 | persistent_ram_update(prz, s, start, c); |
c672528a | 298 | |
9cc05ad9 | 299 | persistent_ram_update_header_ecc(prz); |
c672528a CC |
300 | |
301 | return count; | |
302 | } | |
303 | ||
c672528a CC |
304 | size_t persistent_ram_old_size(struct persistent_ram_zone *prz) |
305 | { | |
306 | return prz->old_log_size; | |
307 | } | |
308 | ||
309 | void *persistent_ram_old(struct persistent_ram_zone *prz) | |
310 | { | |
311 | return prz->old_log; | |
312 | } | |
313 | ||
314 | void persistent_ram_free_old(struct persistent_ram_zone *prz) | |
315 | { | |
316 | kfree(prz->old_log); | |
317 | prz->old_log = NULL; | |
318 | prz->old_log_size = 0; | |
319 | } | |
320 | ||
404a6043 CC |
321 | static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, |
322 | struct persistent_ram_zone *prz) | |
c672528a | 323 | { |
404a6043 CC |
324 | struct page **pages; |
325 | phys_addr_t page_start; | |
326 | unsigned int page_count; | |
327 | pgprot_t prot; | |
328 | unsigned int i; | |
329 | ||
330 | page_start = start - offset_in_page(start); | |
331 | page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); | |
332 | ||
333 | prot = pgprot_noncached(PAGE_KERNEL); | |
334 | ||
335 | pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL); | |
336 | if (!pages) { | |
337 | pr_err("%s: Failed to allocate array for %u pages\n", __func__, | |
338 | page_count); | |
339 | return -ENOMEM; | |
340 | } | |
341 | ||
342 | for (i = 0; i < page_count; i++) { | |
343 | phys_addr_t addr = page_start + i * PAGE_SIZE; | |
344 | pages[i] = pfn_to_page(addr >> PAGE_SHIFT); | |
345 | } | |
346 | prz->vaddr = vmap(pages, page_count, VM_MAP, prot); | |
347 | kfree(pages); | |
348 | if (!prz->vaddr) { | |
349 | pr_err("%s: Failed to map %u pages\n", __func__, page_count); | |
350 | return -ENOMEM; | |
351 | } | |
352 | ||
353 | prz->buffer = prz->vaddr + offset_in_page(start); | |
354 | prz->buffer_size = size - sizeof(struct persistent_ram_buffer); | |
355 | ||
356 | return 0; | |
357 | } | |
358 | ||
359 | static int __init persistent_ram_buffer_init(const char *name, | |
360 | struct persistent_ram_zone *prz) | |
361 | { | |
362 | int i; | |
363 | struct persistent_ram *ram; | |
364 | struct persistent_ram_descriptor *desc; | |
365 | phys_addr_t start; | |
366 | ||
367 | list_for_each_entry(ram, &persistent_ram_list, node) { | |
368 | start = ram->start; | |
369 | for (i = 0; i < ram->num_descs; i++) { | |
370 | desc = &ram->descs[i]; | |
371 | if (!strcmp(desc->name, name)) | |
372 | return persistent_ram_buffer_map(start, | |
373 | desc->size, prz); | |
374 | start += desc->size; | |
375 | } | |
376 | } | |
377 | ||
378 | return -EINVAL; | |
379 | } | |
380 | ||
bb4206f2 | 381 | static int __init persistent_ram_post_init(struct persistent_ram_zone *prz, bool ecc) |
404a6043 | 382 | { |
bb4206f2 | 383 | int ret; |
c672528a | 384 | |
9cc05ad9 | 385 | prz->ecc = ecc; |
bb4206f2 | 386 | |
404a6043 | 387 | ret = persistent_ram_init_ecc(prz, prz->buffer_size); |
9cc05ad9 | 388 | if (ret) |
bb4206f2 | 389 | return ret; |
c672528a | 390 | |
404a6043 | 391 | if (prz->buffer->sig == PERSISTENT_RAM_SIG) { |
808d0387 CC |
392 | if (buffer_size(prz) > prz->buffer_size || |
393 | buffer_start(prz) > buffer_size(prz)) | |
394 | pr_info("persistent_ram: found existing invalid buffer," | |
f56d711b | 395 | " size %zu, start %zu\n", |
808d0387 | 396 | buffer_size(prz), buffer_start(prz)); |
c672528a | 397 | else { |
808d0387 | 398 | pr_info("persistent_ram: found existing buffer," |
f56d711b | 399 | " size %zu, start %zu\n", |
808d0387 | 400 | buffer_size(prz), buffer_start(prz)); |
c672528a CC |
401 | persistent_ram_save_old(prz); |
402 | } | |
403 | } else { | |
808d0387 CC |
404 | pr_info("persistent_ram: no valid data in buffer" |
405 | " (sig = 0x%08x)\n", prz->buffer->sig); | |
c672528a CC |
406 | } |
407 | ||
404a6043 | 408 | prz->buffer->sig = PERSISTENT_RAM_SIG; |
808d0387 CC |
409 | atomic_set(&prz->buffer->start, 0); |
410 | atomic_set(&prz->buffer->size, 0); | |
c672528a | 411 | |
bb4206f2 AV |
412 | return 0; |
413 | } | |
414 | ||
415 | static __init | |
416 | struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc) | |
417 | { | |
418 | struct persistent_ram_zone *prz; | |
419 | int ret = -ENOMEM; | |
420 | ||
421 | prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); | |
422 | if (!prz) { | |
423 | pr_err("persistent_ram: failed to allocate persistent ram zone\n"); | |
424 | goto err; | |
425 | } | |
426 | ||
427 | ret = persistent_ram_buffer_init(dev_name(dev), prz); | |
428 | if (ret) { | |
429 | pr_err("persistent_ram: failed to initialize buffer\n"); | |
430 | goto err; | |
431 | } | |
432 | ||
433 | persistent_ram_post_init(prz, ecc); | |
434 | ||
404a6043 | 435 | return prz; |
474a8988 JJ |
436 | err: |
437 | kfree(prz); | |
438 | return ERR_PTR(ret); | |
404a6043 | 439 | } |
c672528a | 440 | |
404a6043 CC |
441 | struct persistent_ram_zone * __init |
442 | persistent_ram_init_ringbuffer(struct device *dev, bool ecc) | |
443 | { | |
444 | return __persistent_ram_init(dev, ecc); | |
c672528a CC |
445 | } |
446 | ||
404a6043 | 447 | int __init persistent_ram_early_init(struct persistent_ram *ram) |
c672528a | 448 | { |
404a6043 CC |
449 | int ret; |
450 | ||
451 | ret = memblock_reserve(ram->start, ram->size); | |
452 | if (ret) { | |
453 | pr_err("Failed to reserve persistent memory from %08lx-%08lx\n", | |
454 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
455 | return ret; | |
456 | } | |
457 | ||
458 | list_add_tail(&ram->node, &persistent_ram_list); | |
459 | ||
460 | pr_info("Initialized persistent memory from %08lx-%08lx\n", | |
461 | (long)ram->start, (long)(ram->start + ram->size - 1)); | |
462 | ||
463 | return 0; | |
c672528a | 464 | } |