2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #include <linux/device.h>
16 #include <linux/err.h>
17 #include <linux/errno.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/memblock.h>
23 #include <linux/rslib.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <linux/pstore_ram.h>
29 struct persistent_ram_buffer
{
36 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
38 static __initdata
LIST_HEAD(persistent_ram_list
);
40 static inline size_t buffer_size(struct persistent_ram_zone
*prz
)
42 return atomic_read(&prz
->buffer
->size
);
45 static inline size_t buffer_start(struct persistent_ram_zone
*prz
)
47 return atomic_read(&prz
->buffer
->start
);
50 /* increase and wrap the start pointer, returning the old value */
51 static inline size_t buffer_start_add(struct persistent_ram_zone
*prz
, size_t a
)
57 old
= atomic_read(&prz
->buffer
->start
);
59 while (unlikely(new > prz
->buffer_size
))
60 new -= prz
->buffer_size
;
61 } while (atomic_cmpxchg(&prz
->buffer
->start
, old
, new) != old
);
66 /* increase the size counter until it hits the max size */
67 static inline void buffer_size_add(struct persistent_ram_zone
*prz
, size_t a
)
72 if (atomic_read(&prz
->buffer
->size
) == prz
->buffer_size
)
76 old
= atomic_read(&prz
->buffer
->size
);
78 if (new > prz
->buffer_size
)
79 new = prz
->buffer_size
;
80 } while (atomic_cmpxchg(&prz
->buffer
->size
, old
, new) != old
);
83 static void notrace
persistent_ram_encode_rs8(struct persistent_ram_zone
*prz
,
84 uint8_t *data
, size_t len
, uint8_t *ecc
)
87 uint16_t par
[prz
->ecc_size
];
89 /* Initialize the parity buffer */
90 memset(par
, 0, sizeof(par
));
91 encode_rs8(prz
->rs_decoder
, data
, len
, par
, 0);
92 for (i
= 0; i
< prz
->ecc_size
; i
++)
96 static int persistent_ram_decode_rs8(struct persistent_ram_zone
*prz
,
97 void *data
, size_t len
, uint8_t *ecc
)
100 uint16_t par
[prz
->ecc_size
];
102 for (i
= 0; i
< prz
->ecc_size
; i
++)
104 return decode_rs8(prz
->rs_decoder
, data
, par
, len
,
105 NULL
, 0, NULL
, 0, NULL
);
108 static void notrace
persistent_ram_update_ecc(struct persistent_ram_zone
*prz
,
109 unsigned int start
, unsigned int count
)
111 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
112 uint8_t *buffer_end
= buffer
->data
+ prz
->buffer_size
;
115 int ecc_block_size
= prz
->ecc_block_size
;
116 int ecc_size
= prz
->ecc_size
;
117 int size
= prz
->ecc_block_size
;
122 block
= buffer
->data
+ (start
& ~(ecc_block_size
- 1));
123 par
= prz
->par_buffer
+ (start
/ ecc_block_size
) * prz
->ecc_size
;
126 if (block
+ ecc_block_size
> buffer_end
)
127 size
= buffer_end
- block
;
128 persistent_ram_encode_rs8(prz
, block
, size
, par
);
129 block
+= ecc_block_size
;
131 } while (block
< buffer
->data
+ start
+ count
);
134 static void persistent_ram_update_header_ecc(struct persistent_ram_zone
*prz
)
136 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
141 persistent_ram_encode_rs8(prz
, (uint8_t *)buffer
, sizeof(*buffer
),
145 static void persistent_ram_ecc_old(struct persistent_ram_zone
*prz
)
147 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
154 block
= buffer
->data
;
155 par
= prz
->par_buffer
;
156 while (block
< buffer
->data
+ buffer_size(prz
)) {
158 int size
= prz
->ecc_block_size
;
159 if (block
+ size
> buffer
->data
+ prz
->buffer_size
)
160 size
= buffer
->data
+ prz
->buffer_size
- block
;
161 numerr
= persistent_ram_decode_rs8(prz
, block
, size
, par
);
163 pr_devel("persistent_ram: error in block %p, %d\n",
165 prz
->corrected_bytes
+= numerr
;
166 } else if (numerr
< 0) {
167 pr_devel("persistent_ram: uncorrectable error in block %p\n",
171 block
+= prz
->ecc_block_size
;
172 par
+= prz
->ecc_size
;
176 static int persistent_ram_init_ecc(struct persistent_ram_zone
*prz
,
180 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
186 prz
->ecc_block_size
= 128;
188 prz
->ecc_symsize
= 8;
189 prz
->ecc_poly
= 0x11d;
191 ecc_blocks
= DIV_ROUND_UP(prz
->buffer_size
, prz
->ecc_block_size
);
192 prz
->buffer_size
-= (ecc_blocks
+ 1) * prz
->ecc_size
;
194 if (prz
->buffer_size
> buffer_size
) {
195 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n",
196 buffer_size
, prz
->buffer_size
);
200 prz
->par_buffer
= buffer
->data
+ prz
->buffer_size
;
201 prz
->par_header
= prz
->par_buffer
+ ecc_blocks
* prz
->ecc_size
;
204 * first consecutive root is 0
205 * primitive element to generate roots = 1
207 prz
->rs_decoder
= init_rs(prz
->ecc_symsize
, prz
->ecc_poly
, 0, 1,
209 if (prz
->rs_decoder
== NULL
) {
210 pr_info("persistent_ram: init_rs failed\n");
214 prz
->corrected_bytes
= 0;
217 numerr
= persistent_ram_decode_rs8(prz
, buffer
, sizeof(*buffer
),
220 pr_info("persistent_ram: error in header, %d\n", numerr
);
221 prz
->corrected_bytes
+= numerr
;
222 } else if (numerr
< 0) {
223 pr_info("persistent_ram: uncorrectable error in header\n");
230 ssize_t
persistent_ram_ecc_string(struct persistent_ram_zone
*prz
,
231 char *str
, size_t len
)
235 if (prz
->corrected_bytes
|| prz
->bad_blocks
)
236 ret
= snprintf(str
, len
, ""
237 "\n%d Corrected bytes, %d unrecoverable blocks\n",
238 prz
->corrected_bytes
, prz
->bad_blocks
);
240 ret
= snprintf(str
, len
, "\nNo errors detected\n");
245 static void notrace
persistent_ram_update(struct persistent_ram_zone
*prz
,
246 const void *s
, unsigned int start
, unsigned int count
)
248 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
249 memcpy(buffer
->data
+ start
, s
, count
);
250 persistent_ram_update_ecc(prz
, start
, count
);
253 void persistent_ram_save_old(struct persistent_ram_zone
*prz
)
255 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
256 size_t size
= buffer_size(prz
);
257 size_t start
= buffer_start(prz
);
263 persistent_ram_ecc_old(prz
);
264 prz
->old_log
= kmalloc(size
, GFP_KERNEL
);
267 pr_err("persistent_ram: failed to allocate buffer\n");
271 prz
->old_log_size
= size
;
272 memcpy(prz
->old_log
, &buffer
->data
[start
], size
- start
);
273 memcpy(prz
->old_log
+ size
- start
, &buffer
->data
[0], start
);
276 int notrace
persistent_ram_write(struct persistent_ram_zone
*prz
,
277 const void *s
, unsigned int count
)
283 if (unlikely(c
> prz
->buffer_size
)) {
284 s
+= c
- prz
->buffer_size
;
285 c
= prz
->buffer_size
;
288 buffer_size_add(prz
, c
);
290 start
= buffer_start_add(prz
, c
);
292 rem
= prz
->buffer_size
- start
;
293 if (unlikely(rem
< c
)) {
294 persistent_ram_update(prz
, s
, start
, rem
);
299 persistent_ram_update(prz
, s
, start
, c
);
301 persistent_ram_update_header_ecc(prz
);
306 size_t persistent_ram_old_size(struct persistent_ram_zone
*prz
)
308 return prz
->old_log_size
;
311 void *persistent_ram_old(struct persistent_ram_zone
*prz
)
316 void persistent_ram_free_old(struct persistent_ram_zone
*prz
)
320 prz
->old_log_size
= 0;
323 void persistent_ram_zap(struct persistent_ram_zone
*prz
)
325 atomic_set(&prz
->buffer
->start
, 0);
326 atomic_set(&prz
->buffer
->size
, 0);
327 persistent_ram_update_header_ecc(prz
);
330 static void *persistent_ram_vmap(phys_addr_t start
, size_t size
)
333 phys_addr_t page_start
;
334 unsigned int page_count
;
339 page_start
= start
- offset_in_page(start
);
340 page_count
= DIV_ROUND_UP(size
+ offset_in_page(start
), PAGE_SIZE
);
342 prot
= pgprot_noncached(PAGE_KERNEL
);
344 pages
= kmalloc(sizeof(struct page
*) * page_count
, GFP_KERNEL
);
346 pr_err("%s: Failed to allocate array for %u pages\n", __func__
,
351 for (i
= 0; i
< page_count
; i
++) {
352 phys_addr_t addr
= page_start
+ i
* PAGE_SIZE
;
353 pages
[i
] = pfn_to_page(addr
>> PAGE_SHIFT
);
355 vaddr
= vmap(pages
, page_count
, VM_MAP
, prot
);
361 static void *persistent_ram_iomap(phys_addr_t start
, size_t size
)
363 if (!request_mem_region(start
, size
, "persistent_ram")) {
364 pr_err("request mem region (0x%llx@0x%llx) failed\n",
365 (unsigned long long)size
, (unsigned long long)start
);
369 return ioremap(start
, size
);
372 static int persistent_ram_buffer_map(phys_addr_t start
, phys_addr_t size
,
373 struct persistent_ram_zone
*prz
)
378 if (pfn_valid(start
>> PAGE_SHIFT
))
379 prz
->vaddr
= persistent_ram_vmap(start
, size
);
381 prz
->vaddr
= persistent_ram_iomap(start
, size
);
384 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__
,
385 (unsigned long long)size
, (unsigned long long)start
);
389 prz
->buffer
= prz
->vaddr
+ offset_in_page(start
);
390 prz
->buffer_size
= size
- sizeof(struct persistent_ram_buffer
);
395 static int __init
persistent_ram_post_init(struct persistent_ram_zone
*prz
, bool ecc
)
401 ret
= persistent_ram_init_ecc(prz
, prz
->buffer_size
);
405 if (prz
->buffer
->sig
== PERSISTENT_RAM_SIG
) {
406 if (buffer_size(prz
) > prz
->buffer_size
||
407 buffer_start(prz
) > buffer_size(prz
))
408 pr_info("persistent_ram: found existing invalid buffer,"
409 " size %zu, start %zu\n",
410 buffer_size(prz
), buffer_start(prz
));
412 pr_info("persistent_ram: found existing buffer,"
413 " size %zu, start %zu\n",
414 buffer_size(prz
), buffer_start(prz
));
415 persistent_ram_save_old(prz
);
419 pr_info("persistent_ram: no valid data in buffer"
420 " (sig = 0x%08x)\n", prz
->buffer
->sig
);
423 prz
->buffer
->sig
= PERSISTENT_RAM_SIG
;
424 persistent_ram_zap(prz
);
429 void persistent_ram_free(struct persistent_ram_zone
*prz
)
431 if (pfn_valid(prz
->paddr
>> PAGE_SHIFT
)) {
435 release_mem_region(prz
->paddr
, prz
->size
);
437 persistent_ram_free_old(prz
);
441 struct persistent_ram_zone
* __init
persistent_ram_new(phys_addr_t start
,
445 struct persistent_ram_zone
*prz
;
448 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
450 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
454 ret
= persistent_ram_buffer_map(start
, size
, prz
);
458 persistent_ram_post_init(prz
, ecc
);
467 static int __init
persistent_ram_buffer_init(const char *name
,
468 struct persistent_ram_zone
*prz
)
471 struct persistent_ram
*ram
;
472 struct persistent_ram_descriptor
*desc
;
475 list_for_each_entry(ram
, &persistent_ram_list
, node
) {
477 for (i
= 0; i
< ram
->num_descs
; i
++) {
478 desc
= &ram
->descs
[i
];
479 if (!strcmp(desc
->name
, name
))
480 return persistent_ram_buffer_map(start
,
490 struct persistent_ram_zone
*__persistent_ram_init(struct device
*dev
, bool ecc
)
492 struct persistent_ram_zone
*prz
;
495 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
497 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
501 ret
= persistent_ram_buffer_init(dev_name(dev
), prz
);
503 pr_err("persistent_ram: failed to initialize buffer\n");
507 persistent_ram_post_init(prz
, ecc
);
515 struct persistent_ram_zone
* __init
516 persistent_ram_init_ringbuffer(struct device
*dev
, bool ecc
)
518 return __persistent_ram_init(dev
, ecc
);
521 int __init
persistent_ram_early_init(struct persistent_ram
*ram
)
525 ret
= memblock_reserve(ram
->start
, ram
->size
);
527 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n",
528 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));
532 list_add_tail(&ram
->node
, &persistent_ram_list
);
534 pr_info("Initialized persistent memory from %08lx-%08lx\n",
535 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));