2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #include <linux/device.h>
16 #include <linux/err.h>
17 #include <linux/errno.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/memblock.h>
23 #include <linux/rslib.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include "persistent_ram.h"
28 struct persistent_ram_buffer
{
35 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
37 static __initdata
LIST_HEAD(persistent_ram_list
);
39 static inline size_t buffer_size(struct persistent_ram_zone
*prz
)
41 return atomic_read(&prz
->buffer
->size
);
44 static inline size_t buffer_start(struct persistent_ram_zone
*prz
)
46 return atomic_read(&prz
->buffer
->start
);
49 /* increase and wrap the start pointer, returning the old value */
50 static inline size_t buffer_start_add(struct persistent_ram_zone
*prz
, size_t a
)
56 old
= atomic_read(&prz
->buffer
->start
);
58 while (unlikely(new > prz
->buffer_size
))
59 new -= prz
->buffer_size
;
60 } while (atomic_cmpxchg(&prz
->buffer
->start
, old
, new) != old
);
65 /* increase the size counter until it hits the max size */
66 static inline void buffer_size_add(struct persistent_ram_zone
*prz
, size_t a
)
71 if (atomic_read(&prz
->buffer
->size
) == prz
->buffer_size
)
75 old
= atomic_read(&prz
->buffer
->size
);
77 if (new > prz
->buffer_size
)
78 new = prz
->buffer_size
;
79 } while (atomic_cmpxchg(&prz
->buffer
->size
, old
, new) != old
);
82 static void notrace
persistent_ram_encode_rs8(struct persistent_ram_zone
*prz
,
83 uint8_t *data
, size_t len
, uint8_t *ecc
)
86 uint16_t par
[prz
->ecc_size
];
88 /* Initialize the parity buffer */
89 memset(par
, 0, sizeof(par
));
90 encode_rs8(prz
->rs_decoder
, data
, len
, par
, 0);
91 for (i
= 0; i
< prz
->ecc_size
; i
++)
95 static int persistent_ram_decode_rs8(struct persistent_ram_zone
*prz
,
96 void *data
, size_t len
, uint8_t *ecc
)
99 uint16_t par
[prz
->ecc_size
];
101 for (i
= 0; i
< prz
->ecc_size
; i
++)
103 return decode_rs8(prz
->rs_decoder
, data
, par
, len
,
104 NULL
, 0, NULL
, 0, NULL
);
107 static void notrace
persistent_ram_update_ecc(struct persistent_ram_zone
*prz
,
108 unsigned int start
, unsigned int count
)
110 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
111 uint8_t *buffer_end
= buffer
->data
+ prz
->buffer_size
;
114 int ecc_block_size
= prz
->ecc_block_size
;
115 int ecc_size
= prz
->ecc_size
;
116 int size
= prz
->ecc_block_size
;
121 block
= buffer
->data
+ (start
& ~(ecc_block_size
- 1));
122 par
= prz
->par_buffer
+ (start
/ ecc_block_size
) * prz
->ecc_size
;
125 if (block
+ ecc_block_size
> buffer_end
)
126 size
= buffer_end
- block
;
127 persistent_ram_encode_rs8(prz
, block
, size
, par
);
128 block
+= ecc_block_size
;
130 } while (block
< buffer
->data
+ start
+ count
);
133 static void persistent_ram_update_header_ecc(struct persistent_ram_zone
*prz
)
135 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
140 persistent_ram_encode_rs8(prz
, (uint8_t *)buffer
, sizeof(*buffer
),
144 static void persistent_ram_ecc_old(struct persistent_ram_zone
*prz
)
146 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
153 block
= buffer
->data
;
154 par
= prz
->par_buffer
;
155 while (block
< buffer
->data
+ buffer_size(prz
)) {
157 int size
= prz
->ecc_block_size
;
158 if (block
+ size
> buffer
->data
+ prz
->buffer_size
)
159 size
= buffer
->data
+ prz
->buffer_size
- block
;
160 numerr
= persistent_ram_decode_rs8(prz
, block
, size
, par
);
162 pr_devel("persistent_ram: error in block %p, %d\n",
164 prz
->corrected_bytes
+= numerr
;
165 } else if (numerr
< 0) {
166 pr_devel("persistent_ram: uncorrectable error in block %p\n",
170 block
+= prz
->ecc_block_size
;
171 par
+= prz
->ecc_size
;
175 static int persistent_ram_init_ecc(struct persistent_ram_zone
*prz
,
179 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
185 prz
->ecc_block_size
= 128;
187 prz
->ecc_symsize
= 8;
188 prz
->ecc_poly
= 0x11d;
190 ecc_blocks
= DIV_ROUND_UP(prz
->buffer_size
, prz
->ecc_block_size
);
191 prz
->buffer_size
-= (ecc_blocks
+ 1) * prz
->ecc_size
;
193 if (prz
->buffer_size
> buffer_size
) {
194 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n",
195 buffer_size
, prz
->buffer_size
);
199 prz
->par_buffer
= buffer
->data
+ prz
->buffer_size
;
200 prz
->par_header
= prz
->par_buffer
+ ecc_blocks
* prz
->ecc_size
;
203 * first consecutive root is 0
204 * primitive element to generate roots = 1
206 prz
->rs_decoder
= init_rs(prz
->ecc_symsize
, prz
->ecc_poly
, 0, 1,
208 if (prz
->rs_decoder
== NULL
) {
209 pr_info("persistent_ram: init_rs failed\n");
213 prz
->corrected_bytes
= 0;
216 numerr
= persistent_ram_decode_rs8(prz
, buffer
, sizeof(*buffer
),
219 pr_info("persistent_ram: error in header, %d\n", numerr
);
220 prz
->corrected_bytes
+= numerr
;
221 } else if (numerr
< 0) {
222 pr_info("persistent_ram: uncorrectable error in header\n");
229 ssize_t
persistent_ram_ecc_string(struct persistent_ram_zone
*prz
,
230 char *str
, size_t len
)
234 if (prz
->corrected_bytes
|| prz
->bad_blocks
)
235 ret
= snprintf(str
, len
, ""
236 "\n%d Corrected bytes, %d unrecoverable blocks\n",
237 prz
->corrected_bytes
, prz
->bad_blocks
);
239 ret
= snprintf(str
, len
, "\nNo errors detected\n");
244 static void notrace
persistent_ram_update(struct persistent_ram_zone
*prz
,
245 const void *s
, unsigned int start
, unsigned int count
)
247 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
248 memcpy(buffer
->data
+ start
, s
, count
);
249 persistent_ram_update_ecc(prz
, start
, count
);
253 persistent_ram_save_old(struct persistent_ram_zone
*prz
)
255 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
256 size_t size
= buffer_size(prz
);
257 size_t start
= buffer_start(prz
);
260 persistent_ram_ecc_old(prz
);
262 dest
= kmalloc(size
, GFP_KERNEL
);
264 pr_err("persistent_ram: failed to allocate buffer\n");
269 prz
->old_log_size
= size
;
270 memcpy(prz
->old_log
, &buffer
->data
[start
], size
- start
);
271 memcpy(prz
->old_log
+ size
- start
, &buffer
->data
[0], start
);
274 int notrace
persistent_ram_write(struct persistent_ram_zone
*prz
,
275 const void *s
, unsigned int count
)
281 if (unlikely(c
> prz
->buffer_size
)) {
282 s
+= c
- prz
->buffer_size
;
283 c
= prz
->buffer_size
;
286 buffer_size_add(prz
, c
);
288 start
= buffer_start_add(prz
, c
);
290 rem
= prz
->buffer_size
- start
;
291 if (unlikely(rem
< c
)) {
292 persistent_ram_update(prz
, s
, start
, rem
);
297 persistent_ram_update(prz
, s
, start
, c
);
299 persistent_ram_update_header_ecc(prz
);
304 size_t persistent_ram_old_size(struct persistent_ram_zone
*prz
)
306 return prz
->old_log_size
;
309 void *persistent_ram_old(struct persistent_ram_zone
*prz
)
314 void persistent_ram_free_old(struct persistent_ram_zone
*prz
)
318 prz
->old_log_size
= 0;
321 static int persistent_ram_buffer_map(phys_addr_t start
, phys_addr_t size
,
322 struct persistent_ram_zone
*prz
)
325 phys_addr_t page_start
;
326 unsigned int page_count
;
330 page_start
= start
- offset_in_page(start
);
331 page_count
= DIV_ROUND_UP(size
+ offset_in_page(start
), PAGE_SIZE
);
333 prot
= pgprot_noncached(PAGE_KERNEL
);
335 pages
= kmalloc(sizeof(struct page
*) * page_count
, GFP_KERNEL
);
337 pr_err("%s: Failed to allocate array for %u pages\n", __func__
,
342 for (i
= 0; i
< page_count
; i
++) {
343 phys_addr_t addr
= page_start
+ i
* PAGE_SIZE
;
344 pages
[i
] = pfn_to_page(addr
>> PAGE_SHIFT
);
346 prz
->vaddr
= vmap(pages
, page_count
, VM_MAP
, prot
);
349 pr_err("%s: Failed to map %u pages\n", __func__
, page_count
);
353 prz
->buffer
= prz
->vaddr
+ offset_in_page(start
);
354 prz
->buffer_size
= size
- sizeof(struct persistent_ram_buffer
);
359 static int __init
persistent_ram_buffer_init(const char *name
,
360 struct persistent_ram_zone
*prz
)
363 struct persistent_ram
*ram
;
364 struct persistent_ram_descriptor
*desc
;
367 list_for_each_entry(ram
, &persistent_ram_list
, node
) {
369 for (i
= 0; i
< ram
->num_descs
; i
++) {
370 desc
= &ram
->descs
[i
];
371 if (!strcmp(desc
->name
, name
))
372 return persistent_ram_buffer_map(start
,
382 struct persistent_ram_zone
*__persistent_ram_init(struct device
*dev
, bool ecc
)
384 struct persistent_ram_zone
*prz
;
387 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
389 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
393 ret
= persistent_ram_buffer_init(dev_name(dev
), prz
);
395 pr_err("persistent_ram: failed to initialize buffer\n");
400 ret
= persistent_ram_init_ecc(prz
, prz
->buffer_size
);
404 if (prz
->buffer
->sig
== PERSISTENT_RAM_SIG
) {
405 if (buffer_size(prz
) > prz
->buffer_size
||
406 buffer_start(prz
) > buffer_size(prz
))
407 pr_info("persistent_ram: found existing invalid buffer,"
408 " size %zu, start %zu\n",
409 buffer_size(prz
), buffer_start(prz
));
411 pr_info("persistent_ram: found existing buffer,"
412 " size %zu, start %zu\n",
413 buffer_size(prz
), buffer_start(prz
));
414 persistent_ram_save_old(prz
);
417 pr_info("persistent_ram: no valid data in buffer"
418 " (sig = 0x%08x)\n", prz
->buffer
->sig
);
421 prz
->buffer
->sig
= PERSISTENT_RAM_SIG
;
422 atomic_set(&prz
->buffer
->start
, 0);
423 atomic_set(&prz
->buffer
->size
, 0);
431 struct persistent_ram_zone
* __init
432 persistent_ram_init_ringbuffer(struct device
*dev
, bool ecc
)
434 return __persistent_ram_init(dev
, ecc
);
437 int __init
persistent_ram_early_init(struct persistent_ram
*ram
)
441 ret
= memblock_reserve(ram
->start
, ram
->size
);
443 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n",
444 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));
448 list_add_tail(&ram
->node
, &persistent_ram_list
);
450 pr_info("Initialized persistent memory from %08lx-%08lx\n",
451 (long)ram
->start
, (long)(ram
->start
+ ram
->size
- 1));