2 * This file contains shadow memory manipulation code.
4 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <a.ryabinin@samsung.com>
7 * Some of code borrowed from https://github.com/xairy/linux by
8 * Andrey Konovalov <adech.fo@gmail.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as
12 * published by the Free Software Foundation.
16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17 #define DISABLE_BRANCH_PROFILING
19 #include <linux/export.h>
20 #include <linux/init.h>
21 #include <linux/kernel.h>
22 #include <linux/memblock.h>
24 #include <linux/printk.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
27 #include <linux/stacktrace.h>
28 #include <linux/string.h>
29 #include <linux/types.h>
30 #include <linux/kasan.h>
35 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
36 * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
38 static void kasan_poison_shadow(const void *address
, size_t size
, u8 value
)
40 void *shadow_start
, *shadow_end
;
42 shadow_start
= kasan_mem_to_shadow(address
);
43 shadow_end
= kasan_mem_to_shadow(address
+ size
);
45 memset(shadow_start
, value
, shadow_end
- shadow_start
);
48 void kasan_unpoison_shadow(const void *address
, size_t size
)
50 kasan_poison_shadow(address
, size
, 0);
52 if (size
& KASAN_SHADOW_MASK
) {
53 u8
*shadow
= (u8
*)kasan_mem_to_shadow(address
+ size
);
54 *shadow
= size
& KASAN_SHADOW_MASK
;
60 * All functions below always inlined so compiler could
61 * perform better optimizations in each of __asan_loadX/__assn_storeX
62 * depending on memory access size X.
65 static __always_inline
bool memory_is_poisoned_1(unsigned long addr
)
67 s8 shadow_value
= *(s8
*)kasan_mem_to_shadow((void *)addr
);
69 if (unlikely(shadow_value
)) {
70 s8 last_accessible_byte
= addr
& KASAN_SHADOW_MASK
;
71 return unlikely(last_accessible_byte
>= shadow_value
);
77 static __always_inline
bool memory_is_poisoned_2(unsigned long addr
)
79 u16
*shadow_addr
= (u16
*)kasan_mem_to_shadow((void *)addr
);
81 if (unlikely(*shadow_addr
)) {
82 if (memory_is_poisoned_1(addr
+ 1))
85 if (likely(((addr
+ 1) & KASAN_SHADOW_MASK
) != 0))
88 return unlikely(*(u8
*)shadow_addr
);
94 static __always_inline
bool memory_is_poisoned_4(unsigned long addr
)
96 u16
*shadow_addr
= (u16
*)kasan_mem_to_shadow((void *)addr
);
98 if (unlikely(*shadow_addr
)) {
99 if (memory_is_poisoned_1(addr
+ 3))
102 if (likely(((addr
+ 3) & KASAN_SHADOW_MASK
) >= 3))
105 return unlikely(*(u8
*)shadow_addr
);
111 static __always_inline
bool memory_is_poisoned_8(unsigned long addr
)
113 u16
*shadow_addr
= (u16
*)kasan_mem_to_shadow((void *)addr
);
115 if (unlikely(*shadow_addr
)) {
116 if (memory_is_poisoned_1(addr
+ 7))
119 if (likely(((addr
+ 7) & KASAN_SHADOW_MASK
) >= 7))
122 return unlikely(*(u8
*)shadow_addr
);
128 static __always_inline
bool memory_is_poisoned_16(unsigned long addr
)
130 u32
*shadow_addr
= (u32
*)kasan_mem_to_shadow((void *)addr
);
132 if (unlikely(*shadow_addr
)) {
133 u16 shadow_first_bytes
= *(u16
*)shadow_addr
;
134 s8 last_byte
= (addr
+ 15) & KASAN_SHADOW_MASK
;
136 if (unlikely(shadow_first_bytes
))
139 if (likely(!last_byte
))
142 return memory_is_poisoned_1(addr
+ 15);
148 static __always_inline
unsigned long bytes_is_zero(const u8
*start
,
152 if (unlikely(*start
))
153 return (unsigned long)start
;
161 static __always_inline
unsigned long memory_is_zero(const void *start
,
166 unsigned int prefix
= (unsigned long)start
% 8;
168 if (end
- start
<= 16)
169 return bytes_is_zero(start
, end
- start
);
173 ret
= bytes_is_zero(start
, prefix
);
179 words
= (end
- start
) / 8;
181 if (unlikely(*(u64
*)start
))
182 return bytes_is_zero(start
, 8);
187 return bytes_is_zero(start
, (end
- start
) % 8);
190 static __always_inline
bool memory_is_poisoned_n(unsigned long addr
,
195 ret
= memory_is_zero(kasan_mem_to_shadow((void *)addr
),
196 kasan_mem_to_shadow((void *)addr
+ size
- 1) + 1);
199 unsigned long last_byte
= addr
+ size
- 1;
200 s8
*last_shadow
= (s8
*)kasan_mem_to_shadow((void *)last_byte
);
202 if (unlikely(ret
!= (unsigned long)last_shadow
||
203 ((last_byte
& KASAN_SHADOW_MASK
) >= *last_shadow
)))
209 static __always_inline
bool memory_is_poisoned(unsigned long addr
, size_t size
)
211 if (__builtin_constant_p(size
)) {
214 return memory_is_poisoned_1(addr
);
216 return memory_is_poisoned_2(addr
);
218 return memory_is_poisoned_4(addr
);
220 return memory_is_poisoned_8(addr
);
222 return memory_is_poisoned_16(addr
);
228 return memory_is_poisoned_n(addr
, size
);
232 static __always_inline
void check_memory_region(unsigned long addr
,
233 size_t size
, bool write
)
235 struct kasan_access_info info
;
237 if (unlikely(size
== 0))
240 if (unlikely((void *)addr
<
241 kasan_shadow_to_mem((void *)KASAN_SHADOW_START
))) {
242 info
.access_addr
= (void *)addr
;
243 info
.access_size
= size
;
244 info
.is_write
= write
;
246 kasan_report_user_access(&info
);
250 if (likely(!memory_is_poisoned(addr
, size
)))
253 kasan_report(addr
, size
, write
, _RET_IP_
);
256 #define DEFINE_ASAN_LOAD_STORE(size) \
257 void __asan_load##size(unsigned long addr) \
259 check_memory_region(addr, size, false); \
261 EXPORT_SYMBOL(__asan_load##size); \
262 __alias(__asan_load##size) \
263 void __asan_load##size##_noabort(unsigned long); \
264 EXPORT_SYMBOL(__asan_load##size##_noabort); \
265 void __asan_store##size(unsigned long addr) \
267 check_memory_region(addr, size, true); \
269 EXPORT_SYMBOL(__asan_store##size); \
270 __alias(__asan_store##size) \
271 void __asan_store##size##_noabort(unsigned long); \
272 EXPORT_SYMBOL(__asan_store##size##_noabort)
274 DEFINE_ASAN_LOAD_STORE(1);
275 DEFINE_ASAN_LOAD_STORE(2);
276 DEFINE_ASAN_LOAD_STORE(4);
277 DEFINE_ASAN_LOAD_STORE(8);
278 DEFINE_ASAN_LOAD_STORE(16);
280 void __asan_loadN(unsigned long addr
, size_t size
)
282 check_memory_region(addr
, size
, false);
284 EXPORT_SYMBOL(__asan_loadN
);
286 __alias(__asan_loadN
)
287 void __asan_loadN_noabort(unsigned long, size_t);
288 EXPORT_SYMBOL(__asan_loadN_noabort
);
290 void __asan_storeN(unsigned long addr
, size_t size
)
292 check_memory_region(addr
, size
, true);
294 EXPORT_SYMBOL(__asan_storeN
);
296 __alias(__asan_storeN
)
297 void __asan_storeN_noabort(unsigned long, size_t);
298 EXPORT_SYMBOL(__asan_storeN_noabort
);
300 /* to shut up compiler complaints */
301 void __asan_handle_no_return(void) {}
302 EXPORT_SYMBOL(__asan_handle_no_return
);
This page took 0.067533 seconds and 5 git commands to generate.