[deliverable/linux.git] / mm / kasan / kasan.c

/*
 * This file contains shadow memory manipulation code.
 *
 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
 * Author: Andrey Ryabinin <a.ryabinin@samsung.com>
 *
 * Some of code borrowed from https://github.com/xairy/linux by
 *        Andrey Konovalov <adech.fo@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define DISABLE_BRANCH_PROFILING

#include <linux/export.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/printk.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/kasan.h>

#include "kasan.h"

/*
 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
 * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
 */
static void kasan_poison_shadow(const void *address, size_t size, u8 value)
{
	void *shadow_start, *shadow_end;

	shadow_start = kasan_mem_to_shadow(address);
	shadow_end = kasan_mem_to_shadow(address + size);

	memset(shadow_start, value, shadow_end - shadow_start);
}

void kasan_unpoison_shadow(const void *address, size_t size)
{
	kasan_poison_shadow(address, size, 0);

	if (size & KASAN_SHADOW_MASK) {
		u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size);
		*shadow = size & KASAN_SHADOW_MASK;
	}
}


/*
 * All functions below always inlined so compiler could
 * perform better optimizations in each of __asan_loadX/__assn_storeX
 * depending on memory access size X.
 */

static __always_inline bool memory_is_poisoned_1(unsigned long addr)
{
	s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(shadow_value)) {
		s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
		return unlikely(last_accessible_byte >= shadow_value);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_2(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 1))
			return true;

		if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_4(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 3))
			return true;

		if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_8(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 7))
			return true;

		if (likely(((addr + 7) & KASAN_SHADOW_MASK) >= 7))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_16(unsigned long addr)
{
	u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		u16 shadow_first_bytes = *(u16 *)shadow_addr;
		s8 last_byte = (addr + 15) & KASAN_SHADOW_MASK;

		if (unlikely(shadow_first_bytes))
			return true;

		if (likely(!last_byte))
			return false;

		return memory_is_poisoned_1(addr + 15);
	}

	return false;
}

static __always_inline unsigned long bytes_is_zero(const u8 *start,
					size_t size)
{
	while (size) {
		if (unlikely(*start))
			return (unsigned long)start;
		start++;
		size--;
	}

	return 0;
}

static __always_inline unsigned long memory_is_zero(const void *start,
						const void *end)
{
	unsigned int words;
	unsigned long ret;
	unsigned int prefix = (unsigned long)start % 8;

	if (end - start <= 16)
		return bytes_is_zero(start, end - start);

	if (prefix) {
		prefix = 8 - prefix;
		ret = bytes_is_zero(start, prefix);
		if (unlikely(ret))
			return ret;
		start += prefix;
	}

	words = (end - start) / 8;
	while (words) {
		if (unlikely(*(u64 *)start))
			return bytes_is_zero(start, 8);
		start += 8;
		words--;
	}

	return bytes_is_zero(start, (end - start) % 8);
}

static __always_inline bool memory_is_poisoned_n(unsigned long addr,
						size_t size)
{
	unsigned long ret;

	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
			kasan_mem_to_shadow((void *)addr + size - 1) + 1);

	if (unlikely(ret)) {
		unsigned long last_byte = addr + size - 1;
		s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte);

		if (unlikely(ret != (unsigned long)last_shadow ||
			((last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
			return true;
	}
	return false;
}

static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
{
	if (__builtin_constant_p(size)) {
		switch (size) {
		case 1:
			return memory_is_poisoned_1(addr);
		case 2:
			return memory_is_poisoned_2(addr);
		case 4:
			return memory_is_poisoned_4(addr);
		case 8:
			return memory_is_poisoned_8(addr);
		case 16:
			return memory_is_poisoned_16(addr);
		default:
			BUILD_BUG();
		}
	}

	return memory_is_poisoned_n(addr, size);
}


static __always_inline void check_memory_region(unsigned long addr,
						size_t size, bool write)
{
	struct kasan_access_info info;

	if (unlikely(size == 0))
		return;

	if (unlikely((void *)addr <
		kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
		info.access_addr = (void *)addr;
		info.access_size = size;
		info.is_write = write;
		info.ip = _RET_IP_;
		kasan_report_user_access(&info);
		return;
	}

	if (likely(!memory_is_poisoned(addr, size)))
		return;

	kasan_report(addr, size, write, _RET_IP_);
}

#define DEFINE_ASAN_LOAD_STORE(size)				\
	void __asan_load##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, false);		\
	}							\
	EXPORT_SYMBOL(__asan_load##size);			\
	__alias(__asan_load##size)				\
	void __asan_load##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_load##size##_noabort);		\
	void __asan_store##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, true);		\
	}							\
	EXPORT_SYMBOL(__asan_store##size);			\
	__alias(__asan_store##size)				\
	void __asan_store##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_store##size##_noabort)

DEFINE_ASAN_LOAD_STORE(1);
DEFINE_ASAN_LOAD_STORE(2);
DEFINE_ASAN_LOAD_STORE(4);
DEFINE_ASAN_LOAD_STORE(8);
DEFINE_ASAN_LOAD_STORE(16);

void __asan_loadN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, false);
}
EXPORT_SYMBOL(__asan_loadN);

__alias(__asan_loadN)
void __asan_loadN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_loadN_noabort);

void __asan_storeN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, true);
}
EXPORT_SYMBOL(__asan_storeN);

__alias(__asan_storeN)
void __asan_storeN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_storeN_noabort);

/* to shut up compiler complaints */
void __asan_handle_no_return(void) {}
EXPORT_SYMBOL(__asan_handle_no_return);
Commit	Line	Data
0b24becc AR	1	/*
	2	* This file contains shadow memory manipulation code.
	3	*
	4	* Copyright (c) 2014 Samsung Electronics Co., Ltd.
	5	* Author: Andrey Ryabinin <a.ryabinin@samsung.com>
	6	*
	7	* Some of code borrowed from https://github.com/xairy/linux by
	8	* Andrey Konovalov <adech.fo@gmail.com>
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*
	14	*/
	15
	16	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	17	#define DISABLE_BRANCH_PROFILING
	18
	19	#include <linux/export.h>
	20	#include <linux/init.h>
	21	#include <linux/kernel.h>
	22	#include <linux/memblock.h>
	23	#include <linux/mm.h>
	24	#include <linux/printk.h>
	25	#include <linux/sched.h>
	26	#include <linux/slab.h>
	27	#include <linux/stacktrace.h>
	28	#include <linux/string.h>
	29	#include <linux/types.h>
	30	#include <linux/kasan.h>
	31
	32	#include "kasan.h"
	33
	34	/*
	35	* Poisons the shadow memory for 'size' bytes starting from 'addr'.
	36	* Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
	37	*/
	38	static void kasan_poison_shadow(const void *address, size_t size, u8 value)
	39	{
	40	void shadow_start, shadow_end;
	41
	42	shadow_start = kasan_mem_to_shadow(address);
	43	shadow_end = kasan_mem_to_shadow(address + size);
	44
	45	memset(shadow_start, value, shadow_end - shadow_start);
	46	}
	47
	48	void kasan_unpoison_shadow(const void *address, size_t size)
	49	{
	50	kasan_poison_shadow(address, size, 0);
	51
	52	if (size & KASAN_SHADOW_MASK) {
	53	u8 shadow = (u8 )kasan_mem_to_shadow(address + size);
	54	*shadow = size & KASAN_SHADOW_MASK;
	55	}
	56	}
	57
	58
	59	/*
	60	* All functions below always inlined so compiler could
	61	* perform better optimizations in each of __asan_loadX/__assn_storeX
	62	* depending on memory access size X.
	63	*/
	64
65	static __always_inline bool memory_is_poisoned_1(unsigned long addr)
66	{
67	s8 shadow_value = (s8 )kasan_mem_to_shadow((void *)addr);
68
69	if (unlikely(shadow_value)) {
70	s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
71	return unlikely(last_accessible_byte >= shadow_value);
72	}
73
74	return false;
75	}
76
77	static __always_inline bool memory_is_poisoned_2(unsigned long addr)
78	{
79	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
80
81	if (unlikely(*shadow_addr)) {
82	if (memory_is_poisoned_1(addr + 1))
83	return true;
84
85	if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
86	return false;
87
88	return unlikely((u8 )shadow_addr);
89	}
90
91	return false;
92	}
93
94	static __always_inline bool memory_is_poisoned_4(unsigned long addr)
95	{
96	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
97
98	if (unlikely(*shadow_addr)) {
99	if (memory_is_poisoned_1(addr + 3))
100	return true;
101
102	if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
103	return false;
104
105	return unlikely((u8 )shadow_addr);
106	}
107
108	return false;
109	}
110
111	static __always_inline bool memory_is_poisoned_8(unsigned long addr)
112	{
113	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
114
115	if (unlikely(*shadow_addr)) {
116	if (memory_is_poisoned_1(addr + 7))
117	return true;
118
119	if (likely(((addr + 7) & KASAN_SHADOW_MASK) >= 7))
120	return false;
121
122	return unlikely((u8 )shadow_addr);
123	}
124
125	return false;
126	}
127
128	static __always_inline bool memory_is_poisoned_16(unsigned long addr)
129	{
130	u32 shadow_addr = (u32 )kasan_mem_to_shadow((void *)addr);
131
132	if (unlikely(*shadow_addr)) {
133	u16 shadow_first_bytes = (u16 )shadow_addr;
134	s8 last_byte = (addr + 15) & KASAN_SHADOW_MASK;
135
136	if (unlikely(shadow_first_bytes))
137	return true;
138
139	if (likely(!last_byte))
140	return false;
141
142	return memory_is_poisoned_1(addr + 15);
143	}
144
145	return false;
146	}
147
148	static __always_inline unsigned long bytes_is_zero(const u8 *start,
149	size_t size)
150	{
151	while (size) {
152	if (unlikely(*start))
153	return (unsigned long)start;
154	start++;
155	size--;
156	}
157
158	return 0;
159	}
160
161	static __always_inline unsigned long memory_is_zero(const void *start,
162	const void *end)
163	{
164	unsigned int words;
165	unsigned long ret;
166	unsigned int prefix = (unsigned long)start % 8;
167
168	if (end - start <= 16)
169	return bytes_is_zero(start, end - start);
170
171	if (prefix) {
172	prefix = 8 - prefix;
173	ret = bytes_is_zero(start, prefix);
174	if (unlikely(ret))
175	return ret;
176	start += prefix;
177	}
178
179	words = (end - start) / 8;
180	while (words) {
181	if (unlikely((u64 )start))
182	return bytes_is_zero(start, 8);
183	start += 8;
184	words--;
185	}
186
187	return bytes_is_zero(start, (end - start) % 8);
188	}
189
190	static __always_inline bool memory_is_poisoned_n(unsigned long addr,
191	size_t size)
192	{
193	unsigned long ret;
194
195	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
196	kasan_mem_to_shadow((void *)addr + size - 1) + 1);
197
198	if (unlikely(ret)) {
199	unsigned long last_byte = addr + size - 1;
200	s8 last_shadow = (s8 )kasan_mem_to_shadow((void *)last_byte);
201
202	if (unlikely(ret != (unsigned long)last_shadow \|\|
203	((last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
204	return true;
205	}
206	return false;
207	}
208
209	static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
210	{
211	if (__builtin_constant_p(size)) {
212	switch (size) {
213	case 1:
214	return memory_is_poisoned_1(addr);
215	case 2:
216	return memory_is_poisoned_2(addr);
217	case 4:
218	return memory_is_poisoned_4(addr);
219	case 8:
220	return memory_is_poisoned_8(addr);
221	case 16:
222	return memory_is_poisoned_16(addr);
223	default:
224	BUILD_BUG();
225	}
226	}
227
228	return memory_is_poisoned_n(addr, size);
229	}
230
231
232	static __always_inline void check_memory_region(unsigned long addr,
233	size_t size, bool write)
234	{
235	struct kasan_access_info info;
236
237	if (unlikely(size == 0))
238	return;
239
240	if (unlikely((void *)addr <
241	kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
242	info.access_addr = (void *)addr;
243	info.access_size = size;
244	info.is_write = write;
245	info.ip = _RET_IP_;
246	kasan_report_user_access(&info);
247	return;
248	}
249
250	if (likely(!memory_is_poisoned(addr, size)))
251	return;
252
253	kasan_report(addr, size, write, _RET_IP_);
254	}
255
256	#define DEFINE_ASAN_LOAD_STORE(size) \
257	void __asan_load##size(unsigned long addr) \
258	{ \
259	check_memory_region(addr, size, false); \
260	} \
261	EXPORT_SYMBOL(__asan_load##size); \
262	__alias(__asan_load##size) \
263	void __asan_load##size##_noabort(unsigned long); \
264	EXPORT_SYMBOL(__asan_load##size##_noabort); \
265	void __asan_store##size(unsigned long addr) \
266	{ \
267	check_memory_region(addr, size, true); \
268	} \
269	EXPORT_SYMBOL(__asan_store##size); \
270	__alias(__asan_store##size) \
271	void __asan_store##size##_noabort(unsigned long); \
272	EXPORT_SYMBOL(__asan_store##size##_noabort)
273
274	DEFINE_ASAN_LOAD_STORE(1);
275	DEFINE_ASAN_LOAD_STORE(2);
276	DEFINE_ASAN_LOAD_STORE(4);
277	DEFINE_ASAN_LOAD_STORE(8);
278	DEFINE_ASAN_LOAD_STORE(16);
279
280	void __asan_loadN(unsigned long addr, size_t size)
281	{
282	check_memory_region(addr, size, false);
283	}
284	EXPORT_SYMBOL(__asan_loadN);
285
286	__alias(__asan_loadN)
287	void __asan_loadN_noabort(unsigned long, size_t);
288	EXPORT_SYMBOL(__asan_loadN_noabort);
289
290	void __asan_storeN(unsigned long addr, size_t size)
291	{
292	check_memory_region(addr, size, true);
293	}
294	EXPORT_SYMBOL(__asan_storeN);
295
296	__alias(__asan_storeN)
297	void __asan_storeN_noabort(unsigned long, size_t);
298	EXPORT_SYMBOL(__asan_storeN_noabort);
299
300	/* to shut up compiler complaints */
301	void __asan_handle_no_return(void) {}
302	EXPORT_SYMBOL(__asan_handle_no_return);