[deliverable/linux.git] / mm / kasan / kasan.c

/*
 * This file contains shadow memory manipulation code.
 *
 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
 *
 * Some code borrowed from https://github.com/xairy/kasan-prototype by
 *        Andrey Konovalov <adech.fo@gmail.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#define DISABLE_BRANCH_PROFILING

#include <linux/export.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/kmemleak.h>
#include <linux/linkage.h>
#include <linux/memblock.h>
#include <linux/memory.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/printk.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/stacktrace.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/kasan.h>

#include "kasan.h"
#include "../slab.h"

/*
 * Poisons the shadow memory for 'size' bytes starting from 'addr'.
 * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
 */
static void kasan_poison_shadow(const void *address, size_t size, u8 value)
{
	void *shadow_start, *shadow_end;

	shadow_start = kasan_mem_to_shadow(address);
	shadow_end = kasan_mem_to_shadow(address + size);

	memset(shadow_start, value, shadow_end - shadow_start);
}

void kasan_unpoison_shadow(const void *address, size_t size)
{
	kasan_poison_shadow(address, size, 0);

	if (size & KASAN_SHADOW_MASK) {
		u8 *shadow = (u8 *)kasan_mem_to_shadow(address + size);
		*shadow = size & KASAN_SHADOW_MASK;
	}
}

static void __kasan_unpoison_stack(struct task_struct *task, void *sp)
{
	void *base = task_stack_page(task);
	size_t size = sp - base;

	kasan_unpoison_shadow(base, size);
}

/* Unpoison the entire stack for a task. */
void kasan_unpoison_task_stack(struct task_struct *task)
{
	__kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE);
}

/* Unpoison the stack for the current task beyond a watermark sp value. */
asmlinkage void kasan_unpoison_remaining_stack(void *sp)
{
	__kasan_unpoison_stack(current, sp);
}

/*
 * All functions below always inlined so compiler could
 * perform better optimizations in each of __asan_loadX/__assn_storeX
 * depending on memory access size X.
 */

static __always_inline bool memory_is_poisoned_1(unsigned long addr)
{
	s8 shadow_value = *(s8 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(shadow_value)) {
		s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
		return unlikely(last_accessible_byte >= shadow_value);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_2(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 1))
			return true;

		/*
		 * If single shadow byte covers 2-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_4(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 3))
			return true;

		/*
		 * If single shadow byte covers 4-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_8(unsigned long addr)
{
	u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		if (memory_is_poisoned_1(addr + 7))
			return true;

		/*
		 * If single shadow byte covers 8-byte access, we don't
		 * need to do anything more. Otherwise, test the first
		 * shadow byte.
		 */
		if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
			return false;

		return unlikely(*(u8 *)shadow_addr);
	}

	return false;
}

static __always_inline bool memory_is_poisoned_16(unsigned long addr)
{
	u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr);

	if (unlikely(*shadow_addr)) {
		u16 shadow_first_bytes = *(u16 *)shadow_addr;

		if (unlikely(shadow_first_bytes))
			return true;

		/*
		 * If two shadow bytes covers 16-byte access, we don't
		 * need to do anything more. Otherwise, test the last
		 * shadow byte.
		 */
		if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
			return false;

		return memory_is_poisoned_1(addr + 15);
	}

	return false;
}

static __always_inline unsigned long bytes_is_zero(const u8 *start,
					size_t size)
{
	while (size) {
		if (unlikely(*start))
			return (unsigned long)start;
		start++;
		size--;
	}

	return 0;
}

static __always_inline unsigned long memory_is_zero(const void *start,
						const void *end)
{
	unsigned int words;
	unsigned long ret;
	unsigned int prefix = (unsigned long)start % 8;

	if (end - start <= 16)
		return bytes_is_zero(start, end - start);

	if (prefix) {
		prefix = 8 - prefix;
		ret = bytes_is_zero(start, prefix);
		if (unlikely(ret))
			return ret;
		start += prefix;
	}

	words = (end - start) / 8;
	while (words) {
		if (unlikely(*(u64 *)start))
			return bytes_is_zero(start, 8);
		start += 8;
		words--;
	}

	return bytes_is_zero(start, (end - start) % 8);
}

static __always_inline bool memory_is_poisoned_n(unsigned long addr,
						size_t size)
{
	unsigned long ret;

	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
			kasan_mem_to_shadow((void *)addr + size - 1) + 1);

	if (unlikely(ret)) {
		unsigned long last_byte = addr + size - 1;
		s8 *last_shadow = (s8 *)kasan_mem_to_shadow((void *)last_byte);

		if (unlikely(ret != (unsigned long)last_shadow ||
			((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
			return true;
	}
	return false;
}

static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
{
	if (__builtin_constant_p(size)) {
		switch (size) {
		case 1:
			return memory_is_poisoned_1(addr);
		case 2:
			return memory_is_poisoned_2(addr);
		case 4:
			return memory_is_poisoned_4(addr);
		case 8:
			return memory_is_poisoned_8(addr);
		case 16:
			return memory_is_poisoned_16(addr);
		default:
			BUILD_BUG();
		}
	}

	return memory_is_poisoned_n(addr, size);
}


static __always_inline void check_memory_region(unsigned long addr,
						size_t size, bool write)
{
	if (unlikely(size == 0))
		return;

	if (unlikely((void *)addr <
		kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
		kasan_report(addr, size, write, _RET_IP_);
		return;
	}

	if (likely(!memory_is_poisoned(addr, size)))
		return;

	kasan_report(addr, size, write, _RET_IP_);
}

void __asan_loadN(unsigned long addr, size_t size);
void __asan_storeN(unsigned long addr, size_t size);

#undef memset
void *memset(void *addr, int c, size_t len)
{
	__asan_storeN((unsigned long)addr, len);

	return __memset(addr, c, len);
}

#undef memmove
void *memmove(void *dest, const void *src, size_t len)
{
	__asan_loadN((unsigned long)src, len);
	__asan_storeN((unsigned long)dest, len);

	return __memmove(dest, src, len);
}

#undef memcpy
void *memcpy(void *dest, const void *src, size_t len)
{
	__asan_loadN((unsigned long)src, len);
	__asan_storeN((unsigned long)dest, len);

	return __memcpy(dest, src, len);
}

void kasan_alloc_pages(struct page *page, unsigned int order)
{
	if (likely(!PageHighMem(page)))
		kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order);
}

void kasan_free_pages(struct page *page, unsigned int order)
{
	if (likely(!PageHighMem(page)))
		kasan_poison_shadow(page_address(page),
				PAGE_SIZE << order,
				KASAN_FREE_PAGE);
}

#ifdef CONFIG_SLAB
/*
 * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
 * For larger allocations larger redzones are used.
 */
static size_t optimal_redzone(size_t object_size)
{
	int rz =
		object_size <= 64        - 16   ? 16 :
		object_size <= 128       - 32   ? 32 :
		object_size <= 512       - 64   ? 64 :
		object_size <= 4096      - 128  ? 128 :
		object_size <= (1 << 14) - 256  ? 256 :
		object_size <= (1 << 15) - 512  ? 512 :
		object_size <= (1 << 16) - 1024 ? 1024 : 2048;
	return rz;
}

void kasan_cache_create(struct kmem_cache *cache, size_t *size,
			unsigned long *flags)
{
	int redzone_adjust;
	/* Make sure the adjusted size is still less than
	 * KMALLOC_MAX_CACHE_SIZE.
	 * TODO: this check is only useful for SLAB, but not SLUB. We'll need
	 * to skip it for SLUB when it starts using kasan_cache_create().
	 */
	if (*size > KMALLOC_MAX_CACHE_SIZE -
	    sizeof(struct kasan_alloc_meta) -
	    sizeof(struct kasan_free_meta))
		return;
	*flags |= SLAB_KASAN;
	/* Add alloc meta. */
	cache->kasan_info.alloc_meta_offset = *size;
	*size += sizeof(struct kasan_alloc_meta);

	/* Add free meta. */
	if (cache->flags & SLAB_DESTROY_BY_RCU || cache->ctor ||
	    cache->object_size < sizeof(struct kasan_free_meta)) {
		cache->kasan_info.free_meta_offset = *size;
		*size += sizeof(struct kasan_free_meta);
	}
	redzone_adjust = optimal_redzone(cache->object_size) -
		(*size - cache->object_size);
	if (redzone_adjust > 0)
		*size += redzone_adjust;
	*size = min(KMALLOC_MAX_CACHE_SIZE,
		    max(*size,
			cache->object_size +
			optimal_redzone(cache->object_size)));
}
#endif

void kasan_poison_slab(struct page *page)
{
	kasan_poison_shadow(page_address(page),
			PAGE_SIZE << compound_order(page),
			KASAN_KMALLOC_REDZONE);
}

void kasan_unpoison_object_data(struct kmem_cache *cache, void *object)
{
	kasan_unpoison_shadow(object, cache->object_size);
}

void kasan_poison_object_data(struct kmem_cache *cache, void *object)
{
	kasan_poison_shadow(object,
			round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
			KASAN_KMALLOC_REDZONE);
#ifdef CONFIG_SLAB
	if (cache->flags & SLAB_KASAN) {
		struct kasan_alloc_meta *alloc_info =
			get_alloc_info(cache, object);
		alloc_info->state = KASAN_STATE_INIT;
	}
#endif
}

static inline void set_track(struct kasan_track *track)
{
	track->cpu = raw_smp_processor_id();
	track->pid = current->pid;
	track->when = jiffies;
}

#ifdef CONFIG_SLAB
struct kasan_alloc_meta *get_alloc_info(struct kmem_cache *cache,
					const void *object)
{
	return (void *)object + cache->kasan_info.alloc_meta_offset;
}

struct kasan_free_meta *get_free_info(struct kmem_cache *cache,
				      const void *object)
{
	return (void *)object + cache->kasan_info.free_meta_offset;
}
#endif

void kasan_slab_alloc(struct kmem_cache *cache, void *object, gfp_t flags)
{
	kasan_kmalloc(cache, object, cache->object_size, flags);
}

void kasan_slab_free(struct kmem_cache *cache, void *object)
{
	unsigned long size = cache->object_size;
	unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);

	/* RCU slabs could be legally used after free within the RCU period */
	if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
		return;

#ifdef CONFIG_SLAB
	if (cache->flags & SLAB_KASAN) {
		struct kasan_free_meta *free_info =
			get_free_info(cache, object);
		struct kasan_alloc_meta *alloc_info =
			get_alloc_info(cache, object);
		alloc_info->state = KASAN_STATE_FREE;
		set_track(&free_info->track);
	}
#endif

	kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
}

void kasan_kmalloc(struct kmem_cache *cache, const void *object, size_t size,
		   gfp_t flags)
{
	unsigned long redzone_start;
	unsigned long redzone_end;

	if (unlikely(object == NULL))
		return;

	redzone_start = round_up((unsigned long)(object + size),
				KASAN_SHADOW_SCALE_SIZE);
	redzone_end = round_up((unsigned long)object + cache->object_size,
				KASAN_SHADOW_SCALE_SIZE);

	kasan_unpoison_shadow(object, size);
	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
		KASAN_KMALLOC_REDZONE);
#ifdef CONFIG_SLAB
	if (cache->flags & SLAB_KASAN) {
		struct kasan_alloc_meta *alloc_info =
			get_alloc_info(cache, object);

		alloc_info->state = KASAN_STATE_ALLOC;
		alloc_info->alloc_size = size;
		set_track(&alloc_info->track);
	}
#endif
}
EXPORT_SYMBOL(kasan_kmalloc);

void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
	struct page *page;
	unsigned long redzone_start;
	unsigned long redzone_end;

	if (unlikely(ptr == NULL))
		return;

	page = virt_to_page(ptr);
	redzone_start = round_up((unsigned long)(ptr + size),
				KASAN_SHADOW_SCALE_SIZE);
	redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));

	kasan_unpoison_shadow(ptr, size);
	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
		KASAN_PAGE_REDZONE);
}

void kasan_krealloc(const void *object, size_t size, gfp_t flags)
{
	struct page *page;

	if (unlikely(object == ZERO_SIZE_PTR))
		return;

	page = virt_to_head_page(object);

	if (unlikely(!PageSlab(page)))
		kasan_kmalloc_large(object, size, flags);
	else
		kasan_kmalloc(page->slab_cache, object, size, flags);
}

void kasan_kfree(void *ptr)
{
	struct page *page;

	page = virt_to_head_page(ptr);

	if (unlikely(!PageSlab(page)))
		kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
				KASAN_FREE_PAGE);
	else
		kasan_slab_free(page->slab_cache, ptr);
}

void kasan_kfree_large(const void *ptr)
{
	struct page *page = virt_to_page(ptr);

	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
			KASAN_FREE_PAGE);
}

int kasan_module_alloc(void *addr, size_t size)
{
	void *ret;
	size_t shadow_size;
	unsigned long shadow_start;

	shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
	shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT,
			PAGE_SIZE);

	if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
		return -EINVAL;

	ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
			shadow_start + shadow_size,
			GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
			PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
			__builtin_return_address(0));

	if (ret) {
		find_vm_area(addr)->flags |= VM_KASAN;
		kmemleak_ignore(ret);
		return 0;
	}

	return -ENOMEM;
}

void kasan_free_shadow(const struct vm_struct *vm)
{
	if (vm->flags & VM_KASAN)
		vfree(kasan_mem_to_shadow(vm->addr));
}

static void register_global(struct kasan_global *global)
{
	size_t aligned_size = round_up(global->size, KASAN_SHADOW_SCALE_SIZE);

	kasan_unpoison_shadow(global->beg, global->size);

	kasan_poison_shadow(global->beg + aligned_size,
		global->size_with_redzone - aligned_size,
		KASAN_GLOBAL_REDZONE);
}

void __asan_register_globals(struct kasan_global *globals, size_t size)
{
	int i;

	for (i = 0; i < size; i++)
		register_global(&globals[i]);
}
EXPORT_SYMBOL(__asan_register_globals);

void __asan_unregister_globals(struct kasan_global *globals, size_t size)
{
}
EXPORT_SYMBOL(__asan_unregister_globals);

#define DEFINE_ASAN_LOAD_STORE(size)				\
	void __asan_load##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, false);		\
	}							\
	EXPORT_SYMBOL(__asan_load##size);			\
	__alias(__asan_load##size)				\
	void __asan_load##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_load##size##_noabort);		\
	void __asan_store##size(unsigned long addr)		\
	{							\
		check_memory_region(addr, size, true);		\
	}							\
	EXPORT_SYMBOL(__asan_store##size);			\
	__alias(__asan_store##size)				\
	void __asan_store##size##_noabort(unsigned long);	\
	EXPORT_SYMBOL(__asan_store##size##_noabort)

DEFINE_ASAN_LOAD_STORE(1);
DEFINE_ASAN_LOAD_STORE(2);
DEFINE_ASAN_LOAD_STORE(4);
DEFINE_ASAN_LOAD_STORE(8);
DEFINE_ASAN_LOAD_STORE(16);

void __asan_loadN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, false);
}
EXPORT_SYMBOL(__asan_loadN);

__alias(__asan_loadN)
void __asan_loadN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_loadN_noabort);

void __asan_storeN(unsigned long addr, size_t size)
{
	check_memory_region(addr, size, true);
}
EXPORT_SYMBOL(__asan_storeN);

__alias(__asan_storeN)
void __asan_storeN_noabort(unsigned long, size_t);
EXPORT_SYMBOL(__asan_storeN_noabort);

/* to shut up compiler complaints */
void __asan_handle_no_return(void) {}
EXPORT_SYMBOL(__asan_handle_no_return);

#ifdef CONFIG_MEMORY_HOTPLUG
static int kasan_mem_notifier(struct notifier_block *nb,
			unsigned long action, void *data)
{
	return (action == MEM_GOING_ONLINE) ? NOTIFY_BAD : NOTIFY_OK;
}

static int __init kasan_memhotplug_init(void)
{
	pr_err("WARNING: KASAN doesn't support memory hot-add\n");
	pr_err("Memory hot-add will be disabled\n");

	hotplug_memory_notifier(kasan_mem_notifier, 0);

	return 0;
}

module_init(kasan_memhotplug_init);
#endif
Commit	Line	Data
0b24becc AR	1	/*
	2	* This file contains shadow memory manipulation code.
	3	*
	4	* Copyright (c) 2014 Samsung Electronics Co., Ltd.
2baf9e89	5	* Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
0b24becc	6	*
5d0926ef	7	* Some code borrowed from https://github.com/xairy/kasan-prototype by
0b24becc AR	8	* Andrey Konovalov <adech.fo@gmail.com>
	9	*
	10	* This program is free software; you can redistribute it and/or modify
	11	* it under the terms of the GNU General Public License version 2 as
	12	* published by the Free Software Foundation.
	13	*
	14	*/
	15
	16	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	17	#define DISABLE_BRANCH_PROFILING
	18
	19	#include <linux/export.h>
	20	#include <linux/init.h>
	21	#include <linux/kernel.h>
45937254	22	#include <linux/kmemleak.h>
e3ae1163	23	#include <linux/linkage.h>
0b24becc	24	#include <linux/memblock.h>
786a8959	25	#include <linux/memory.h>
0b24becc	26	#include <linux/mm.h>
bebf56a1	27	#include <linux/module.h>
0b24becc AR	28	#include <linux/printk.h>
	29	#include <linux/sched.h>
	30	#include <linux/slab.h>
	31	#include <linux/stacktrace.h>
	32	#include <linux/string.h>
	33	#include <linux/types.h>
a5af5aa8	34	#include <linux/vmalloc.h>
0b24becc AR	35	#include <linux/kasan.h>
	36
	37	#include "kasan.h"
0316bec2	38	#include "../slab.h"
0b24becc AR	39
	40	/*
	41	* Poisons the shadow memory for 'size' bytes starting from 'addr'.
	42	* Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
	43	*/
	44	static void kasan_poison_shadow(const void *address, size_t size, u8 value)
	45	{
	46	void shadow_start, shadow_end;
	47
	48	shadow_start = kasan_mem_to_shadow(address);
	49	shadow_end = kasan_mem_to_shadow(address + size);
	50
	51	memset(shadow_start, value, shadow_end - shadow_start);
	52	}
	53
	54	void kasan_unpoison_shadow(const void *address, size_t size)
	55	{
	56	kasan_poison_shadow(address, size, 0);
	57
	58	if (size & KASAN_SHADOW_MASK) {
	59	u8 shadow = (u8 )kasan_mem_to_shadow(address + size);
	60	*shadow = size & KASAN_SHADOW_MASK;
	61	}
	62	}
	63
e3ae1163 MR	64	static void __kasan_unpoison_stack(struct task_struct task, void sp)
	65	{
	66	void *base = task_stack_page(task);
	67	size_t size = sp - base;
	68
	69	kasan_unpoison_shadow(base, size);
	70	}
	71
	72	/* Unpoison the entire stack for a task. */
	73	void kasan_unpoison_task_stack(struct task_struct *task)
	74	{
	75	__kasan_unpoison_stack(task, task_stack_page(task) + THREAD_SIZE);
	76	}
	77
	78	/* Unpoison the stack for the current task beyond a watermark sp value. */
	79	asmlinkage void kasan_unpoison_remaining_stack(void *sp)
	80	{
	81	__kasan_unpoison_stack(current, sp);
	82	}
0b24becc AR	83
	84	/*
	85	* All functions below always inlined so compiler could
	86	* perform better optimizations in each of __asan_loadX/__assn_storeX
	87	* depending on memory access size X.
	88	*/
	89
	90	static __always_inline bool memory_is_poisoned_1(unsigned long addr)
	91	{
	92	s8 shadow_value = (s8 )kasan_mem_to_shadow((void *)addr);
	93
	94	if (unlikely(shadow_value)) {
	95	s8 last_accessible_byte = addr & KASAN_SHADOW_MASK;
	96	return unlikely(last_accessible_byte >= shadow_value);
	97	}
	98
	99	return false;
	100	}
	101
	102	static __always_inline bool memory_is_poisoned_2(unsigned long addr)
	103	{
	104	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	105
	106	if (unlikely(*shadow_addr)) {
	107	if (memory_is_poisoned_1(addr + 1))
	108	return true;
	109
10f70262 XQ	110	/*
	111	* If single shadow byte covers 2-byte access, we don't
	112	* need to do anything more. Otherwise, test the first
	113	* shadow byte.
	114	*/
0b24becc AR	115	if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0))
	116	return false;
	117
	118	return unlikely((u8 )shadow_addr);
	119	}
	120
	121	return false;
	122	}
	123
	124	static __always_inline bool memory_is_poisoned_4(unsigned long addr)
	125	{
	126	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	127
	128	if (unlikely(*shadow_addr)) {
	129	if (memory_is_poisoned_1(addr + 3))
	130	return true;
	131
10f70262 XQ	132	/*
	133	* If single shadow byte covers 4-byte access, we don't
	134	* need to do anything more. Otherwise, test the first
	135	* shadow byte.
	136	*/
0b24becc AR	137	if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3))
	138	return false;
	139
	140	return unlikely((u8 )shadow_addr);
	141	}
	142
	143	return false;
	144	}
	145
	146	static __always_inline bool memory_is_poisoned_8(unsigned long addr)
	147	{
	148	u16 shadow_addr = (u16 )kasan_mem_to_shadow((void *)addr);
	149
	150	if (unlikely(*shadow_addr)) {
	151	if (memory_is_poisoned_1(addr + 7))
	152	return true;
	153
10f70262 XQ	154	/*
	155	* If single shadow byte covers 8-byte access, we don't
	156	* need to do anything more. Otherwise, test the first
	157	* shadow byte.
	158	*/
	159	if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
0b24becc AR	160	return false;
	161
	162	return unlikely((u8 )shadow_addr);
	163	}
	164
	165	return false;
	166	}
	167
	168	static __always_inline bool memory_is_poisoned_16(unsigned long addr)
	169	{
	170	u32 shadow_addr = (u32 )kasan_mem_to_shadow((void *)addr);
	171
	172	if (unlikely(*shadow_addr)) {
	173	u16 shadow_first_bytes = (u16 )shadow_addr;
0b24becc AR	174
	175	if (unlikely(shadow_first_bytes))
	176	return true;
	177
10f70262 XQ	178	/*
	179	* If two shadow bytes covers 16-byte access, we don't
	180	* need to do anything more. Otherwise, test the last
	181	* shadow byte.
	182	*/
	183	if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE)))
0b24becc AR	184	return false;
	185
	186	return memory_is_poisoned_1(addr + 15);
	187	}
	188
	189	return false;
	190	}
	191
	192	static __always_inline unsigned long bytes_is_zero(const u8 *start,
	193	size_t size)
	194	{
	195	while (size) {
	196	if (unlikely(*start))
	197	return (unsigned long)start;
	198	start++;
	199	size--;
	200	}
	201
	202	return 0;
	203	}
	204
	205	static __always_inline unsigned long memory_is_zero(const void *start,
	206	const void *end)
	207	{
	208	unsigned int words;
	209	unsigned long ret;
	210	unsigned int prefix = (unsigned long)start % 8;
	211
	212	if (end - start <= 16)
	213	return bytes_is_zero(start, end - start);
	214
	215	if (prefix) {
	216	prefix = 8 - prefix;
	217	ret = bytes_is_zero(start, prefix);
	218	if (unlikely(ret))
	219	return ret;
	220	start += prefix;
	221	}
	222
	223	words = (end - start) / 8;
	224	while (words) {
	225	if (unlikely((u64 )start))
	226	return bytes_is_zero(start, 8);
	227	start += 8;
	228	words--;
	229	}
	230
	231	return bytes_is_zero(start, (end - start) % 8);
	232	}
	233
	234	static __always_inline bool memory_is_poisoned_n(unsigned long addr,
	235	size_t size)
	236	{
	237	unsigned long ret;
	238
	239	ret = memory_is_zero(kasan_mem_to_shadow((void *)addr),
	240	kasan_mem_to_shadow((void *)addr + size - 1) + 1);
	241
	242	if (unlikely(ret)) {
	243	unsigned long last_byte = addr + size - 1;
	244	s8 last_shadow = (s8 )kasan_mem_to_shadow((void *)last_byte);
	245
	246	if (unlikely(ret != (unsigned long)last_shadow \|\|
e0d57714	247	((long)(last_byte & KASAN_SHADOW_MASK) >= *last_shadow)))
0b24becc AR	248	return true;
	249	}
	250	return false;
	251	}
	252
	253	static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size)
	254	{
	255	if (__builtin_constant_p(size)) {
	256	switch (size) {
	257	case 1:
	258	return memory_is_poisoned_1(addr);
	259	case 2:
	260	return memory_is_poisoned_2(addr);
	261	case 4:
	262	return memory_is_poisoned_4(addr);
	263	case 8:
	264	return memory_is_poisoned_8(addr);
	265	case 16:
	266	return memory_is_poisoned_16(addr);
	267	default:
	268	BUILD_BUG();
	269	}
	270	}
	271
	272	return memory_is_poisoned_n(addr, size);
	273	}
	274
	275
	276	static __always_inline void check_memory_region(unsigned long addr,
	277	size_t size, bool write)
	278	{
0b24becc AR	279	if (unlikely(size == 0))
	280	return;
	281
	282	if (unlikely((void *)addr <
	283	kasan_shadow_to_mem((void *)KASAN_SHADOW_START))) {
e9121076	284	kasan_report(addr, size, write, _RET_IP_);
0b24becc AR	285	return;
	286	}
	287
	288	if (likely(!memory_is_poisoned(addr, size)))
	289	return;
	290
	291	kasan_report(addr, size, write, _RET_IP_);
	292	}
	293
393f203f AR	294	void __asan_loadN(unsigned long addr, size_t size);
	295	void __asan_storeN(unsigned long addr, size_t size);
	296
	297	#undef memset
	298	void memset(void addr, int c, size_t len)
	299	{
	300	__asan_storeN((unsigned long)addr, len);
	301
	302	return __memset(addr, c, len);
	303	}
	304
	305	#undef memmove
	306	void memmove(void dest, const void *src, size_t len)
	307	{
	308	__asan_loadN((unsigned long)src, len);
	309	__asan_storeN((unsigned long)dest, len);
	310
	311	return __memmove(dest, src, len);
	312	}
	313
	314	#undef memcpy
	315	void memcpy(void dest, const void *src, size_t len)
	316	{
	317	__asan_loadN((unsigned long)src, len);
	318	__asan_storeN((unsigned long)dest, len);
	319
	320	return __memcpy(dest, src, len);
	321	}
	322
b8c73fc2 AR	323	void kasan_alloc_pages(struct page *page, unsigned int order)
	324	{
	325	if (likely(!PageHighMem(page)))
	326	kasan_unpoison_shadow(page_address(page), PAGE_SIZE << order);
	327	}
	328
	329	void kasan_free_pages(struct page *page, unsigned int order)
	330	{
	331	if (likely(!PageHighMem(page)))
	332	kasan_poison_shadow(page_address(page),
	333	PAGE_SIZE << order,
	334	KASAN_FREE_PAGE);
	335	}
	336
7ed2f9e6 AP	337	#ifdef CONFIG_SLAB
	338	/*
	339	* Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
	340	* For larger allocations larger redzones are used.
	341	*/
	342	static size_t optimal_redzone(size_t object_size)
	343	{
	344	int rz =
	345	object_size <= 64 - 16 ? 16 :
	346	object_size <= 128 - 32 ? 32 :
	347	object_size <= 512 - 64 ? 64 :
	348	object_size <= 4096 - 128 ? 128 :
	349	object_size <= (1 << 14) - 256 ? 256 :
	350	object_size <= (1 << 15) - 512 ? 512 :
	351	object_size <= (1 << 16) - 1024 ? 1024 : 2048;
	352	return rz;
	353	}
	354
	355	void kasan_cache_create(struct kmem_cache cache, size_t size,
	356	unsigned long *flags)
	357	{
	358	int redzone_adjust;
	359	/* Make sure the adjusted size is still less than
	360	* KMALLOC_MAX_CACHE_SIZE.
	361	* TODO: this check is only useful for SLAB, but not SLUB. We'll need
	362	* to skip it for SLUB when it starts using kasan_cache_create().
	363	*/
	364	if (*size > KMALLOC_MAX_CACHE_SIZE -
	365	sizeof(struct kasan_alloc_meta) -
	366	sizeof(struct kasan_free_meta))
	367	return;
	368	*flags \|= SLAB_KASAN;
	369	/* Add alloc meta. */
	370	cache->kasan_info.alloc_meta_offset = *size;
	371	*size += sizeof(struct kasan_alloc_meta);
	372
	373	/* Add free meta. */
	374	if (cache->flags & SLAB_DESTROY_BY_RCU \|\| cache->ctor \|\|
	375	cache->object_size < sizeof(struct kasan_free_meta)) {
	376	cache->kasan_info.free_meta_offset = *size;
	377	*size += sizeof(struct kasan_free_meta);
	378	}
	379	redzone_adjust = optimal_redzone(cache->object_size) -
	380	(*size - cache->object_size);
	381	if (redzone_adjust > 0)
	382	*size += redzone_adjust;
	383	*size = min(KMALLOC_MAX_CACHE_SIZE,
	384	max(*size,
	385	cache->object_size +
	386	optimal_redzone(cache->object_size)));
	387	}
	388	#endif
	389
0316bec2 AR	390	void kasan_poison_slab(struct page *page)
	391	{
	392	kasan_poison_shadow(page_address(page),
	393	PAGE_SIZE << compound_order(page),
	394	KASAN_KMALLOC_REDZONE);
	395	}
	396
	397	void kasan_unpoison_object_data(struct kmem_cache cache, void object)
	398	{
	399	kasan_unpoison_shadow(object, cache->object_size);
	400	}
	401
	402	void kasan_poison_object_data(struct kmem_cache cache, void object)
	403	{
	404	kasan_poison_shadow(object,
	405	round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE),
	406	KASAN_KMALLOC_REDZONE);
7ed2f9e6 AP	407	#ifdef CONFIG_SLAB
	408	if (cache->flags & SLAB_KASAN) {
	409	struct kasan_alloc_meta *alloc_info =
	410	get_alloc_info(cache, object);
	411	alloc_info->state = KASAN_STATE_INIT;
	412	}
	413	#endif
0316bec2 AR	414	}
0316bec2 AR	415
7ed2f9e6 AP	416	static inline void set_track(struct kasan_track *track)
	417	{
	418	track->cpu = raw_smp_processor_id();
	419	track->pid = current->pid;
	420	track->when = jiffies;
	421	}
	422
	423	#ifdef CONFIG_SLAB
	424	struct kasan_alloc_meta get_alloc_info(struct kmem_cache cache,
	425	const void *object)
	426	{
	427	return (void *)object + cache->kasan_info.alloc_meta_offset;
	428	}
	429
	430	struct kasan_free_meta get_free_info(struct kmem_cache cache,
	431	const void *object)
	432	{
	433	return (void *)object + cache->kasan_info.free_meta_offset;
	434	}
	435	#endif
	436
505f5dcb	437	void kasan_slab_alloc(struct kmem_cache cache, void object, gfp_t flags)
0316bec2	438	{
505f5dcb	439	kasan_kmalloc(cache, object, cache->object_size, flags);
0316bec2 AR	440	}
	441
	442	void kasan_slab_free(struct kmem_cache cache, void object)
	443	{
	444	unsigned long size = cache->object_size;
	445	unsigned long rounded_up_size = round_up(size, KASAN_SHADOW_SCALE_SIZE);
	446
	447	/* RCU slabs could be legally used after free within the RCU period */
	448	if (unlikely(cache->flags & SLAB_DESTROY_BY_RCU))
	449	return;
	450
7ed2f9e6 AP	451	#ifdef CONFIG_SLAB
	452	if (cache->flags & SLAB_KASAN) {
	453	struct kasan_free_meta *free_info =
	454	get_free_info(cache, object);
	455	struct kasan_alloc_meta *alloc_info =
	456	get_alloc_info(cache, object);
	457	alloc_info->state = KASAN_STATE_FREE;
	458	set_track(&free_info->track);
	459	}
	460	#endif
	461
0316bec2 AR	462	kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
	463	}
	464
505f5dcb AP	465	void kasan_kmalloc(struct kmem_cache cache, const void object, size_t size,
505f5dcb AP	466	gfp_t flags)
0316bec2 AR	467	{
	468	unsigned long redzone_start;
	469	unsigned long redzone_end;
	470
	471	if (unlikely(object == NULL))
	472	return;
	473
	474	redzone_start = round_up((unsigned long)(object + size),
	475	KASAN_SHADOW_SCALE_SIZE);
	476	redzone_end = round_up((unsigned long)object + cache->object_size,
	477	KASAN_SHADOW_SCALE_SIZE);
	478
	479	kasan_unpoison_shadow(object, size);
	480	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
	481	KASAN_KMALLOC_REDZONE);
7ed2f9e6 AP	482	#ifdef CONFIG_SLAB
	483	if (cache->flags & SLAB_KASAN) {
	484	struct kasan_alloc_meta *alloc_info =
	485	get_alloc_info(cache, object);
	486
	487	alloc_info->state = KASAN_STATE_ALLOC;
	488	alloc_info->alloc_size = size;
	489	set_track(&alloc_info->track);
	490	}
	491	#endif
0316bec2 AR	492	}
	493	EXPORT_SYMBOL(kasan_kmalloc);
	494
505f5dcb	495	void kasan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
0316bec2 AR	496	{
	497	struct page *page;
	498	unsigned long redzone_start;
	499	unsigned long redzone_end;
	500
	501	if (unlikely(ptr == NULL))
	502	return;
	503
	504	page = virt_to_page(ptr);
	505	redzone_start = round_up((unsigned long)(ptr + size),
	506	KASAN_SHADOW_SCALE_SIZE);
	507	redzone_end = (unsigned long)ptr + (PAGE_SIZE << compound_order(page));
	508
	509	kasan_unpoison_shadow(ptr, size);
	510	kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start,
	511	KASAN_PAGE_REDZONE);
	512	}
	513
505f5dcb	514	void kasan_krealloc(const void *object, size_t size, gfp_t flags)
0316bec2 AR	515	{
	516	struct page *page;
	517
	518	if (unlikely(object == ZERO_SIZE_PTR))
	519	return;
	520
	521	page = virt_to_head_page(object);
	522
	523	if (unlikely(!PageSlab(page)))
505f5dcb	524	kasan_kmalloc_large(object, size, flags);
0316bec2	525	else
505f5dcb	526	kasan_kmalloc(page->slab_cache, object, size, flags);
0316bec2 AR	527	}
0316bec2 AR	528
92393615 AR	529	void kasan_kfree(void *ptr)
	530	{
	531	struct page *page;
	532
	533	page = virt_to_head_page(ptr);
	534
	535	if (unlikely(!PageSlab(page)))
	536	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
	537	KASAN_FREE_PAGE);
	538	else
	539	kasan_slab_free(page->slab_cache, ptr);
	540	}
	541
0316bec2 AR	542	void kasan_kfree_large(const void *ptr)
	543	{
	544	struct page *page = virt_to_page(ptr);
	545
	546	kasan_poison_shadow(ptr, PAGE_SIZE << compound_order(page),
	547	KASAN_FREE_PAGE);
	548	}
	549
bebf56a1 AR	550	int kasan_module_alloc(void *addr, size_t size)
	551	{
	552	void *ret;
	553	size_t shadow_size;
	554	unsigned long shadow_start;
	555
	556	shadow_start = (unsigned long)kasan_mem_to_shadow(addr);
	557	shadow_size = round_up(size >> KASAN_SHADOW_SCALE_SHIFT,
	558	PAGE_SIZE);
	559
	560	if (WARN_ON(!PAGE_ALIGNED(shadow_start)))
	561	return -EINVAL;
	562
	563	ret = __vmalloc_node_range(shadow_size, 1, shadow_start,
	564	shadow_start + shadow_size,
	565	GFP_KERNEL \| __GFP_HIGHMEM \| __GFP_ZERO,
	566	PAGE_KERNEL, VM_NO_GUARD, NUMA_NO_NODE,
	567	__builtin_return_address(0));
a5af5aa8 AR	568
	569	if (ret) {
	570	find_vm_area(addr)->flags \|= VM_KASAN;
45937254	571	kmemleak_ignore(ret);
a5af5aa8 AR	572	return 0;
	573	}
	574
	575	return -ENOMEM;
bebf56a1 AR	576	}
bebf56a1 AR	577
a5af5aa8	578	void kasan_free_shadow(const struct vm_struct *vm)
bebf56a1	579	{
a5af5aa8 AR	580	if (vm->flags & VM_KASAN)
a5af5aa8 AR	581	vfree(kasan_mem_to_shadow(vm->addr));
bebf56a1 AR	582	}
	583
	584	static void register_global(struct kasan_global *global)
	585	{
	586	size_t aligned_size = round_up(global->size, KASAN_SHADOW_SCALE_SIZE);
	587
	588	kasan_unpoison_shadow(global->beg, global->size);
	589
	590	kasan_poison_shadow(global->beg + aligned_size,
	591	global->size_with_redzone - aligned_size,
	592	KASAN_GLOBAL_REDZONE);
	593	}
	594
	595	void __asan_register_globals(struct kasan_global *globals, size_t size)
	596	{
	597	int i;
	598
	599	for (i = 0; i < size; i++)
	600	register_global(&globals[i]);
	601	}
	602	EXPORT_SYMBOL(__asan_register_globals);
	603
	604	void __asan_unregister_globals(struct kasan_global *globals, size_t size)
	605	{
	606	}
	607	EXPORT_SYMBOL(__asan_unregister_globals);
	608
0b24becc AR	609	#define DEFINE_ASAN_LOAD_STORE(size) \
	610	void __asan_load##size(unsigned long addr) \
	611	{ \
	612	check_memory_region(addr, size, false); \
	613	} \
	614	EXPORT_SYMBOL(__asan_load##size); \
	615	__alias(__asan_load##size) \
	616	void __asan_load##size##_noabort(unsigned long); \
	617	EXPORT_SYMBOL(__asan_load##size##_noabort); \
	618	void __asan_store##size(unsigned long addr) \
	619	{ \
	620	check_memory_region(addr, size, true); \
	621	} \
	622	EXPORT_SYMBOL(__asan_store##size); \
	623	__alias(__asan_store##size) \
	624	void __asan_store##size##_noabort(unsigned long); \
	625	EXPORT_SYMBOL(__asan_store##size##_noabort)
	626
	627	DEFINE_ASAN_LOAD_STORE(1);
	628	DEFINE_ASAN_LOAD_STORE(2);
	629	DEFINE_ASAN_LOAD_STORE(4);
	630	DEFINE_ASAN_LOAD_STORE(8);
	631	DEFINE_ASAN_LOAD_STORE(16);
	632
	633	void __asan_loadN(unsigned long addr, size_t size)
	634	{
	635	check_memory_region(addr, size, false);
	636	}
	637	EXPORT_SYMBOL(__asan_loadN);
	638
	639	__alias(__asan_loadN)
	640	void __asan_loadN_noabort(unsigned long, size_t);
	641	EXPORT_SYMBOL(__asan_loadN_noabort);
	642
	643	void __asan_storeN(unsigned long addr, size_t size)
	644	{
	645	check_memory_region(addr, size, true);
	646	}
	647	EXPORT_SYMBOL(__asan_storeN);
	648
	649	__alias(__asan_storeN)
	650	void __asan_storeN_noabort(unsigned long, size_t);
	651	EXPORT_SYMBOL(__asan_storeN_noabort);
	652
	653	/* to shut up compiler complaints */
	654	void __asan_handle_no_return(void) {}
	655	EXPORT_SYMBOL(__asan_handle_no_return);
786a8959 AR	656
	657	#ifdef CONFIG_MEMORY_HOTPLUG
	658	static int kasan_mem_notifier(struct notifier_block *nb,
	659	unsigned long action, void *data)
	660	{
	661	return (action == MEM_GOING_ONLINE) ? NOTIFY_BAD : NOTIFY_OK;
	662	}
	663
	664	static int __init kasan_memhotplug_init(void)
	665	{
25add7ec	666	pr_err("WARNING: KASAN doesn't support memory hot-add\n");
786a8959 AR	667	pr_err("Memory hot-add will be disabled\n");
	668
	669	hotplug_memory_notifier(kasan_mem_notifier, 0);
	670
	671	return 0;
	672	}
	673
	674	module_init(kasan_memhotplug_init);
	675	#endif