[deliverable/linux.git] / arch / s390 / mm / maccess.c

/*
 * Access kernel memory without faulting -- s390 specific implementation.
 *
 * Copyright IBM Corp. 2009
 *
 *   Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>,
 *
 */

#include <linux/uaccess.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/cpu.h>
#include <asm/ctl_reg.h>

/*
 * This function writes to kernel memory bypassing DAT and possible
 * write protection. It copies one to four bytes from src to dst
 * using the stura instruction.
 * Returns the number of bytes copied or -EFAULT.
 */
static long probe_kernel_write_odd(void *dst, const void *src, size_t size)
{
	unsigned long count, aligned;
	int offset, mask;
	int rc = -EFAULT;

	aligned = (unsigned long) dst & ~3UL;
	offset = (unsigned long) dst & 3;
	count = min_t(unsigned long, 4 - offset, size);
	mask = (0xf << (4 - count)) & 0xf;
	mask >>= offset;
	asm volatile(
		"	bras	1,0f\n"
		"	icm	0,0,0(%3)\n"
		"0:	l	0,0(%1)\n"
		"	lra	%1,0(%1)\n"
		"1:	ex	%2,0(1)\n"
		"2:	stura	0,%1\n"
		"	la	%0,0\n"
		"3:\n"
		EX_TABLE(0b,3b) EX_TABLE(1b,3b) EX_TABLE(2b,3b)
		: "+d" (rc), "+a" (aligned)
		: "a" (mask), "a" (src) : "cc", "memory", "0", "1");
	return rc ? rc : count;
}

long probe_kernel_write(void *dst, const void *src, size_t size)
{
	long copied = 0;

	while (size) {
		copied = probe_kernel_write_odd(dst, src, size);
		if (copied < 0)
			break;
		dst += copied;
		src += copied;
		size -= copied;
	}
	return copied < 0 ? -EFAULT : 0;
}

static int __memcpy_real(void *dest, void *src, size_t count)
{
	register unsigned long _dest asm("2") = (unsigned long) dest;
	register unsigned long _len1 asm("3") = (unsigned long) count;
	register unsigned long _src  asm("4") = (unsigned long) src;
	register unsigned long _len2 asm("5") = (unsigned long) count;
	int rc = -EFAULT;

	asm volatile (
		"0:	mvcle	%1,%2,0x0\n"
		"1:	jo	0b\n"
		"	lhi	%0,0x0\n"
		"2:\n"
		EX_TABLE(1b,2b)
		: "+d" (rc), "+d" (_dest), "+d" (_src), "+d" (_len1),
		  "+d" (_len2), "=m" (*((long *) dest))
		: "m" (*((long *) src))
		: "cc", "memory");
	return rc;
}

/*
 * Copy memory in real mode (kernel to kernel)
 */
int memcpy_real(void *dest, void *src, size_t count)
{
	unsigned long flags;
	int rc;

	if (!count)
		return 0;
	local_irq_save(flags);
	__arch_local_irq_stnsm(0xfbUL);
	rc = __memcpy_real(dest, src, count);
	local_irq_restore(flags);
	return rc;
}

/*
 * Copy memory to absolute zero
 */
void copy_to_absolute_zero(void *dest, void *src, size_t count)
{
	unsigned long cr0;

	BUG_ON((unsigned long) dest + count >= sizeof(struct _lowcore));
	preempt_disable();
	__ctl_store(cr0, 0, 0);
	__ctl_clear_bit(0, 28); /* disable lowcore protection */
	memcpy_real(dest + store_prefix(), src, count);
	__ctl_load(cr0, 0, 0);
	preempt_enable();
}

/*
 * Copy memory from kernel (real) to user (virtual)
 */
int copy_to_user_real(void __user *dest, void *src, size_t count)
{
	int offs = 0, size, rc;
	char *buf;

	buf = (char *) __get_free_page(GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
	rc = -EFAULT;
	while (offs < count) {
		size = min(PAGE_SIZE, count - offs);
		if (memcpy_real(buf, src + offs, size))
			goto out;
		if (copy_to_user(dest + offs, buf, size))
			goto out;
		offs += size;
	}
	rc = 0;
out:
	free_page((unsigned long) buf);
	return rc;
}

/*
 * Copy memory from user (virtual) to kernel (real)
 */
int copy_from_user_real(void *dest, void __user *src, size_t count)
{
	int offs = 0, size, rc;
	char *buf;

	buf = (char *) __get_free_page(GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
	rc = -EFAULT;
	while (offs < count) {
		size = min(PAGE_SIZE, count - offs);
		if (copy_from_user(buf, src + offs, size))
			goto out;
		if (memcpy_real(dest + offs, buf, size))
			goto out;
		offs += size;
	}
	rc = 0;
out:
	free_page((unsigned long) buf);
	return rc;
}

/*
 * Check if physical address is within prefix or zero page
 */
static int is_swapped(unsigned long addr)
{
	unsigned long lc;
	int cpu;

	if (addr < sizeof(struct _lowcore))
		return 1;
	for_each_online_cpu(cpu) {
		lc = (unsigned long) lowcore_ptr[cpu];
		if (addr > lc + sizeof(struct _lowcore) - 1 || addr < lc)
			continue;
		return 1;
	}
	return 0;
}

/*
 * Return swapped prefix or zero page address
 */
static unsigned long get_swapped(unsigned long addr)
{
	unsigned long prefix = store_prefix();

	if (addr < sizeof(struct _lowcore))
		return addr + prefix;
	if (addr >= prefix && addr < prefix + sizeof(struct _lowcore))
		return addr - prefix;
	return addr;
}

/*
 * Convert a physical pointer for /dev/mem access
 *
 * For swapped prefix pages a new buffer is returned that contains a copy of
 * the absolute memory. The buffer size is maximum one page large.
 */
void *xlate_dev_mem_ptr(unsigned long addr)
{
	void *bounce = (void *) addr;
	unsigned long size;

	get_online_cpus();
	preempt_disable();
	if (is_swapped(addr)) {
		size = PAGE_SIZE - (addr & ~PAGE_MASK);
		bounce = (void *) __get_free_page(GFP_ATOMIC);
		if (bounce)
			memcpy_real(bounce, (void *) get_swapped(addr), size);
	}
	preempt_enable();
	put_online_cpus();
	return bounce;
}

/*
 * Free converted buffer for /dev/mem access (if necessary)
 */
void unxlate_dev_mem_ptr(unsigned long addr, void *buf)
{
	if ((void *) addr != buf)
		free_page((unsigned long) buf);
}
Commit	Line	Data
88df125f HC	1	/*
	2	* Access kernel memory without faulting -- s390 specific implementation.
	3	*
	4	* Copyright IBM Corp. 2009
	5	*
	6	* Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>,
	7	*
	8	*/
	9
	10	#include <linux/uaccess.h>
	11	#include <linux/kernel.h>
	12	#include <linux/types.h>
	13	#include <linux/errno.h>
7f0bf656	14	#include <linux/gfp.h>
b2a68c23	15	#include <linux/cpu.h>
a0616cde	16	#include <asm/ctl_reg.h>
88df125f HC	17
	18	/*
	19	* This function writes to kernel memory bypassing DAT and possible
	20	* write protection. It copies one to four bytes from src to dst
	21	* using the stura instruction.
	22	* Returns the number of bytes copied or -EFAULT.
	23	*/
f29c5041	24	static long probe_kernel_write_odd(void dst, const void src, size_t size)
88df125f HC	25	{
	26	unsigned long count, aligned;
	27	int offset, mask;
	28	int rc = -EFAULT;
	29
	30	aligned = (unsigned long) dst & ~3UL;
	31	offset = (unsigned long) dst & 3;
	32	count = min_t(unsigned long, 4 - offset, size);
	33	mask = (0xf << (4 - count)) & 0xf;
	34	mask >>= offset;
	35	asm volatile(
	36	" bras 1,0f\n"
	37	" icm 0,0,0(%3)\n"
	38	"0: l 0,0(%1)\n"
	39	" lra %1,0(%1)\n"
	40	"1: ex %2,0(1)\n"
	41	"2: stura 0,%1\n"
	42	" la %0,0\n"
	43	"3:\n"
	44	EX_TABLE(0b,3b) EX_TABLE(1b,3b) EX_TABLE(2b,3b)
	45	: "+d" (rc), "+a" (aligned)
	46	: "a" (mask), "a" (src) : "cc", "memory", "0", "1");
	47	return rc ? rc : count;
	48	}
	49
f29c5041	50	long probe_kernel_write(void dst, const void src, size_t size)
88df125f HC	51	{
	52	long copied = 0;
	53
	54	while (size) {
	55	copied = probe_kernel_write_odd(dst, src, size);
	56	if (copied < 0)
	57	break;
	58	dst += copied;
	59	src += copied;
	60	size -= copied;
	61	}
	62	return copied < 0 ? -EFAULT : 0;
	63	}
92fe3132	64
b785e0d0	65	static int __memcpy_real(void dest, void src, size_t count)
92fe3132 MH	66	{
	67	register unsigned long _dest asm("2") = (unsigned long) dest;
	68	register unsigned long _len1 asm("3") = (unsigned long) count;
	69	register unsigned long _src asm("4") = (unsigned long) src;
	70	register unsigned long _len2 asm("5") = (unsigned long) count;
92fe3132 MH	71	int rc = -EFAULT;
92fe3132 MH	72
92fe3132 MH	73	asm volatile (
	74	"0: mvcle %1,%2,0x0\n"
	75	"1: jo 0b\n"
	76	" lhi %0,0x0\n"
	77	"2:\n"
	78	EX_TABLE(1b,2b)
	79	: "+d" (rc), "+d" (_dest), "+d" (_src), "+d" (_len1),
	80	"+d" (_len2), "=m" (((long ) dest))
	81	: "m" (((long ) src))
	82	: "cc", "memory");
b785e0d0 MH	83	return rc;
	84	}
	85
	86	/*
	87	* Copy memory in real mode (kernel to kernel)
	88	*/
	89	int memcpy_real(void dest, void src, size_t count)
	90	{
	91	unsigned long flags;
	92	int rc;
	93
	94	if (!count)
	95	return 0;
	96	local_irq_save(flags);
	97	__arch_local_irq_stnsm(0xfbUL);
	98	rc = __memcpy_real(dest, src, count);
	99	local_irq_restore(flags);
92fe3132 MH	100	return rc;
92fe3132 MH	101	}
7dd6b334 MH	102
	103	/*
	104	* Copy memory to absolute zero
	105	*/
	106	void copy_to_absolute_zero(void dest, void src, size_t count)
	107	{
	108	unsigned long cr0;
	109
	110	BUG_ON((unsigned long) dest + count >= sizeof(struct _lowcore));
	111	preempt_disable();
	112	__ctl_store(cr0, 0, 0);
	113	__ctl_clear_bit(0, 28); /* disable lowcore protection */
	114	memcpy_real(dest + store_prefix(), src, count);
	115	__ctl_load(cr0, 0, 0);
	116	preempt_enable();
	117	}
7f0bf656 MH	118
	119	/*
	120	* Copy memory from kernel (real) to user (virtual)
	121	*/
	122	int copy_to_user_real(void __user dest, void src, size_t count)
	123	{
	124	int offs = 0, size, rc;
	125	char *buf;
	126
	127	buf = (char *) __get_free_page(GFP_KERNEL);
	128	if (!buf)
	129	return -ENOMEM;
	130	rc = -EFAULT;
	131	while (offs < count) {
	132	size = min(PAGE_SIZE, count - offs);
	133	if (memcpy_real(buf, src + offs, size))
	134	goto out;
	135	if (copy_to_user(dest + offs, buf, size))
	136	goto out;
	137	offs += size;
	138	}
	139	rc = 0;
	140	out:
	141	free_page((unsigned long) buf);
	142	return rc;
	143	}
	144
	145	/*
	146	* Copy memory from user (virtual) to kernel (real)
	147	*/
	148	int copy_from_user_real(void dest, void __user src, size_t count)
	149	{
	150	int offs = 0, size, rc;
	151	char *buf;
	152
	153	buf = (char *) __get_free_page(GFP_KERNEL);
	154	if (!buf)
	155	return -ENOMEM;
	156	rc = -EFAULT;
	157	while (offs < count) {
	158	size = min(PAGE_SIZE, count - offs);
	159	if (copy_from_user(buf, src + offs, size))
	160	goto out;
	161	if (memcpy_real(dest + offs, buf, size))
	162	goto out;
	163	offs += size;
	164	}
	165	rc = 0;
	166	out:
	167	free_page((unsigned long) buf);
	168	return rc;
	169	}
b2a68c23 MH	170
	171	/*
	172	* Check if physical address is within prefix or zero page
	173	*/
	174	static int is_swapped(unsigned long addr)
	175	{
	176	unsigned long lc;
	177	int cpu;
	178
	179	if (addr < sizeof(struct _lowcore))
	180	return 1;
	181	for_each_online_cpu(cpu) {
	182	lc = (unsigned long) lowcore_ptr[cpu];
	183	if (addr > lc + sizeof(struct _lowcore) - 1 \|\| addr < lc)
	184	continue;
	185	return 1;
	186	}
	187	return 0;
	188	}
	189
	190	/*
	191	* Return swapped prefix or zero page address
	192	*/
	193	static unsigned long get_swapped(unsigned long addr)
	194	{
	195	unsigned long prefix = store_prefix();
	196
	197	if (addr < sizeof(struct _lowcore))
	198	return addr + prefix;
	199	if (addr >= prefix && addr < prefix + sizeof(struct _lowcore))
	200	return addr - prefix;
	201	return addr;
	202	}
	203
	204	/*
	205	* Convert a physical pointer for /dev/mem access
	206	*
	207	* For swapped prefix pages a new buffer is returned that contains a copy of
	208	* the absolute memory. The buffer size is maximum one page large.
	209	*/
	210	void *xlate_dev_mem_ptr(unsigned long addr)
	211	{
	212	void bounce = (void ) addr;
	213	unsigned long size;
	214
	215	get_online_cpus();
	216	preempt_disable();
	217	if (is_swapped(addr)) {
	218	size = PAGE_SIZE - (addr & ~PAGE_MASK);
	219	bounce = (void *) __get_free_page(GFP_ATOMIC);
	220	if (bounce)
	221	memcpy_real(bounce, (void *) get_swapped(addr), size);
	222	}
	223	preempt_enable();
	224	put_online_cpus();
	225	return bounce;
	226	}
	227
	228	/*
	229	* Free converted buffer for /dev/mem access (if necessary)
	230	*/
	231	void unxlate_dev_mem_ptr(unsigned long addr, void *buf)
	232	{
	233	if ((void *) addr != buf)
234	free_page((unsigned long) buf);
235	}