[deliverable/linux.git] / arch / xtensa / kernel / align.S

/*
 * arch/xtensa/kernel/align.S
 *
 * Handle unalignment exceptions in kernel space.
 *
 * This file is subject to the terms and conditions of the GNU General
 * Public License.  See the file "COPYING" in the main directory of
 * this archive for more details.
 *
 * Copyright (C) 2001 - 2005 Tensilica, Inc.
 *
 * Rewritten by Chris Zankel <chris@zankel.net>
 *
 * Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
 * and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
 */

#include <linux/linkage.h>
#include <asm/current.h>
#include <asm/asm-offsets.h>
#include <asm/processor.h>

#if XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION

/*  First-level exception handler for unaligned exceptions.
 *
 *  Note: This handler works only for kernel exceptions.  Unaligned user
 *        access should get a seg fault.
 */

/* Big and little endian 16-bit values are located in
 * different halves of a register.  HWORD_START helps to
 * abstract the notion of extracting a 16-bit value from a
 * register.
 * We also have to define new shifting instructions because
 * lsb and msb are on 'opposite' ends in a register for
 * different endian machines.
 *
 * Assume a memory region in ascending address:
 *   	0 1 2 3|4 5 6 7
 *
 * When loading one word into a register, the content of that register is:
 *  LE	3 2 1 0, 7 6 5 4
 *  BE  0 1 2 3, 4 5 6 7
 *
 * Masking the bits of the higher/lower address means:
 *  LE  X X 0 0, 0 0 X X
 *  BE	0 0 X X, X X 0 0
 *
 * Shifting to higher/lower addresses, means:
 *  LE  shift left / shift right
 *  BE  shift right / shift left
 *
 * Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
 *  LE  mask 0 0 X X / shift left
 *  BE  shift left / mask 0 0 X X
 */

#define UNALIGNED_USER_EXCEPTION

#if XCHAL_HAVE_BE

#define HWORD_START	16
#define	INSN_OP0	28
#define	INSN_T		24
#define	INSN_OP1	16

.macro __src_b	r, w0, w1;	src	\r, \w0, \w1;	.endm
.macro __ssa8	r;		ssa8b	\r;		.endm
.macro __ssa8r	r;		ssa8l	\r;		.endm
.macro __sh	r, s;		srl	\r, \s;		.endm
.macro __sl	r, s;		sll	\r, \s;		.endm
.macro __exth	r, s;		extui	\r, \s, 0, 16;	.endm
.macro __extl	r, s;		slli	\r, \s, 16;	.endm

#else

#define HWORD_START	0
#define	INSN_OP0	0
#define	INSN_T		4
#define	INSN_OP1	12

.macro __src_b	r, w0, w1;	src	\r, \w1, \w0;	.endm
.macro __ssa8	r;		ssa8l	\r;		.endm
.macro __ssa8r	r;		ssa8b	\r;		.endm
.macro __sh	r, s;		sll	\r, \s;		.endm
.macro __sl	r, s;		srl	\r, \s;		.endm
.macro __exth	r, s;		slli	\r, \s, 16;	.endm
.macro __extl	r, s;		extui	\r, \s, 0, 16;	.endm

#endif

/*
 *	xxxx xxxx = imm8 field
 *	     yyyy = imm4 field
 *	     ssss = s field
 *	     tttt = t field
 *
 *	       		 16		    0
 *		          -------------------
 *	L32I.N		  yyyy ssss tttt 1000
 *	S32I.N	          yyyy ssss tttt 1001
 *
 *	       23			    0
 *		-----------------------------
 *	res	          0000           0010
 *	L16UI	xxxx xxxx 0001 ssss tttt 0010
 *	L32I	xxxx xxxx 0010 ssss tttt 0010
 *	XXX	          0011 ssss tttt 0010
 *	XXX	          0100 ssss tttt 0010
 *	S16I	xxxx xxxx 0101 ssss tttt 0010
 *	S32I	xxxx xxxx 0110 ssss tttt 0010
 *	XXX	          0111 ssss tttt 0010
 *	XXX	          1000 ssss tttt 0010
 *	L16SI	xxxx xxxx 1001 ssss tttt 0010
 *	XXX	          1010           0010
 *      **L32AI	xxxx xxxx 1011 ssss tttt 0010 unsupported
 *	XXX	          1100           0010
 *	XXX	          1101           0010
 *	XXX	          1110           0010
 *	**S32RI	xxxx xxxx 1111 ssss tttt 0010 unsupported
 *		-----------------------------
 *                           ^         ^    ^
 *    sub-opcode (NIBBLE_R) -+         |    |
 *       t field (NIBBLE_T) -----------+    |
 *  major opcode (NIBBLE_OP0) --------------+
 */

#define OP0_L32I_N	0x8		/* load immediate narrow */
#define OP0_S32I_N	0x9		/* store immediate narrow */
#define OP1_SI_MASK	0x4		/* OP1 bit set for stores */
#define OP1_SI_BIT	2		/* OP1 bit number for stores */

#define OP1_L32I	0x2
#define OP1_L16UI	0x1
#define OP1_L16SI	0x9
#define OP1_L32AI	0xb

#define OP1_S32I	0x6
#define OP1_S16I	0x5
#define OP1_S32RI	0xf

/*
 * Entry condition:
 *
 *   a0:	trashed, original value saved on stack (PT_AREG0)
 *   a1:	a1
 *   a2:	new stack pointer, original in DEPC
 *   a3:	a3
 *   depc:	a2, original value saved on stack (PT_DEPC)
 *   excsave_1:	dispatch table
 *
 *   PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
 *	     <  VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
 */


ENTRY(fast_unaligned)

	/* Note: We don't expect the address to be aligned on a word
	 *       boundary. After all, the processor generated that exception
	 *       and it would be a hardware fault.
	 */

	/* Save some working register */

	s32i	a4, a2, PT_AREG4
	s32i	a5, a2, PT_AREG5
	s32i	a6, a2, PT_AREG6
	s32i	a7, a2, PT_AREG7
	s32i	a8, a2, PT_AREG8

	rsr	a0, depc
	s32i	a0, a2, PT_AREG2
	s32i	a3, a2, PT_AREG3

	/* Keep value of SAR in a0 */

	rsr	a0, sar
	rsr	a8, excvaddr		# load unaligned memory address

	/* Now, identify one of the following load/store instructions.
	 *
	 * The only possible danger of a double exception on the
	 * following l32i instructions is kernel code in vmalloc
	 * memory. The processor was just executing at the EPC_1
	 * address, and indeed, already fetched the instruction.  That
	 * guarantees a TLB mapping, which hasn't been replaced by
	 * this unaligned exception handler that uses only static TLB
	 * mappings. However, high-level interrupt handlers might
	 * modify TLB entries, so for the generic case, we register a
	 * TABLE_FIXUP handler here, too.
	 */

	/* a3...a6 saved on stack, a2 = SP */

	/* Extract the instruction that caused the unaligned access. */

	rsr	a7, epc1	# load exception address
	movi	a3, ~3
	and	a3, a3, a7	# mask lower bits

	l32i	a4, a3, 0	# load 2 words
	l32i	a5, a3, 4

	__ssa8	a7
	__src_b	a4, a4, a5	# a4 has the instruction

	/* Analyze the instruction (load or store?). */

	extui	a5, a4, INSN_OP0, 4	# get insn.op0 nibble

#if XCHAL_HAVE_DENSITY
	_beqi	a5, OP0_L32I_N, .Lload	# L32I.N, jump
	addi	a6, a5, -OP0_S32I_N
	_beqz	a6, .Lstore		# S32I.N, do a store
#endif
	/* 'store indicator bit' not set, jump */
	_bbci.l	a4, OP1_SI_BIT + INSN_OP1, .Lload

	/* Store: Jump to table entry to get the value in the source register.*/

.Lstore:movi	a5, .Lstore_table	# table
	extui	a6, a4, INSN_T, 4	# get source register
	addx8	a5, a6, a5
	jx	a5			# jump into table

	/* Invalid instruction, CRITICAL! */
.Linvalid_instruction_load:
	j	.Linvalid_instruction

	/* Load: Load memory address. */

.Lload: movi	a3, ~3
	and	a3, a3, a8		# align memory address

	__ssa8	a8
#ifdef UNALIGNED_USER_EXCEPTION
	addi	a3, a3, 8
	l32e	a5, a3, -8
	l32e	a6, a3, -4
#else
	l32i	a5, a3, 0
	l32i	a6, a3, 4
#endif
	__src_b	a3, a5, a6		# a3 has the data word

#if XCHAL_HAVE_DENSITY
	addi	a7, a7, 2		# increment PC (assume 16-bit insn)

	extui	a5, a4, INSN_OP0, 4
	_beqi	a5, OP0_L32I_N, 1f	# l32i.n: jump

	addi	a7, a7, 1
#else
	addi	a7, a7, 3
#endif

	extui	a5, a4, INSN_OP1, 4
	_beqi	a5, OP1_L32I, 1f	# l32i: jump

	extui	a3, a3, 0, 16		# extract lower 16 bits
	_beqi	a5, OP1_L16UI, 1f
	addi	a5, a5, -OP1_L16SI
	_bnez	a5, .Linvalid_instruction_load

	/* sign extend value */

	slli	a3, a3, 16
	srai	a3, a3, 16

	/* Set target register. */

1:

#if XCHAL_HAVE_LOOPS
	rsr	a5, lend		# check if we reached LEND
	bne	a7, a5, 1f
	rsr	a5, lcount		# and LCOUNT != 0
	beqz	a5, 1f
	addi	a5, a5, -1		# decrement LCOUNT and set
	rsr	a7, lbeg		# set PC to LBEGIN
	wsr	a5, lcount
#endif

1:	wsr	a7, epc1		# skip load instruction
	extui	a4, a4, INSN_T, 4	# extract target register
	movi	a5, .Lload_table
	addx8	a4, a4, a5
	jx	a4			# jump to entry for target register

	.align	8
.Lload_table:
	s32i	a3, a2, PT_AREG0;	_j .Lexit;	.align 8
	mov	a1, a3;			_j .Lexit;	.align 8 # fishy??
	s32i	a3, a2, PT_AREG2;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG3;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG4;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG5;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG6;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG7;	_j .Lexit;	.align 8
	s32i	a3, a2, PT_AREG8;	_j .Lexit;	.align 8
	mov	a9, a3		;	_j .Lexit;	.align 8
	mov	a10, a3		;	_j .Lexit;	.align 8
	mov	a11, a3		;	_j .Lexit;	.align 8
	mov	a12, a3		;	_j .Lexit;	.align 8
	mov	a13, a3		;	_j .Lexit;	.align 8
	mov	a14, a3		;	_j .Lexit;	.align 8
	mov	a15, a3		;	_j .Lexit;	.align 8

.Lstore_table:
	l32i	a3, a2, PT_AREG0;	_j 1f;	.align 8
	mov	a3, a1;			_j 1f;	.align 8	# fishy??
	l32i	a3, a2, PT_AREG2;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG3;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG4;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG5;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG6;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG7;	_j 1f;	.align 8
	l32i	a3, a2, PT_AREG8;	_j 1f;	.align 8
	mov	a3, a9		;	_j 1f;	.align 8
	mov	a3, a10		;	_j 1f;	.align 8
	mov	a3, a11		;	_j 1f;	.align 8
	mov	a3, a12		;	_j 1f;	.align 8
	mov	a3, a13		;	_j 1f;	.align 8
	mov	a3, a14		;	_j 1f;	.align 8
	mov	a3, a15		;	_j 1f;	.align 8

1: 	# a7: instruction pointer, a4: instruction, a3: value

	movi	a6, 0			# mask: ffffffff:00000000

#if XCHAL_HAVE_DENSITY
	addi	a7, a7, 2		# incr. PC,assume 16-bit instruction

	extui	a5, a4, INSN_OP0, 4	# extract OP0
	addi	a5, a5, -OP0_S32I_N
	_beqz	a5, 1f			# s32i.n: jump

	addi	a7, a7, 1		# increment PC, 32-bit instruction
#else
	addi	a7, a7, 3		# increment PC, 32-bit instruction
#endif

	extui	a5, a4, INSN_OP1, 4	# extract OP1
	_beqi	a5, OP1_S32I, 1f	# jump if 32 bit store
	_bnei	a5, OP1_S16I, .Linvalid_instruction_store

	movi	a5, -1
	__extl	a3, a3			# get 16-bit value
	__exth	a6, a5			# get 16-bit mask ffffffff:ffff0000

	/* Get memory address */

1:
#if XCHAL_HAVE_LOOPS
	rsr	a4, lend		# check if we reached LEND
	bne	a7, a4, 1f
	rsr	a4, lcount		# and LCOUNT != 0
	beqz	a4, 1f
	addi	a4, a4, -1		# decrement LCOUNT and set
	rsr	a7, lbeg		# set PC to LBEGIN
	wsr	a4, lcount
#endif

1:	wsr	a7, epc1		# skip store instruction
	movi	a4, ~3
	and	a4, a4, a8		# align memory address

	/* Insert value into memory */

	movi	a5, -1			# mask: ffffffff:XXXX0000
#ifdef UNALIGNED_USER_EXCEPTION
	addi	a4, a4, 8
#endif

	__ssa8r a8
	__src_b	a7, a5, a6		# lo-mask  F..F0..0 (BE) 0..0F..F (LE)
	__src_b	a6, a6, a5		# hi-mask  0..0F..F (BE) F..F0..0 (LE)
#ifdef UNALIGNED_USER_EXCEPTION
	l32e	a5, a4, -8
#else
	l32i	a5, a4, 0		# load lower address word
#endif
	and	a5, a5, a7		# mask
	__sh	a7, a3 			# shift value
	or	a5, a5, a7		# or with original value
#ifdef UNALIGNED_USER_EXCEPTION
	s32e	a5, a4, -8
	l32e	a7, a4, -4
#else
	s32i	a5, a4, 0		# store
	l32i	a7, a4, 4		# same for upper address word
#endif
	__sl	a5, a3
	and	a6, a7, a6
	or	a6, a6, a5
#ifdef UNALIGNED_USER_EXCEPTION
	s32e	a6, a4, -4
#else
	s32i	a6, a4, 4
#endif

	/* Done. restore stack and return */

.Lexit:
	movi	a4, 0
	rsr	a3, excsave1
	s32i	a4, a3, EXC_TABLE_FIXUP

	/* Restore working register */

	l32i	a8, a2, PT_AREG8
	l32i	a7, a2, PT_AREG7
	l32i	a6, a2, PT_AREG6
	l32i	a5, a2, PT_AREG5
	l32i	a4, a2, PT_AREG4
	l32i	a3, a2, PT_AREG3

	/* restore SAR and return */

	wsr	a0, sar
	l32i	a0, a2, PT_AREG0
	l32i	a2, a2, PT_AREG2
	rfe

	/* We cannot handle this exception. */

	.extern _kernel_exception
.Linvalid_instruction_store:
.Linvalid_instruction:

	/* Restore a4...a8 and SAR, set SP, and jump to default exception. */

	l32i	a8, a2, PT_AREG8
	l32i	a7, a2, PT_AREG7
	l32i	a6, a2, PT_AREG6
	l32i	a5, a2, PT_AREG5
	l32i	a4, a2, PT_AREG4
	wsr	a0, sar
	mov	a1, a2

	rsr	a0, ps
	bbsi.l  a2, PS_UM_BIT, 1f     # jump if user mode

	movi	a0, _kernel_exception
	jx	a0

1:	movi	a0, _user_exception
	jx	a0

ENDPROC(fast_unaligned)

#endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION || XCHAL_UNALIGNED_STORE_EXCEPTION */
Commit	Line	Data
5a0015d6 CZ	1	/*
	2	* arch/xtensa/kernel/align.S
	3	*
	4	* Handle unalignment exceptions in kernel space.
	5	*
	6	* This file is subject to the terms and conditions of the GNU General
	7	* Public License. See the file "COPYING" in the main directory of
	8	* this archive for more details.
	9	*
	10	* Copyright (C) 2001 - 2005 Tensilica, Inc.
	11	*
	12	* Rewritten by Chris Zankel <chris@zankel.net>
	13	*
	14	* Based on work from Joe Taylor <joe@tensilica.com, joetylr@yahoo.com>
	15	* and Marc Gauthier <marc@tensilica.com, marc@alimni.uwaterloo.ca>
	16	*/
	17
	18	#include <linux/linkage.h>
5a0015d6	19	#include <asm/current.h>
0013a854	20	#include <asm/asm-offsets.h>
5a0015d6	21	#include <asm/processor.h>
5a0015d6 CZ	22
	23	#if XCHAL_UNALIGNED_LOAD_EXCEPTION \|\| XCHAL_UNALIGNED_STORE_EXCEPTION
	24
	25	/* First-level exception handler for unaligned exceptions.
	26	*
	27	* Note: This handler works only for kernel exceptions. Unaligned user
	28	* access should get a seg fault.
	29	*/
	30
	31	/* Big and little endian 16-bit values are located in
	32	* different halves of a register. HWORD_START helps to
	33	* abstract the notion of extracting a 16-bit value from a
	34	* register.
	35	* We also have to define new shifting instructions because
	36	* lsb and msb are on 'opposite' ends in a register for
	37	* different endian machines.
	38	*
	39	* Assume a memory region in ascending address:
	40	* 0 1 2 3\|4 5 6 7
	41	*
	42	* When loading one word into a register, the content of that register is:
	43	* LE 3 2 1 0, 7 6 5 4
	44	* BE 0 1 2 3, 4 5 6 7
	45	*
	46	* Masking the bits of the higher/lower address means:
	47	* LE X X 0 0, 0 0 X X
	48	* BE 0 0 X X, X X 0 0
	49	*
	50	* Shifting to higher/lower addresses, means:
	51	* LE shift left / shift right
	52	* BE shift right / shift left
	53	*
	54	* Extracting 16 bits from a 32 bit reg. value to higher/lower address means:
	55	* LE mask 0 0 X X / shift left
	56	* BE shift left / mask 0 0 X X
	57	*/
	58
	59	#define UNALIGNED_USER_EXCEPTION
	60
	61	#if XCHAL_HAVE_BE
	62
	63	#define HWORD_START 16
	64	#define INSN_OP0 28
	65	#define INSN_T 24
	66	#define INSN_OP1 16
	67
	68	.macro __src_b r, w0, w1; src \r, \w0, \w1; .endm
	69	.macro __ssa8 r; ssa8b \r; .endm
	70	.macro __ssa8r r; ssa8l \r; .endm
	71	.macro __sh r, s; srl \r, \s; .endm
	72	.macro __sl r, s; sll \r, \s; .endm
	73	.macro __exth r, s; extui \r, \s, 0, 16; .endm
	74	.macro __extl r, s; slli \r, \s, 16; .endm
	75
	76	#else
	77
	78	#define HWORD_START 0
	79	#define INSN_OP0 0
	80	#define INSN_T 4
	81	#define INSN_OP1 12
	82
	83	.macro __src_b r, w0, w1; src \r, \w1, \w0; .endm
	84	.macro __ssa8 r; ssa8l \r; .endm
	85	.macro __ssa8r r; ssa8b \r; .endm
86	.macro __sh r, s; sll \r, \s; .endm
87	.macro __sl r, s; srl \r, \s; .endm
88	.macro __exth r, s; slli \r, \s, 16; .endm
89	.macro __extl r, s; extui \r, \s, 0, 16; .endm
90
91	#endif
92
93	/*
94	* xxxx xxxx = imm8 field
95	* yyyy = imm4 field
96	* ssss = s field
97	* tttt = t field
98	*
99	* 16 0
100	* -------------------
101	* L32I.N yyyy ssss tttt 1000
102	* S32I.N yyyy ssss tttt 1001
103	*
104	* 23 0
105	* -----------------------------
106	* res 0000 0010
107	* L16UI xxxx xxxx 0001 ssss tttt 0010
108	* L32I xxxx xxxx 0010 ssss tttt 0010
109	* XXX 0011 ssss tttt 0010
110	* XXX 0100 ssss tttt 0010
111	* S16I xxxx xxxx 0101 ssss tttt 0010
112	* S32I xxxx xxxx 0110 ssss tttt 0010
113	* XXX 0111 ssss tttt 0010
114	* XXX 1000 ssss tttt 0010
115	* L16SI xxxx xxxx 1001 ssss tttt 0010
116	* XXX 1010 0010
117	* **L32AI xxxx xxxx 1011 ssss tttt 0010 unsupported
118	* XXX 1100 0010
119	* XXX 1101 0010
120	* XXX 1110 0010
121	* **S32RI xxxx xxxx 1111 ssss tttt 0010 unsupported
122	* -----------------------------
123	* ^ ^ ^
124	* sub-opcode (NIBBLE_R) -+ \| \|
125	* t field (NIBBLE_T) -----------+ \|
126	* major opcode (NIBBLE_OP0) --------------+
127	*/
128
129	#define OP0_L32I_N 0x8 /* load immediate narrow */
130	#define OP0_S32I_N 0x9 /* store immediate narrow */
131	#define OP1_SI_MASK 0x4 /* OP1 bit set for stores */
132	#define OP1_SI_BIT 2 /* OP1 bit number for stores */
133
134	#define OP1_L32I 0x2
135	#define OP1_L16UI 0x1
136	#define OP1_L16SI 0x9
137	#define OP1_L32AI 0xb
138
139	#define OP1_S32I 0x6
140	#define OP1_S16I 0x5
141	#define OP1_S32RI 0xf
142
143	/*
144	* Entry condition:
145	*
146	* a0: trashed, original value saved on stack (PT_AREG0)
147	* a1: a1
148	* a2: new stack pointer, original in DEPC
99d5040e	149	* a3: a3
5a0015d6	150	* depc: a2, original value saved on stack (PT_DEPC)
99d5040e	151	* excsave_1: dispatch table
5a0015d6 CZ	152	*
	153	* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
	154	* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
	155	*/
	156
	157
	158	ENTRY(fast_unaligned)
	159
	160	/* Note: We don't expect the address to be aligned on a word
	161	* boundary. After all, the processor generated that exception
	162	* and it would be a hardware fault.
	163	*/
	164
	165	/* Save some working register */
	166
	167	s32i a4, a2, PT_AREG4
	168	s32i a5, a2, PT_AREG5
	169	s32i a6, a2, PT_AREG6
	170	s32i a7, a2, PT_AREG7
	171	s32i a8, a2, PT_AREG8
	172
bc5378fc	173	rsr a0, depc
5a0015d6 CZ	174	s32i a0, a2, PT_AREG2
	175	s32i a3, a2, PT_AREG3
	176
	177	/* Keep value of SAR in a0 */
	178
bc5378fc MF	179	rsr a0, sar
bc5378fc MF	180	rsr a8, excvaddr # load unaligned memory address
5a0015d6 CZ	181
	182	/* Now, identify one of the following load/store instructions.
	183	*
	184	* The only possible danger of a double exception on the
	185	* following l32i instructions is kernel code in vmalloc
	186	* memory. The processor was just executing at the EPC_1
	187	* address, and indeed, already fetched the instruction. That
	188	* guarantees a TLB mapping, which hasn't been replaced by
	189	* this unaligned exception handler that uses only static TLB
	190	* mappings. However, high-level interrupt handlers might
	191	* modify TLB entries, so for the generic case, we register a
	192	* TABLE_FIXUP handler here, too.
	193	*/
	194
	195	/* a3...a6 saved on stack, a2 = SP */
	196
	197	/* Extract the instruction that caused the unaligned access. */
	198
bc5378fc	199	rsr a7, epc1 # load exception address
5a0015d6 CZ	200	movi a3, ~3
	201	and a3, a3, a7 # mask lower bits
	202
	203	l32i a4, a3, 0 # load 2 words
	204	l32i a5, a3, 4
	205
	206	__ssa8 a7
	207	__src_b a4, a4, a5 # a4 has the instruction
	208
	209	/* Analyze the instruction (load or store?). */
	210
	211	extui a5, a4, INSN_OP0, 4 # get insn.op0 nibble
	212
173d6681	213	#if XCHAL_HAVE_DENSITY
5a0015d6 CZ	214	_beqi a5, OP0_L32I_N, .Lload # L32I.N, jump
	215	addi a6, a5, -OP0_S32I_N
	216	_beqz a6, .Lstore # S32I.N, do a store
	217	#endif
	218	/* 'store indicator bit' not set, jump */
	219	_bbci.l a4, OP1_SI_BIT + INSN_OP1, .Lload
	220
	221	/* Store: Jump to table entry to get the value in the source register.*/
	222
	223	.Lstore:movi a5, .Lstore_table # table
	224	extui a6, a4, INSN_T, 4 # get source register
	225	addx8 a5, a6, a5
	226	jx a5 # jump into table
	227
	228	/* Invalid instruction, CRITICAL! */
	229	.Linvalid_instruction_load:
	230	j .Linvalid_instruction
	231
	232	/* Load: Load memory address. */
	233
	234	.Lload: movi a3, ~3
	235	and a3, a3, a8 # align memory address
	236
	237	__ssa8 a8
	238	#ifdef UNALIGNED_USER_EXCEPTION
	239	addi a3, a3, 8
	240	l32e a5, a3, -8
	241	l32e a6, a3, -4
	242	#else
	243	l32i a5, a3, 0
	244	l32i a6, a3, 4
	245	#endif
	246	__src_b a3, a5, a6 # a3 has the data word
	247
173d6681	248	#if XCHAL_HAVE_DENSITY
5a0015d6 CZ	249	addi a7, a7, 2 # increment PC (assume 16-bit insn)
	250
	251	extui a5, a4, INSN_OP0, 4
	252	_beqi a5, OP0_L32I_N, 1f # l32i.n: jump
	253
	254	addi a7, a7, 1
	255	#else
	256	addi a7, a7, 3
	257	#endif
	258
	259	extui a5, a4, INSN_OP1, 4
	260	_beqi a5, OP1_L32I, 1f # l32i: jump
	261
	262	extui a3, a3, 0, 16 # extract lower 16 bits
	263	_beqi a5, OP1_L16UI, 1f
	264	addi a5, a5, -OP1_L16SI
	265	_bnez a5, .Linvalid_instruction_load
	266
	267	/* sign extend value */
	268
	269	slli a3, a3, 16
	270	srai a3, a3, 16
	271
	272	/* Set target register. */
	273
	274	1:
	275
173d6681	276	#if XCHAL_HAVE_LOOPS
bc5378fc	277	rsr a5, lend # check if we reached LEND
173d6681	278	bne a7, a5, 1f
bc5378fc	279	rsr a5, lcount # and LCOUNT != 0
173d6681 CZ	280	beqz a5, 1f
173d6681 CZ	281	addi a5, a5, -1 # decrement LCOUNT and set
bc5378fc MF	282	rsr a7, lbeg # set PC to LBEGIN
bc5378fc MF	283	wsr a5, lcount
5a0015d6 CZ	284	#endif
5a0015d6 CZ	285
bc5378fc	286	1: wsr a7, epc1 # skip load instruction
5a0015d6 CZ	287	extui a4, a4, INSN_T, 4 # extract target register
	288	movi a5, .Lload_table
	289	addx8 a4, a4, a5
	290	jx a4 # jump to entry for target register
	291
	292	.align 8
	293	.Lload_table:
	294	s32i a3, a2, PT_AREG0; _j .Lexit; .align 8
	295	mov a1, a3; _j .Lexit; .align 8 # fishy??
	296	s32i a3, a2, PT_AREG2; _j .Lexit; .align 8
	297	s32i a3, a2, PT_AREG3; _j .Lexit; .align 8
	298	s32i a3, a2, PT_AREG4; _j .Lexit; .align 8
	299	s32i a3, a2, PT_AREG5; _j .Lexit; .align 8
	300	s32i a3, a2, PT_AREG6; _j .Lexit; .align 8
	301	s32i a3, a2, PT_AREG7; _j .Lexit; .align 8
	302	s32i a3, a2, PT_AREG8; _j .Lexit; .align 8
	303	mov a9, a3 ; _j .Lexit; .align 8
	304	mov a10, a3 ; _j .Lexit; .align 8
	305	mov a11, a3 ; _j .Lexit; .align 8
	306	mov a12, a3 ; _j .Lexit; .align 8
	307	mov a13, a3 ; _j .Lexit; .align 8
	308	mov a14, a3 ; _j .Lexit; .align 8
	309	mov a15, a3 ; _j .Lexit; .align 8
	310
	311	.Lstore_table:
	312	l32i a3, a2, PT_AREG0; _j 1f; .align 8
	313	mov a3, a1; _j 1f; .align 8 # fishy??
	314	l32i a3, a2, PT_AREG2; _j 1f; .align 8
	315	l32i a3, a2, PT_AREG3; _j 1f; .align 8
	316	l32i a3, a2, PT_AREG4; _j 1f; .align 8
	317	l32i a3, a2, PT_AREG5; _j 1f; .align 8
	318	l32i a3, a2, PT_AREG6; _j 1f; .align 8
	319	l32i a3, a2, PT_AREG7; _j 1f; .align 8
	320	l32i a3, a2, PT_AREG8; _j 1f; .align 8
	321	mov a3, a9 ; _j 1f; .align 8
	322	mov a3, a10 ; _j 1f; .align 8
	323	mov a3, a11 ; _j 1f; .align 8
	324	mov a3, a12 ; _j 1f; .align 8
	325	mov a3, a13 ; _j 1f; .align 8
	326	mov a3, a14 ; _j 1f; .align 8
	327	mov a3, a15 ; _j 1f; .align 8
	328
	329	1: # a7: instruction pointer, a4: instruction, a3: value
	330
	331	movi a6, 0 # mask: ffffffff:00000000
	332
173d6681	333	#if XCHAL_HAVE_DENSITY
5a0015d6 CZ	334	addi a7, a7, 2 # incr. PC,assume 16-bit instruction
	335
	336	extui a5, a4, INSN_OP0, 4 # extract OP0
	337	addi a5, a5, -OP0_S32I_N
	338	_beqz a5, 1f # s32i.n: jump
	339
	340	addi a7, a7, 1 # increment PC, 32-bit instruction
	341	#else
	342	addi a7, a7, 3 # increment PC, 32-bit instruction
	343	#endif
	344
	345	extui a5, a4, INSN_OP1, 4 # extract OP1
	346	_beqi a5, OP1_S32I, 1f # jump if 32 bit store
	347	_bnei a5, OP1_S16I, .Linvalid_instruction_store
	348
	349	movi a5, -1
	350	__extl a3, a3 # get 16-bit value
	351	__exth a6, a5 # get 16-bit mask ffffffff:ffff0000
	352
	353	/* Get memory address */
	354
	355	1:
173d6681	356	#if XCHAL_HAVE_LOOPS
bc5378fc	357	rsr a4, lend # check if we reached LEND
173d6681	358	bne a7, a4, 1f
bc5378fc	359	rsr a4, lcount # and LCOUNT != 0
173d6681 CZ	360	beqz a4, 1f
173d6681 CZ	361	addi a4, a4, -1 # decrement LCOUNT and set
bc5378fc MF	362	rsr a7, lbeg # set PC to LBEGIN
bc5378fc MF	363	wsr a4, lcount
5a0015d6 CZ	364	#endif
5a0015d6 CZ	365
bc5378fc	366	1: wsr a7, epc1 # skip store instruction
5a0015d6 CZ	367	movi a4, ~3
	368	and a4, a4, a8 # align memory address
	369
	370	/* Insert value into memory */
	371
	372	movi a5, -1 # mask: ffffffff:XXXX0000
	373	#ifdef UNALIGNED_USER_EXCEPTION
	374	addi a4, a4, 8
	375	#endif
	376
	377	__ssa8r a8
	378	__src_b a7, a5, a6 # lo-mask F..F0..0 (BE) 0..0F..F (LE)
	379	__src_b a6, a6, a5 # hi-mask 0..0F..F (BE) F..F0..0 (LE)
	380	#ifdef UNALIGNED_USER_EXCEPTION
	381	l32e a5, a4, -8
	382	#else
	383	l32i a5, a4, 0 # load lower address word
	384	#endif
	385	and a5, a5, a7 # mask
	386	__sh a7, a3 # shift value
	387	or a5, a5, a7 # or with original value
	388	#ifdef UNALIGNED_USER_EXCEPTION
	389	s32e a5, a4, -8
	390	l32e a7, a4, -4
	391	#else
	392	s32i a5, a4, 0 # store
	393	l32i a7, a4, 4 # same for upper address word
	394	#endif
	395	__sl a5, a3
	396	and a6, a7, a6
	397	or a6, a6, a5
	398	#ifdef UNALIGNED_USER_EXCEPTION
	399	s32e a6, a4, -4
	400	#else
	401	s32i a6, a4, 4
	402	#endif
	403
	404	/* Done. restore stack and return */
	405
	406	.Lexit:
	407	movi a4, 0
bc5378fc	408	rsr a3, excsave1
5a0015d6 CZ	409	s32i a4, a3, EXC_TABLE_FIXUP
	410
	411	/* Restore working register */
	412
173d6681	413	l32i a8, a2, PT_AREG8
5a0015d6 CZ	414	l32i a7, a2, PT_AREG7
	415	l32i a6, a2, PT_AREG6
	416	l32i a5, a2, PT_AREG5
	417	l32i a4, a2, PT_AREG4
	418	l32i a3, a2, PT_AREG3
	419
	420	/* restore SAR and return */
	421
bc5378fc	422	wsr a0, sar
5a0015d6 CZ	423	l32i a0, a2, PT_AREG0
	424	l32i a2, a2, PT_AREG2
	425	rfe
	426
	427	/* We cannot handle this exception. */
	428
	429	.extern _kernel_exception
	430	.Linvalid_instruction_store:
	431	.Linvalid_instruction:
	432
	433	/* Restore a4...a8 and SAR, set SP, and jump to default exception. */
	434
	435	l32i a8, a2, PT_AREG8
	436	l32i a7, a2, PT_AREG7
	437	l32i a6, a2, PT_AREG6
	438	l32i a5, a2, PT_AREG5
	439	l32i a4, a2, PT_AREG4
bc5378fc	440	wsr a0, sar
5a0015d6 CZ	441	mov a1, a2
5a0015d6 CZ	442
bc5378fc	443	rsr a0, ps
c4c4594b	444	bbsi.l a2, PS_UM_BIT, 1f # jump if user mode
5a0015d6 CZ	445
	446	movi a0, _kernel_exception
	447	jx a0
	448
	449	1: movi a0, _user_exception
	450	jx a0
	451
d1538c46	452	ENDPROC(fast_unaligned)
5a0015d6 CZ	453
5a0015d6 CZ	454	#endif /* XCHAL_UNALIGNED_LOAD_EXCEPTION \|\| XCHAL_UNALIGNED_STORE_EXCEPTION */