[deliverable/linux.git] / arch / sparc64 / kernel / winfixup.S

/* $Id: winfixup.S,v 1.30 2002/02/09 19:49:30 davem Exp $
 *
 * winfixup.S: Handle cases where user stack pointer is found to be bogus.
 *
 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
 */

#include <asm/asi.h>
#include <asm/head.h>
#include <asm/page.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
#include <asm/spitfire.h>
#include <asm/thread_info.h>

	.text

set_pcontext:
	sethi	%hi(sparc64_kern_pri_context), %l1
	ldx	[%l1 + %lo(sparc64_kern_pri_context)], %l1
	mov	PRIMARY_CONTEXT, %g1
	stxa	%l1, [%g1] ASI_DMMU
	flush	%g6
	retl
	 nop

	.align	32

	/* Here are the rules, pay attention.
	 *
	 * The kernel is disallowed from touching user space while
	 * the trap level is greater than zero, except for from within
	 * the window spill/fill handlers.  This must be followed
	 * so that we can easily detect the case where we tried to
	 * spill/fill with a bogus (or unmapped) user stack pointer.
	 *
	 * These are layed out in a special way for cache reasons,
	 * don't touch...
	 */
	.globl	fill_fixup, spill_fixup
fill_fixup:
	rdpr		%tstate, %g1
	andcc		%g1, TSTATE_PRIV, %g0
	or		%g4, FAULT_CODE_WINFIXUP, %g4
	be,pt		%xcc, window_scheisse_from_user_common
	 and		%g1, TSTATE_CWP, %g1

	/* This is the extremely complex case, but it does happen from
	 * time to time if things are just right.  Essentially the restore
	 * done in rtrap right before going back to user mode, with tl=1
	 * and that levels trap stack registers all setup, took a fill trap,
	 * the user stack was not mapped in the tlb, and tlb miss occurred,
	 * the pte found was not valid, and a simple ref bit watch update
	 * could not satisfy the miss, so we got here.
	 *
	 * We must carefully unwind the state so we get back to tl=0, preserve
	 * all the register values we were going to give to the user.  Luckily
	 * most things are where they need to be, we also have the address
	 * which triggered the fault handy as well.
	 *
	 * Also note that we must preserve %l5 and %l6.  If the user was
	 * returning from a system call, we must make it look this way
	 * after we process the fill fault on the users stack.
	 *
	 * First, get into the window where the original restore was executed.
	 */

	rdpr		%wstate, %g2			! Grab user mode wstate.
	wrpr		%g1, %cwp			! Get into the right window.
	sll		%g2, 3, %g2			! NORMAL-->OTHER

	wrpr		%g0, 0x0, %canrestore		! Standard etrap stuff.
	wrpr		%g2, 0x0, %wstate		! This must be consistent.
	wrpr		%g0, 0x0, %otherwin		! We know this.
	call		set_pcontext			! Change contexts...
	 nop
	rdpr		%pstate, %l1			! Prepare to change globals.
	mov		%g6, %o7			! Get current.

	andn		%l1, PSTATE_MM, %l1		! We want to be in RMO
	stb		%g4, [%g6 + TI_FAULT_CODE]
	stx		%g5, [%g6 + TI_FAULT_ADDR]
	wrpr		%g0, 0x0, %tl			! Out of trap levels.
	wrpr		%l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
	mov		%o7, %g6
	ldx		[%g6 + TI_TASK], %g4
#ifdef CONFIG_SMP
#error IMMU TSB usage must be fixed
	mov		TSB_REG, %g1
	ldxa		[%g1] ASI_IMMU, %g5
#endif

	/* This is the same as below, except we handle this a bit special
	 * since we must preserve %l5 and %l6, see comment above.
	 */
	call		do_sparc64_fault
	 add		%sp, PTREGS_OFF, %o0
	ba,pt		%xcc, rtrap
	 nop						! yes, nop is correct

	/* Be very careful about usage of the alternate globals here.
	 * You cannot touch %g4/%g5 as that has the fault information
	 * should this be from usermode.  Also be careful for the case
	 * where we get here from the save instruction in etrap.S when
	 * coming from either user or kernel (does not matter which, it
	 * is the same problem in both cases).  Essentially this means
	 * do not touch %g7 or %g2 so we handle the two cases fine.
	 */
spill_fixup:
	ldx		[%g6 + TI_FLAGS], %g1
	andcc		%g1, _TIF_32BIT, %g0
	ldub		[%g6 + TI_WSAVED], %g1

	sll		%g1, 3, %g3
	add		%g6, %g3, %g3
	stx		%sp, [%g3 + TI_RWIN_SPTRS]
	sll		%g1, 7, %g3
	bne,pt		%xcc, 1f
	 add		%g6, %g3, %g3
	stx		%l0, [%g3 + TI_REG_WINDOW + 0x00]
	stx		%l1, [%g3 + TI_REG_WINDOW + 0x08]

	stx		%l2, [%g3 + TI_REG_WINDOW + 0x10]
	stx		%l3, [%g3 + TI_REG_WINDOW + 0x18]
	stx		%l4, [%g3 + TI_REG_WINDOW + 0x20]
	stx		%l5, [%g3 + TI_REG_WINDOW + 0x28]
	stx		%l6, [%g3 + TI_REG_WINDOW + 0x30]
	stx		%l7, [%g3 + TI_REG_WINDOW + 0x38]
	stx		%i0, [%g3 + TI_REG_WINDOW + 0x40]
	stx		%i1, [%g3 + TI_REG_WINDOW + 0x48]

	stx		%i2, [%g3 + TI_REG_WINDOW + 0x50]
	stx		%i3, [%g3 + TI_REG_WINDOW + 0x58]
	stx		%i4, [%g3 + TI_REG_WINDOW + 0x60]
	stx		%i5, [%g3 + TI_REG_WINDOW + 0x68]
	stx		%i6, [%g3 + TI_REG_WINDOW + 0x70]
	b,pt		%xcc, 2f
	 stx		%i7, [%g3 + TI_REG_WINDOW + 0x78]
1:	stw		%l0, [%g3 + TI_REG_WINDOW + 0x00]

	stw		%l1, [%g3 + TI_REG_WINDOW + 0x04]
	stw		%l2, [%g3 + TI_REG_WINDOW + 0x08]
	stw		%l3, [%g3 + TI_REG_WINDOW + 0x0c]
	stw		%l4, [%g3 + TI_REG_WINDOW + 0x10]
	stw		%l5, [%g3 + TI_REG_WINDOW + 0x14]
	stw		%l6, [%g3 + TI_REG_WINDOW + 0x18]
	stw		%l7, [%g3 + TI_REG_WINDOW + 0x1c]
	stw		%i0, [%g3 + TI_REG_WINDOW + 0x20]

	stw		%i1, [%g3 + TI_REG_WINDOW + 0x24]
	stw		%i2, [%g3 + TI_REG_WINDOW + 0x28]
	stw		%i3, [%g3 + TI_REG_WINDOW + 0x2c]
	stw		%i4, [%g3 + TI_REG_WINDOW + 0x30]
	stw		%i5, [%g3 + TI_REG_WINDOW + 0x34]
	stw		%i6, [%g3 + TI_REG_WINDOW + 0x38]
	stw		%i7, [%g3 + TI_REG_WINDOW + 0x3c]
2:	add		%g1, 1, %g1

	stb		%g1, [%g6 + TI_WSAVED]
	rdpr		%tstate, %g1
	andcc		%g1, TSTATE_PRIV, %g0
	saved
	and		%g1, TSTATE_CWP, %g1
	be,pn		%xcc, window_scheisse_from_user_common
	 mov		FAULT_CODE_WRITE | FAULT_CODE_DTLB | FAULT_CODE_WINFIXUP, %g4
	retry

window_scheisse_from_user_common:
	stb		%g4, [%g6 + TI_FAULT_CODE]
	stx		%g5, [%g6 + TI_FAULT_ADDR]
	wrpr		%g1, %cwp
	ba,pt		%xcc, etrap
	 rd		%pc, %g7
	call		do_sparc64_fault
	 add		%sp, PTREGS_OFF, %o0
	ba,a,pt		%xcc, rtrap_clr_l6

	.globl		winfix_mna, fill_fixup_mna, spill_fixup_mna
winfix_mna:
	andn		%g3, 0x7f, %g3
	add		%g3, 0x78, %g3
	wrpr		%g3, %tnpc
	done
fill_fixup_mna:
	rdpr		%tstate, %g1
	andcc		%g1, TSTATE_PRIV, %g0
	be,pt		%xcc, window_mna_from_user_common
	 and		%g1, TSTATE_CWP, %g1

	/* Please, see fill_fixup commentary about why we must preserve
	 * %l5 and %l6 to preserve absolute correct semantics.
	 */
	rdpr		%wstate, %g2			! Grab user mode wstate.
	wrpr		%g1, %cwp			! Get into the right window.
	sll		%g2, 3, %g2			! NORMAL-->OTHER
	wrpr		%g0, 0x0, %canrestore		! Standard etrap stuff.

	wrpr		%g2, 0x0, %wstate		! This must be consistent.
	wrpr		%g0, 0x0, %otherwin		! We know this.
	call		set_pcontext			! Change contexts...
	 nop
	rdpr		%pstate, %l1			! Prepare to change globals.
	mov		%g4, %o2			! Setup args for
	mov		%g5, %o1			! final call to mem_address_unaligned.
	andn		%l1, PSTATE_MM, %l1		! We want to be in RMO

	mov		%g6, %o7			! Stash away current.
	wrpr		%g0, 0x0, %tl			! Out of trap levels.
	wrpr		%l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
	mov		%o7, %g6			! Get current back.
	ldx		[%g6 + TI_TASK], %g4		! Finish it.
#ifdef CONFIG_SMP
#error IMMU TSB usage must be fixed
	mov		TSB_REG, %g1
	ldxa		[%g1] ASI_IMMU, %g5
#endif
	call		mem_address_unaligned
	 add		%sp, PTREGS_OFF, %o0

	b,pt		%xcc, rtrap
	 nop						! yes, the nop is correct
spill_fixup_mna:
	ldx		[%g6 + TI_FLAGS], %g1
	andcc		%g1, _TIF_32BIT, %g0
	ldub		[%g6 + TI_WSAVED], %g1
	sll		%g1, 3, %g3
	add		%g6, %g3, %g3
	stx		%sp, [%g3 + TI_RWIN_SPTRS]

	sll		%g1, 7, %g3
	bne,pt		%xcc, 1f
	 add		%g6, %g3, %g3
	stx		%l0, [%g3 + TI_REG_WINDOW + 0x00]
	stx		%l1, [%g3 + TI_REG_WINDOW + 0x08]
	stx		%l2, [%g3 + TI_REG_WINDOW + 0x10]
	stx		%l3, [%g3 + TI_REG_WINDOW + 0x18]
	stx		%l4, [%g3 + TI_REG_WINDOW + 0x20]

	stx		%l5, [%g3 + TI_REG_WINDOW + 0x28]
	stx		%l6, [%g3 + TI_REG_WINDOW + 0x30]
	stx		%l7, [%g3 + TI_REG_WINDOW + 0x38]
	stx		%i0, [%g3 + TI_REG_WINDOW + 0x40]
	stx		%i1, [%g3 + TI_REG_WINDOW + 0x48]
	stx		%i2, [%g3 + TI_REG_WINDOW + 0x50]
	stx		%i3, [%g3 + TI_REG_WINDOW + 0x58]
	stx		%i4, [%g3 + TI_REG_WINDOW + 0x60]

	stx		%i5, [%g3 + TI_REG_WINDOW + 0x68]
	stx		%i6, [%g3 + TI_REG_WINDOW + 0x70]
	stx		%i7, [%g3 + TI_REG_WINDOW + 0x78]
	b,pt		%xcc, 2f
	 add		%g1, 1, %g1
1:	std		%l0, [%g3 + TI_REG_WINDOW + 0x00]
	std		%l2, [%g3 + TI_REG_WINDOW + 0x08]
	std		%l4, [%g3 + TI_REG_WINDOW + 0x10]

	std		%l6, [%g3 + TI_REG_WINDOW + 0x18]
	std		%i0, [%g3 + TI_REG_WINDOW + 0x20]
	std		%i2, [%g3 + TI_REG_WINDOW + 0x28]
	std		%i4, [%g3 + TI_REG_WINDOW + 0x30]
	std		%i6, [%g3 + TI_REG_WINDOW + 0x38]
	add		%g1, 1, %g1
2:	stb		%g1, [%g6 + TI_WSAVED]
	rdpr		%tstate, %g1

	andcc		%g1, TSTATE_PRIV, %g0
	saved
	be,pn		%xcc, window_mna_from_user_common
	 and		%g1, TSTATE_CWP, %g1
	retry
window_mna_from_user_common:
	wrpr		%g1, %cwp
	sethi		%hi(109f), %g7
	ba,pt		%xcc, etrap
109:	 or		%g7, %lo(109b), %g7
	mov		%l4, %o2
	mov		%l5, %o1
	call		mem_address_unaligned
	 add		%sp, PTREGS_OFF, %o0
	ba,pt		%xcc, rtrap
	 clr		%l6
	
	.globl		winfix_dax, fill_fixup_dax, spill_fixup_dax
winfix_dax:
	andn		%g3, 0x7f, %g3
	add		%g3, 0x74, %g3
	wrpr		%g3, %tnpc
	done
fill_fixup_dax:
	rdpr		%tstate, %g1
	andcc		%g1, TSTATE_PRIV, %g0
	be,pt		%xcc, window_dax_from_user_common
	 and		%g1, TSTATE_CWP, %g1

	/* Please, see fill_fixup commentary about why we must preserve
	 * %l5 and %l6 to preserve absolute correct semantics.
	 */
	rdpr		%wstate, %g2			! Grab user mode wstate.
	wrpr		%g1, %cwp			! Get into the right window.
	sll		%g2, 3, %g2			! NORMAL-->OTHER
	wrpr		%g0, 0x0, %canrestore		! Standard etrap stuff.

	wrpr		%g2, 0x0, %wstate		! This must be consistent.
	wrpr		%g0, 0x0, %otherwin		! We know this.
	call		set_pcontext			! Change contexts...
	 nop
	rdpr		%pstate, %l1			! Prepare to change globals.
	mov		%g4, %o1			! Setup args for
	mov		%g5, %o2			! final call to spitfire_data_access_exception.
	andn		%l1, PSTATE_MM, %l1		! We want to be in RMO

	mov		%g6, %o7			! Stash away current.
	wrpr		%g0, 0x0, %tl			! Out of trap levels.
	wrpr		%l1, (PSTATE_IE | PSTATE_AG | PSTATE_RMO), %pstate
	mov		%o7, %g6			! Get current back.
	ldx		[%g6 + TI_TASK], %g4		! Finish it.
#ifdef CONFIG_SMP
#error IMMU TSB usage must be fixed
	mov		TSB_REG, %g1
	ldxa		[%g1] ASI_IMMU, %g5
#endif
	call		spitfire_data_access_exception
	 add		%sp, PTREGS_OFF, %o0

	b,pt		%xcc, rtrap
	 nop						! yes, the nop is correct
spill_fixup_dax:
	ldx		[%g6 + TI_FLAGS], %g1
	andcc		%g1, _TIF_32BIT, %g0
	ldub		[%g6 + TI_WSAVED], %g1
	sll		%g1, 3, %g3
	add		%g6, %g3, %g3
	stx		%sp, [%g3 + TI_RWIN_SPTRS]

	sll		%g1, 7, %g3
	bne,pt		%xcc, 1f
	 add		%g6, %g3, %g3
	stx		%l0, [%g3 + TI_REG_WINDOW + 0x00]
	stx		%l1, [%g3 + TI_REG_WINDOW + 0x08]
	stx		%l2, [%g3 + TI_REG_WINDOW + 0x10]
	stx		%l3, [%g3 + TI_REG_WINDOW + 0x18]
	stx		%l4, [%g3 + TI_REG_WINDOW + 0x20]

	stx		%l5, [%g3 + TI_REG_WINDOW + 0x28]
	stx		%l6, [%g3 + TI_REG_WINDOW + 0x30]
	stx		%l7, [%g3 + TI_REG_WINDOW + 0x38]
	stx		%i0, [%g3 + TI_REG_WINDOW + 0x40]
	stx		%i1, [%g3 + TI_REG_WINDOW + 0x48]
	stx		%i2, [%g3 + TI_REG_WINDOW + 0x50]
	stx		%i3, [%g3 + TI_REG_WINDOW + 0x58]
	stx		%i4, [%g3 + TI_REG_WINDOW + 0x60]

	stx		%i5, [%g3 + TI_REG_WINDOW + 0x68]
	stx		%i6, [%g3 + TI_REG_WINDOW + 0x70]
	stx		%i7, [%g3 + TI_REG_WINDOW + 0x78]
	b,pt		%xcc, 2f
	 add		%g1, 1, %g1
1:	std		%l0, [%g3 + TI_REG_WINDOW + 0x00]
	std		%l2, [%g3 + TI_REG_WINDOW + 0x08]
	std		%l4, [%g3 + TI_REG_WINDOW + 0x10]

	std		%l6, [%g3 + TI_REG_WINDOW + 0x18]
	std		%i0, [%g3 + TI_REG_WINDOW + 0x20]
	std		%i2, [%g3 + TI_REG_WINDOW + 0x28]
	std		%i4, [%g3 + TI_REG_WINDOW + 0x30]
	std		%i6, [%g3 + TI_REG_WINDOW + 0x38]
	add		%g1, 1, %g1
2:	stb		%g1, [%g6 + TI_WSAVED]
	rdpr		%tstate, %g1

	andcc		%g1, TSTATE_PRIV, %g0
	saved
	be,pn		%xcc, window_dax_from_user_common
	 and		%g1, TSTATE_CWP, %g1
	retry
window_dax_from_user_common:
	wrpr		%g1, %cwp
	sethi		%hi(109f), %g7
	ba,pt		%xcc, etrap
109:	 or		%g7, %lo(109b), %g7
	mov		%l4, %o1
	mov		%l5, %o2
	call		spitfire_data_access_exception
	 add		%sp, PTREGS_OFF, %o0
	ba,pt		%xcc, rtrap
	 clr		%l6
Commit	Line	Data
1da177e4 LT	1	/* $Id: winfixup.S,v 1.30 2002/02/09 19:49:30 davem Exp $
	2	*
	3	* winfixup.S: Handle cases where user stack pointer is found to be bogus.
	4	*
	5	* Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
	6	*/
	7
	8	#include <asm/asi.h>
	9	#include <asm/head.h>
	10	#include <asm/page.h>
	11	#include <asm/ptrace.h>
	12	#include <asm/processor.h>
	13	#include <asm/spitfire.h>
	14	#include <asm/thread_info.h>
	15
	16	.text
	17
	18	set_pcontext:
0835ae0f DM	19	sethi %hi(sparc64_kern_pri_context), %l1
0835ae0f DM	20	ldx [%l1 + %lo(sparc64_kern_pri_context)], %l1
1da177e4	21	mov PRIMARY_CONTEXT, %g1
1da177e4 LT	22	stxa %l1, [%g1] ASI_DMMU
	23	flush %g6
	24	retl
	25	nop
	26
1da177e4 LT	27	.align 32
	28
	29	/* Here are the rules, pay attention.
	30	*
	31	* The kernel is disallowed from touching user space while
	32	* the trap level is greater than zero, except for from within
	33	* the window spill/fill handlers. This must be followed
	34	* so that we can easily detect the case where we tried to
	35	* spill/fill with a bogus (or unmapped) user stack pointer.
	36	*
	37	* These are layed out in a special way for cache reasons,
	38	* don't touch...
	39	*/
	40	.globl fill_fixup, spill_fixup
	41	fill_fixup:
	42	rdpr %tstate, %g1
	43	andcc %g1, TSTATE_PRIV, %g0
	44	or %g4, FAULT_CODE_WINFIXUP, %g4
	45	be,pt %xcc, window_scheisse_from_user_common
	46	and %g1, TSTATE_CWP, %g1
	47
	48	/* This is the extremely complex case, but it does happen from
	49	* time to time if things are just right. Essentially the restore
	50	* done in rtrap right before going back to user mode, with tl=1
	51	* and that levels trap stack registers all setup, took a fill trap,
	52	* the user stack was not mapped in the tlb, and tlb miss occurred,
	53	* the pte found was not valid, and a simple ref bit watch update
	54	* could not satisfy the miss, so we got here.
	55	*
	56	* We must carefully unwind the state so we get back to tl=0, preserve
	57	* all the register values we were going to give to the user. Luckily
	58	* most things are where they need to be, we also have the address
	59	* which triggered the fault handy as well.
	60	*
	61	* Also note that we must preserve %l5 and %l6. If the user was
	62	* returning from a system call, we must make it look this way
	63	* after we process the fill fault on the users stack.
	64	*
	65	* First, get into the window where the original restore was executed.
	66	*/
	67
	68	rdpr %wstate, %g2 ! Grab user mode wstate.
	69	wrpr %g1, %cwp ! Get into the right window.
	70	sll %g2, 3, %g2 ! NORMAL-->OTHER
	71
	72	wrpr %g0, 0x0, %canrestore ! Standard etrap stuff.
	73	wrpr %g2, 0x0, %wstate ! This must be consistent.
	74	wrpr %g0, 0x0, %otherwin ! We know this.
	75	call set_pcontext ! Change contexts...
	76	nop
	77	rdpr %pstate, %l1 ! Prepare to change globals.
	78	mov %g6, %o7 ! Get current.
	79
	80	andn %l1, PSTATE_MM, %l1 ! We want to be in RMO
	81	stb %g4, [%g6 + TI_FAULT_CODE]
	82	stx %g5, [%g6 + TI_FAULT_ADDR]
	83	wrpr %g0, 0x0, %tl ! Out of trap levels.
	84	wrpr %l1, (PSTATE_IE \| PSTATE_AG \| PSTATE_RMO), %pstate
	85	mov %o7, %g6
	86	ldx [%g6 + TI_TASK], %g4
	87	#ifdef CONFIG_SMP
74bf4312	88	#error IMMU TSB usage must be fixed
1da177e4 LT	89	mov TSB_REG, %g1
	90	ldxa [%g1] ASI_IMMU, %g5
	91	#endif
	92
	93	/* This is the same as below, except we handle this a bit special
	94	* since we must preserve %l5 and %l6, see comment above.
	95	*/
	96	call do_sparc64_fault
	97	add %sp, PTREGS_OFF, %o0
	98	ba,pt %xcc, rtrap
	99	nop ! yes, nop is correct
	100
	101	/* Be very careful about usage of the alternate globals here.
	102	* You cannot touch %g4/%g5 as that has the fault information
	103	* should this be from usermode. Also be careful for the case
	104	* where we get here from the save instruction in etrap.S when
	105	* coming from either user or kernel (does not matter which, it
	106	* is the same problem in both cases). Essentially this means
	107	* do not touch %g7 or %g2 so we handle the two cases fine.
	108	*/
	109	spill_fixup:
	110	ldx [%g6 + TI_FLAGS], %g1
	111	andcc %g1, _TIF_32BIT, %g0
	112	ldub [%g6 + TI_WSAVED], %g1
	113
	114	sll %g1, 3, %g3
	115	add %g6, %g3, %g3
	116	stx %sp, [%g3 + TI_RWIN_SPTRS]
	117	sll %g1, 7, %g3
	118	bne,pt %xcc, 1f
	119	add %g6, %g3, %g3
	120	stx %l0, [%g3 + TI_REG_WINDOW + 0x00]
	121	stx %l1, [%g3 + TI_REG_WINDOW + 0x08]
	122
	123	stx %l2, [%g3 + TI_REG_WINDOW + 0x10]
	124	stx %l3, [%g3 + TI_REG_WINDOW + 0x18]
	125	stx %l4, [%g3 + TI_REG_WINDOW + 0x20]
	126	stx %l5, [%g3 + TI_REG_WINDOW + 0x28]
	127	stx %l6, [%g3 + TI_REG_WINDOW + 0x30]
	128	stx %l7, [%g3 + TI_REG_WINDOW + 0x38]
	129	stx %i0, [%g3 + TI_REG_WINDOW + 0x40]
	130	stx %i1, [%g3 + TI_REG_WINDOW + 0x48]
	131
	132	stx %i2, [%g3 + TI_REG_WINDOW + 0x50]
	133	stx %i3, [%g3 + TI_REG_WINDOW + 0x58]
	134	stx %i4, [%g3 + TI_REG_WINDOW + 0x60]
	135	stx %i5, [%g3 + TI_REG_WINDOW + 0x68]
	136	stx %i6, [%g3 + TI_REG_WINDOW + 0x70]
	137	b,pt %xcc, 2f
	138	stx %i7, [%g3 + TI_REG_WINDOW + 0x78]
	139	1: stw %l0, [%g3 + TI_REG_WINDOW + 0x00]
	140
	141	stw %l1, [%g3 + TI_REG_WINDOW + 0x04]
	142	stw %l2, [%g3 + TI_REG_WINDOW + 0x08]
	143	stw %l3, [%g3 + TI_REG_WINDOW + 0x0c]
	144	stw %l4, [%g3 + TI_REG_WINDOW + 0x10]
	145	stw %l5, [%g3 + TI_REG_WINDOW + 0x14]
	146	stw %l6, [%g3 + TI_REG_WINDOW + 0x18]
	147	stw %l7, [%g3 + TI_REG_WINDOW + 0x1c]
	148	stw %i0, [%g3 + TI_REG_WINDOW + 0x20]
	149
	150	stw %i1, [%g3 + TI_REG_WINDOW + 0x24]
	151	stw %i2, [%g3 + TI_REG_WINDOW + 0x28]
	152	stw %i3, [%g3 + TI_REG_WINDOW + 0x2c]
153	stw %i4, [%g3 + TI_REG_WINDOW + 0x30]
154	stw %i5, [%g3 + TI_REG_WINDOW + 0x34]
155	stw %i6, [%g3 + TI_REG_WINDOW + 0x38]
156	stw %i7, [%g3 + TI_REG_WINDOW + 0x3c]
157	2: add %g1, 1, %g1
158
159	stb %g1, [%g6 + TI_WSAVED]
160	rdpr %tstate, %g1
161	andcc %g1, TSTATE_PRIV, %g0
162	saved
163	and %g1, TSTATE_CWP, %g1
164	be,pn %xcc, window_scheisse_from_user_common
165	mov FAULT_CODE_WRITE \| FAULT_CODE_DTLB \| FAULT_CODE_WINFIXUP, %g4
166	retry
167
168	window_scheisse_from_user_common:
169	stb %g4, [%g6 + TI_FAULT_CODE]
170	stx %g5, [%g6 + TI_FAULT_ADDR]
171	wrpr %g1, %cwp
172	ba,pt %xcc, etrap
173	rd %pc, %g7
174	call do_sparc64_fault
175	add %sp, PTREGS_OFF, %o0
176	ba,a,pt %xcc, rtrap_clr_l6
177
178	.globl winfix_mna, fill_fixup_mna, spill_fixup_mna
179	winfix_mna:
180	andn %g3, 0x7f, %g3
181	add %g3, 0x78, %g3
182	wrpr %g3, %tnpc
183	done
184	fill_fixup_mna:
185	rdpr %tstate, %g1
186	andcc %g1, TSTATE_PRIV, %g0
187	be,pt %xcc, window_mna_from_user_common
188	and %g1, TSTATE_CWP, %g1
189
190	/* Please, see fill_fixup commentary about why we must preserve
191	* %l5 and %l6 to preserve absolute correct semantics.
192	*/
193	rdpr %wstate, %g2 ! Grab user mode wstate.
194	wrpr %g1, %cwp ! Get into the right window.
195	sll %g2, 3, %g2 ! NORMAL-->OTHER
196	wrpr %g0, 0x0, %canrestore ! Standard etrap stuff.
197
198	wrpr %g2, 0x0, %wstate ! This must be consistent.
199	wrpr %g0, 0x0, %otherwin ! We know this.
200	call set_pcontext ! Change contexts...
201	nop
202	rdpr %pstate, %l1 ! Prepare to change globals.
203	mov %g4, %o2 ! Setup args for
204	mov %g5, %o1 ! final call to mem_address_unaligned.
205	andn %l1, PSTATE_MM, %l1 ! We want to be in RMO
206
207	mov %g6, %o7 ! Stash away current.
208	wrpr %g0, 0x0, %tl ! Out of trap levels.
209	wrpr %l1, (PSTATE_IE \| PSTATE_AG \| PSTATE_RMO), %pstate
210	mov %o7, %g6 ! Get current back.
211	ldx [%g6 + TI_TASK], %g4 ! Finish it.
212	#ifdef CONFIG_SMP
74bf4312	213	#error IMMU TSB usage must be fixed
1da177e4 LT	214	mov TSB_REG, %g1
	215	ldxa [%g1] ASI_IMMU, %g5
	216	#endif
	217	call mem_address_unaligned
	218	add %sp, PTREGS_OFF, %o0
	219
	220	b,pt %xcc, rtrap
	221	nop ! yes, the nop is correct
	222	spill_fixup_mna:
	223	ldx [%g6 + TI_FLAGS], %g1
	224	andcc %g1, _TIF_32BIT, %g0
	225	ldub [%g6 + TI_WSAVED], %g1
	226	sll %g1, 3, %g3
	227	add %g6, %g3, %g3
	228	stx %sp, [%g3 + TI_RWIN_SPTRS]
	229
	230	sll %g1, 7, %g3
	231	bne,pt %xcc, 1f
	232	add %g6, %g3, %g3
	233	stx %l0, [%g3 + TI_REG_WINDOW + 0x00]
	234	stx %l1, [%g3 + TI_REG_WINDOW + 0x08]
	235	stx %l2, [%g3 + TI_REG_WINDOW + 0x10]
	236	stx %l3, [%g3 + TI_REG_WINDOW + 0x18]
	237	stx %l4, [%g3 + TI_REG_WINDOW + 0x20]
	238
	239	stx %l5, [%g3 + TI_REG_WINDOW + 0x28]
	240	stx %l6, [%g3 + TI_REG_WINDOW + 0x30]
	241	stx %l7, [%g3 + TI_REG_WINDOW + 0x38]
	242	stx %i0, [%g3 + TI_REG_WINDOW + 0x40]
	243	stx %i1, [%g3 + TI_REG_WINDOW + 0x48]
	244	stx %i2, [%g3 + TI_REG_WINDOW + 0x50]
	245	stx %i3, [%g3 + TI_REG_WINDOW + 0x58]
	246	stx %i4, [%g3 + TI_REG_WINDOW + 0x60]
	247
	248	stx %i5, [%g3 + TI_REG_WINDOW + 0x68]
	249	stx %i6, [%g3 + TI_REG_WINDOW + 0x70]
	250	stx %i7, [%g3 + TI_REG_WINDOW + 0x78]
	251	b,pt %xcc, 2f
	252	add %g1, 1, %g1
	253	1: std %l0, [%g3 + TI_REG_WINDOW + 0x00]
	254	std %l2, [%g3 + TI_REG_WINDOW + 0x08]
	255	std %l4, [%g3 + TI_REG_WINDOW + 0x10]
	256
	257	std %l6, [%g3 + TI_REG_WINDOW + 0x18]
	258	std %i0, [%g3 + TI_REG_WINDOW + 0x20]
	259	std %i2, [%g3 + TI_REG_WINDOW + 0x28]
	260	std %i4, [%g3 + TI_REG_WINDOW + 0x30]
	261	std %i6, [%g3 + TI_REG_WINDOW + 0x38]
	262	add %g1, 1, %g1
	263	2: stb %g1, [%g6 + TI_WSAVED]
	264	rdpr %tstate, %g1
	265
	266	andcc %g1, TSTATE_PRIV, %g0
	267	saved
	268	be,pn %xcc, window_mna_from_user_common
	269	and %g1, TSTATE_CWP, %g1
	270	retry
	271	window_mna_from_user_common:
	272	wrpr %g1, %cwp
	273	sethi %hi(109f), %g7
	274	ba,pt %xcc, etrap
	275	109: or %g7, %lo(109b), %g7
	276	mov %l4, %o2
	277	mov %l5, %o1
278	call mem_address_unaligned
279	add %sp, PTREGS_OFF, %o0
280	ba,pt %xcc, rtrap
281	clr %l6
282
1da177e4 LT	283	.globl winfix_dax, fill_fixup_dax, spill_fixup_dax
	284	winfix_dax:
	285	andn %g3, 0x7f, %g3
	286	add %g3, 0x74, %g3
	287	wrpr %g3, %tnpc
	288	done
	289	fill_fixup_dax:
	290	rdpr %tstate, %g1
	291	andcc %g1, TSTATE_PRIV, %g0
	292	be,pt %xcc, window_dax_from_user_common
	293	and %g1, TSTATE_CWP, %g1
	294
	295	/* Please, see fill_fixup commentary about why we must preserve
	296	* %l5 and %l6 to preserve absolute correct semantics.
	297	*/
	298	rdpr %wstate, %g2 ! Grab user mode wstate.
	299	wrpr %g1, %cwp ! Get into the right window.
	300	sll %g2, 3, %g2 ! NORMAL-->OTHER
	301	wrpr %g0, 0x0, %canrestore ! Standard etrap stuff.
	302
	303	wrpr %g2, 0x0, %wstate ! This must be consistent.
	304	wrpr %g0, 0x0, %otherwin ! We know this.
	305	call set_pcontext ! Change contexts...
	306	nop
	307	rdpr %pstate, %l1 ! Prepare to change globals.
	308	mov %g4, %o1 ! Setup args for
6c52a96e	309	mov %g5, %o2 ! final call to spitfire_data_access_exception.
1da177e4 LT	310	andn %l1, PSTATE_MM, %l1 ! We want to be in RMO
	311
	312	mov %g6, %o7 ! Stash away current.
	313	wrpr %g0, 0x0, %tl ! Out of trap levels.
	314	wrpr %l1, (PSTATE_IE \| PSTATE_AG \| PSTATE_RMO), %pstate
	315	mov %o7, %g6 ! Get current back.
	316	ldx [%g6 + TI_TASK], %g4 ! Finish it.
	317	#ifdef CONFIG_SMP
74bf4312	318	#error IMMU TSB usage must be fixed
1da177e4 LT	319	mov TSB_REG, %g1
	320	ldxa [%g1] ASI_IMMU, %g5
	321	#endif
6c52a96e	322	call spitfire_data_access_exception
1da177e4 LT	323	add %sp, PTREGS_OFF, %o0
	324
	325	b,pt %xcc, rtrap
	326	nop ! yes, the nop is correct
	327	spill_fixup_dax:
	328	ldx [%g6 + TI_FLAGS], %g1
	329	andcc %g1, _TIF_32BIT, %g0
	330	ldub [%g6 + TI_WSAVED], %g1
	331	sll %g1, 3, %g3
	332	add %g6, %g3, %g3
	333	stx %sp, [%g3 + TI_RWIN_SPTRS]
	334
	335	sll %g1, 7, %g3
	336	bne,pt %xcc, 1f
	337	add %g6, %g3, %g3
	338	stx %l0, [%g3 + TI_REG_WINDOW + 0x00]
	339	stx %l1, [%g3 + TI_REG_WINDOW + 0x08]
	340	stx %l2, [%g3 + TI_REG_WINDOW + 0x10]
	341	stx %l3, [%g3 + TI_REG_WINDOW + 0x18]
	342	stx %l4, [%g3 + TI_REG_WINDOW + 0x20]
	343
	344	stx %l5, [%g3 + TI_REG_WINDOW + 0x28]
	345	stx %l6, [%g3 + TI_REG_WINDOW + 0x30]
	346	stx %l7, [%g3 + TI_REG_WINDOW + 0x38]
	347	stx %i0, [%g3 + TI_REG_WINDOW + 0x40]
	348	stx %i1, [%g3 + TI_REG_WINDOW + 0x48]
	349	stx %i2, [%g3 + TI_REG_WINDOW + 0x50]
	350	stx %i3, [%g3 + TI_REG_WINDOW + 0x58]
	351	stx %i4, [%g3 + TI_REG_WINDOW + 0x60]
	352
	353	stx %i5, [%g3 + TI_REG_WINDOW + 0x68]
	354	stx %i6, [%g3 + TI_REG_WINDOW + 0x70]
	355	stx %i7, [%g3 + TI_REG_WINDOW + 0x78]
	356	b,pt %xcc, 2f
	357	add %g1, 1, %g1
	358	1: std %l0, [%g3 + TI_REG_WINDOW + 0x00]
	359	std %l2, [%g3 + TI_REG_WINDOW + 0x08]
	360	std %l4, [%g3 + TI_REG_WINDOW + 0x10]
	361
	362	std %l6, [%g3 + TI_REG_WINDOW + 0x18]
	363	std %i0, [%g3 + TI_REG_WINDOW + 0x20]
	364	std %i2, [%g3 + TI_REG_WINDOW + 0x28]
	365	std %i4, [%g3 + TI_REG_WINDOW + 0x30]
	366	std %i6, [%g3 + TI_REG_WINDOW + 0x38]
	367	add %g1, 1, %g1
	368	2: stb %g1, [%g6 + TI_WSAVED]
	369	rdpr %tstate, %g1
	370
	371	andcc %g1, TSTATE_PRIV, %g0
	372	saved
	373	be,pn %xcc, window_dax_from_user_common
	374	and %g1, TSTATE_CWP, %g1
	375	retry
	376	window_dax_from_user_common:
	377	wrpr %g1, %cwp
	378	sethi %hi(109f), %g7
	379	ba,pt %xcc, etrap
	380	109: or %g7, %lo(109b), %g7
	381	mov %l4, %o1
	382	mov %l5, %o2
6c52a96e	383	call spitfire_data_access_exception
1da177e4 LT	384	add %sp, PTREGS_OFF, %o0
	385	ba,pt %xcc, rtrap
	386	clr %l6