[deliverable/linux.git] / arch / sparc64 / kernel / sun4v_tlb_miss.S

/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
 *
 * Copyright (C) 2006 <davem@davemloft.net>
 */

	.text
	.align	32

sun4v_itlb_miss:
	/* Load MMU Miss base into %g2.  */
	ldxa	[%g0] ASI_SCRATCHPAD, %g3
	
	/* Load UTSB reg into %g1.  */
	mov	SCRATCHPAD_UTSBREG1, %g1
	ldxa	[%g1] ASI_SCRATCHPAD, %g1

	/* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6.
	 * Branch if kernel TLB miss.  The kernel TSB and user TSB miss
	 * code wants the missing virtual address in %g4, so that value
	 * cannot be modified through the entirety of this handler.
	 */
	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
	srlx	%g4, 22, %g3
	sllx	%g5, 48, %g6
	or	%g6, %g3, %g6
	brz,pn	%g5, kvmap_itlb_4v
	 nop

	/* Create TSB pointer.  This is something like:
	 *
	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	 * tsb_base = tsb_reg & ~0x7UL;
	 */
	and	%g1, 0x7, %g3
	andn	%g1, 0x7, %g1
	mov	512, %g7
	sllx	%g7, %g3, %g7
	sub	%g7, 1, %g7

	/* TSB index mask is in %g7, tsb base is in %g1.  Compute
	 * the TSB entry pointer into %g1:
	 *
	 * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	 * tsb_ptr = tsb_base + (tsb_index * 16);
	 */
	srlx	%g4, PAGE_SHIFT, %g3
	and	%g3, %g7, %g3
	sllx	%g3, 4, %g3
	add	%g1, %g3, %g1

	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
	ldda	[%g1] ASI_QUAD_LDD_PHYS, %g2
	cmp	%g2, %g6
	sethi	%hi(_PAGE_EXEC), %g7
	bne,a,pn %xcc, tsb_miss_page_table_walk
	 mov	FAULT_CODE_ITLB, %g3
	andcc	%g3, %g7, %g0
	be,a,pn	%xcc, tsb_do_fault
	 mov	FAULT_CODE_ITLB, %g3

	/* We have a valid entry, make hypervisor call to load
	 * I-TLB and return from trap.
	 *
	 * %g3:	PTE
	 * %g4:	vaddr
	 * %g6:	TAG TARGET (only "CTX << 48" part matters)
	 */
sun4v_itlb_load:
	mov	%o0, %g1		! save %o0
	mov	%o1, %g2		! save %o1
	mov	%o2, %g5		! save %o2
	mov	%o3, %g7		! save %o3
	mov	%g4, %o0		! vaddr
	srlx	%g6, 48, %o1		! ctx
	mov	%g3, %o2		! PTE
	mov	HV_MMU_IMMU, %o3	! flags
	ta	HV_MMU_MAP_ADDR_TRAP
	mov	%g1, %o0		! restore %o0
	mov	%g2, %o1		! restore %o1
	mov	%g5, %o2		! restore %o2
	mov	%g7, %o3		! restore %o3

	retry

sun4v_dtlb_miss:
	/* Load MMU Miss base into %g2.  */
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	
	/* Load UTSB reg into %g1.  */
	mov	SCRATCHPAD_UTSBREG1, %g1
	ldxa	[%g1 + %g1] ASI_SCRATCHPAD, %g1

	/* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6.
	 * Branch if kernel TLB miss.  The kernel TSB and user TSB miss
	 * code wants the missing virtual address in %g4, so that value
	 * cannot be modified through the entirety of this handler.
	 */
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	srlx	%g4, 22, %g3
	sllx	%g5, 48, %g6
	or	%g6, %g3, %g6
	brz,pn	%g5, kvmap_dtlb_4v
	 nop

	/* Create TSB pointer.  This is something like:
	 *
	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	 * tsb_base = tsb_reg & ~0x7UL;
	 */
	and	%g1, 0x7, %g3
	andn	%g1, 0x7, %g1
	mov	512, %g7
	sllx	%g7, %g3, %g7
	sub	%g7, 1, %g7

	/* TSB index mask is in %g7, tsb base is in %g1.  Compute
	 * the TSB entry pointer into %g1:
	 *
	 * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	 * tsb_ptr = tsb_base + (tsb_index * 16);
	 */
	srlx	%g4, PAGE_SHIFT, %g3
	and	%g3, %g7, %g3
	sllx	%g3, 4, %g3
	add	%g1, %g3, %g1

	/* Load TSB tag/pte into %g2/%g3 and compare the tag.  */
	ldda	[%g1] ASI_QUAD_LDD_PHYS, %g2
	cmp	%g2, %g6
	bne,a,pn %xcc, tsb_miss_page_table_walk
	 mov	FAULT_CODE_ITLB, %g3

	/* We have a valid entry, make hypervisor call to load
	 * D-TLB and return from trap.
	 *
	 * %g3:	PTE
	 * %g4:	vaddr
	 * %g6:	TAG TARGET (only "CTX << 48" part matters)
	 */
sun4v_dtlb_load:
	mov	%o0, %g1		! save %o0
	mov	%o1, %g2		! save %o1
	mov	%o2, %g5		! save %o2
	mov	%o3, %g7		! save %o3
	mov	%g4, %o0		! vaddr
	srlx	%g6, 48, %o1		! ctx
	mov	%g3, %o2		! PTE
	mov	HV_MMU_DMMU, %o3	! flags
	ta	HV_MMU_MAP_ADDR_TRAP
	mov	%g1, %o0		! restore %o0
	mov	%g2, %o1		! restore %o1
	mov	%g5, %o2		! restore %o2
	mov	%g7, %o3		! restore %o3

	retry

sun4v_dtlb_prot:
	/* Load MMU Miss base into %g2.  */
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
	rdpr	%tl, %g1
	cmp	%g1, 1
	bgu,pn		%xcc, winfix_trampoline
	 nop
	ba,pt		%xcc, sparc64_realfault_common
	 mov		FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4

	/* Called from trap table with TAG TARGET placed into
	 * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1.
	 */
sun4v_itsb_miss:
	ba,pt	%xcc, sun4v_tsb_miss_common
	 mov	FAULT_CODE_ITLB, %g3

	/* Called from trap table with TAG TARGET placed into
	 * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1.
	 */
sun4v_dtsb_miss:
	mov	FAULT_CODE_DTLB, %g3

	/* Create TSB pointer into %g1.  This is something like:
	 *
	 * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	 * tsb_base = tsb_reg & ~0x7UL;
	 * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	 * tsb_ptr = tsb_base + (tsb_index * 16);
	 */
sun4v_tsb_miss_common:
	and	%g1, 0x7, %g2
	andn	%g1, 0x7, %g1
	mov	512, %g7
	sllx	%g7, %g2, %g7
	sub	%g7, 1, %g7
	srlx	%g4, PAGE_SHIFT, %g2
	and	%g2, %g7, %g2
	sllx	%g2, 4, %g2
	ba,pt	%xcc, tsb_miss_page_table_walk
	 add	%g1, %g2, %g1

	/* Instruction Access Exception, tl0. */
sun4v_iacc:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	sun4v_insn_access_exception
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Instruction Access Exception, tl1. */
sun4v_iacc_tl1:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_I_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etraptl1
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	sun4v_insn_access_exception_tl1
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Data Access Exception, tl0. */
sun4v_dacc:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	sun4v_data_access_exception
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Data Access Exception, tl1. */
sun4v_dacc_tl1:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etraptl1
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	sun4v_data_access_exception_tl1
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Memory Address Unaligned.  */
sun4v_mna:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	mov	HV_FAULT_TYPE_UNALIGNED, %g3
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5

	/* Window fixup? */
	rdpr	%tl, %g2
	cmp	%g2, 1
	bgu,pn	%icc, winfix_mna
	 rdpr	%tpc, %g3

	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	sun4v_mna
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Privileged Action.  */
sun4v_privact:
	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	call	do_privact
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Unaligned ldd float, tl0. */
sun4v_lddfmna:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	handle_lddfmna
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

	/* Unaligned std float, tl0. */
sun4v_stdfmna:
	ldxa	[%g0] ASI_SCRATCHPAD, %g2
	ldx	[%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	ldx	[%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	ldx	[%g2 + HV_FAULT_D_CTX_OFFSET], %g5
	sllx	%g3, 16, %g3
	or	%g5, %g3, %g5
	ba,pt	%xcc, etrap
	 rd	%pc, %g7
	mov	%l4, %o1
	mov	%l5, %o2
	call	handle_stdfmna
	 add	%sp, PTREGS_OFF, %o0
	ba,a,pt	%xcc, rtrap_clr_l6

#define BRANCH_ALWAYS	0x10680000
#define NOP		0x01000000
#define SUN4V_DO_PATCH(OLD, NEW)	\
	sethi	%hi(NEW), %g1; \
	or	%g1, %lo(NEW), %g1; \
	sethi	%hi(OLD), %g2; \
	or	%g2, %lo(OLD), %g2; \
	sub	%g1, %g2, %g1; \
	sethi	%hi(BRANCH_ALWAYS), %g3; \
	srl	%g1, 2, %g1; \
	or	%g3, %lo(BRANCH_ALWAYS), %g3; \
	or	%g3, %g1, %g3; \
	stw	%g3, [%g2]; \
	sethi	%hi(NOP), %g3; \
	or	%g3, %lo(NOP), %g3; \
	stw	%g3, [%g2 + 0x4]; \
	flush	%g2;

	.globl	sun4v_patch_tlb_handlers
	.type	sun4v_patch_tlb_handlers,#function
sun4v_patch_tlb_handlers:
	SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
	SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
	SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
	retl
	 nop
	.size	sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers
Commit	Line	Data
d257d5da DM	1	/* sun4v_tlb_miss.S: Sun4v TLB miss handlers.
	2	*
	3	* Copyright (C) 2006 <davem@davemloft.net>
	4	*/
	5
	6	.text
	7	.align 32
	8
	9	sun4v_itlb_miss:
12eaa328 DM	10	/* Load MMU Miss base into %g2. */
12eaa328 DM	11	ldxa [%g0] ASI_SCRATCHPAD, %g3
d257d5da DM	12
d257d5da DM	13	/* Load UTSB reg into %g1. */
12eaa328 DM	14	mov SCRATCHPAD_UTSBREG1, %g1
12eaa328 DM	15	ldxa [%g1] ASI_SCRATCHPAD, %g1
d257d5da DM	16
	17	/* Create a TAG TARGET, "(vaddr>>22) \| (ctx << 48)", in %g6.
	18	* Branch if kernel TLB miss. The kernel TSB and user TSB miss
	19	* code wants the missing virtual address in %g4, so that value
	20	* cannot be modified through the entirety of this handler.
	21	*/
12eaa328 DM	22	ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
12eaa328 DM	23	ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
d257d5da DM	24	srlx %g4, 22, %g3
	25	sllx %g5, 48, %g6
	26	or %g6, %g3, %g6
	27	brz,pn %g5, kvmap_itlb_4v
	28	nop
	29
	30	/* Create TSB pointer. This is something like:
	31	*
	32	* index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	33	* tsb_base = tsb_reg & ~0x7UL;
	34	*/
	35	and %g1, 0x7, %g3
	36	andn %g1, 0x7, %g1
	37	mov 512, %g7
	38	sllx %g7, %g3, %g7
	39	sub %g7, 1, %g7
	40
	41	/* TSB index mask is in %g7, tsb base is in %g1. Compute
	42	* the TSB entry pointer into %g1:
	43	*
	44	* tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	45	* tsb_ptr = tsb_base + (tsb_index * 16);
	46	*/
	47	srlx %g4, PAGE_SHIFT, %g3
	48	and %g3, %g7, %g3
	49	sllx %g3, 4, %g3
	50	add %g1, %g3, %g1
	51
	52	/* Load TSB tag/pte into %g2/%g3 and compare the tag. */
	53	ldda [%g1] ASI_QUAD_LDD_PHYS, %g2
	54	cmp %g2, %g6
	55	sethi %hi(_PAGE_EXEC), %g7
	56	bne,a,pn %xcc, tsb_miss_page_table_walk
	57	mov FAULT_CODE_ITLB, %g3
	58	andcc %g3, %g7, %g0
	59	be,a,pn %xcc, tsb_do_fault
	60	mov FAULT_CODE_ITLB, %g3
	61
	62	/* We have a valid entry, make hypervisor call to load
	63	* I-TLB and return from trap.
	64	*
	65	* %g3: PTE
	66	* %g4: vaddr
	67	* %g6: TAG TARGET (only "CTX << 48" part matters)
	68	*/
	69	sun4v_itlb_load:
	70	mov %o0, %g1 ! save %o0
	71	mov %o1, %g2 ! save %o1
	72	mov %o2, %g5 ! save %o2
	73	mov %o3, %g7 ! save %o3
	74	mov %g4, %o0 ! vaddr
	75	srlx %g6, 48, %o1 ! ctx
	76	mov %g3, %o2 ! PTE
	77	mov HV_MMU_IMMU, %o3 ! flags
	78	ta HV_MMU_MAP_ADDR_TRAP
	79	mov %g1, %o0 ! restore %o0
	80	mov %g2, %o1 ! restore %o1
	81	mov %g5, %o2 ! restore %o2
	82	mov %g7, %o3 ! restore %o3
	83
	84	retry
	85
	86	sun4v_dtlb_miss:
12eaa328 DM	87	/* Load MMU Miss base into %g2. */
12eaa328 DM	88	ldxa [%g0] ASI_SCRATCHPAD, %g2
d257d5da DM	89
d257d5da DM	90	/* Load UTSB reg into %g1. */
12eaa328	91	mov SCRATCHPAD_UTSBREG1, %g1
d257d5da DM	92	ldxa [%g1 + %g1] ASI_SCRATCHPAD, %g1
d257d5da DM	93
d257d5da DM	94	/* Create a TAG TARGET, "(vaddr>>22) \| (ctx << 48)", in %g6.
	95	* Branch if kernel TLB miss. The kernel TSB and user TSB miss
	96	* code wants the missing virtual address in %g4, so that value
	97	* cannot be modified through the entirety of this handler.
	98	*/
12eaa328 DM	99	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
12eaa328 DM	100	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
d257d5da DM	101	srlx %g4, 22, %g3
	102	sllx %g5, 48, %g6
	103	or %g6, %g3, %g6
	104	brz,pn %g5, kvmap_dtlb_4v
	105	nop
	106
	107	/* Create TSB pointer. This is something like:
	108	*
	109	* index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	110	* tsb_base = tsb_reg & ~0x7UL;
	111	*/
	112	and %g1, 0x7, %g3
	113	andn %g1, 0x7, %g1
	114	mov 512, %g7
	115	sllx %g7, %g3, %g7
	116	sub %g7, 1, %g7
	117
	118	/* TSB index mask is in %g7, tsb base is in %g1. Compute
	119	* the TSB entry pointer into %g1:
	120	*
	121	* tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	122	* tsb_ptr = tsb_base + (tsb_index * 16);
	123	*/
	124	srlx %g4, PAGE_SHIFT, %g3
	125	and %g3, %g7, %g3
	126	sllx %g3, 4, %g3
	127	add %g1, %g3, %g1
	128
	129	/* Load TSB tag/pte into %g2/%g3 and compare the tag. */
	130	ldda [%g1] ASI_QUAD_LDD_PHYS, %g2
	131	cmp %g2, %g6
	132	bne,a,pn %xcc, tsb_miss_page_table_walk
	133	mov FAULT_CODE_ITLB, %g3
	134
	135	/* We have a valid entry, make hypervisor call to load
	136	* D-TLB and return from trap.
	137	*
	138	* %g3: PTE
	139	* %g4: vaddr
	140	* %g6: TAG TARGET (only "CTX << 48" part matters)
	141	*/
	142	sun4v_dtlb_load:
	143	mov %o0, %g1 ! save %o0
	144	mov %o1, %g2 ! save %o1
	145	mov %o2, %g5 ! save %o2
	146	mov %o3, %g7 ! save %o3
	147	mov %g4, %o0 ! vaddr
	148	srlx %g6, 48, %o1 ! ctx
	149	mov %g3, %o2 ! PTE
	150	mov HV_MMU_DMMU, %o3 ! flags
	151	ta HV_MMU_MAP_ADDR_TRAP
	152	mov %g1, %o0 ! restore %o0
	153	mov %g2, %o1 ! restore %o1
	154	mov %g5, %o2 ! restore %o2
	155	mov %g7, %o3 ! restore %o3
	156
	157	retry
	158
	159	sun4v_dtlb_prot:
12eaa328 DM	160	/* Load MMU Miss base into %g2. */
12eaa328 DM	161	ldxa [%g0] ASI_SCRATCHPAD, %g2
d257d5da	162
12eaa328	163	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5
d257d5da DM	164	rdpr %tl, %g1
	165	cmp %g1, 1
	166	bgu,pn %xcc, winfix_trampoline
	167	nop
	168	ba,pt %xcc, sparc64_realfault_common
	169	mov FAULT_CODE_DTLB \| FAULT_CODE_WRITE, %g4
	170
12eaa328 DM	171	/* Called from trap table with TAG TARGET placed into
12eaa328 DM	172	* %g6 and SCRATCHPAD_UTSBREG1 contents in %g1.
aa9143b9 DM	173	*/
aa9143b9 DM	174	sun4v_itsb_miss:
aa9143b9 DM	175	ba,pt %xcc, sun4v_tsb_miss_common
	176	mov FAULT_CODE_ITLB, %g3
	177
12eaa328 DM	178	/* Called from trap table with TAG TARGET placed into
12eaa328 DM	179	* %g6 and SCRATCHPAD_UTSBREG1 contents in %g1.
aa9143b9 DM	180	*/
aa9143b9 DM	181	sun4v_dtsb_miss:
aa9143b9 DM	182	mov FAULT_CODE_DTLB, %g3
	183
	184	/* Create TSB pointer into %g1. This is something like:
	185	*
	186	* index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL;
	187	* tsb_base = tsb_reg & ~0x7UL;
	188	* tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask);
	189	* tsb_ptr = tsb_base + (tsb_index * 16);
	190	*/
	191	sun4v_tsb_miss_common:
	192	and %g1, 0x7, %g2
	193	andn %g1, 0x7, %g1
	194	mov 512, %g7
	195	sllx %g7, %g2, %g7
	196	sub %g7, 1, %g7
	197	srlx %g4, PAGE_SHIFT, %g2
	198	and %g2, %g7, %g2
	199	sllx %g2, 4, %g2
	200	ba,pt %xcc, tsb_miss_page_table_walk
	201	add %g1, %g2, %g1
	202
ed6b0b45 DM	203	/* Instruction Access Exception, tl0. */
ed6b0b45 DM	204	sun4v_iacc:
12eaa328 DM	205	ldxa [%g0] ASI_SCRATCHPAD, %g2
	206	ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
	207	ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
	208	ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
ed6b0b45 DM	209	sllx %g3, 16, %g3
	210	or %g5, %g3, %g5
	211	ba,pt %xcc, etrap
	212	rd %pc, %g7
	213	mov %l4, %o1
	214	mov %l5, %o2
	215	call sun4v_insn_access_exception
	216	add %sp, PTREGS_OFF, %o0
	217	ba,a,pt %xcc, rtrap_clr_l6
	218
	219	/* Instruction Access Exception, tl1. */
	220	sun4v_iacc_tl1:
12eaa328 DM	221	ldxa [%g0] ASI_SCRATCHPAD, %g2
	222	ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3
	223	ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4
	224	ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5
ed6b0b45 DM	225	sllx %g3, 16, %g3
	226	or %g5, %g3, %g5
	227	ba,pt %xcc, etraptl1
	228	rd %pc, %g7
	229	mov %l4, %o1
	230	mov %l5, %o2
	231	call sun4v_insn_access_exception_tl1
	232	add %sp, PTREGS_OFF, %o0
	233	ba,a,pt %xcc, rtrap_clr_l6
	234
	235	/* Data Access Exception, tl0. */
	236	sun4v_dacc:
12eaa328 DM	237	ldxa [%g0] ASI_SCRATCHPAD, %g2
	238	ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	239	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	240	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
ed6b0b45 DM	241	sllx %g3, 16, %g3
	242	or %g5, %g3, %g5
	243	ba,pt %xcc, etrap
	244	rd %pc, %g7
	245	mov %l4, %o1
	246	mov %l5, %o2
	247	call sun4v_data_access_exception
	248	add %sp, PTREGS_OFF, %o0
	249	ba,a,pt %xcc, rtrap_clr_l6
	250
	251	/* Data Access Exception, tl1. */
	252	sun4v_dacc_tl1:
12eaa328 DM	253	ldxa [%g0] ASI_SCRATCHPAD, %g2
	254	ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	255	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	256	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
ed6b0b45 DM	257	sllx %g3, 16, %g3
	258	or %g5, %g3, %g5
	259	ba,pt %xcc, etraptl1
	260	rd %pc, %g7
	261	mov %l4, %o1
	262	mov %l5, %o2
	263	call sun4v_data_access_exception_tl1
	264	add %sp, PTREGS_OFF, %o0
	265	ba,a,pt %xcc, rtrap_clr_l6
	266
	267	/* Memory Address Unaligned. */
	268	sun4v_mna:
12eaa328	269	ldxa [%g0] ASI_SCRATCHPAD, %g2
ed6b0b45	270	mov HV_FAULT_TYPE_UNALIGNED, %g3
12eaa328 DM	271	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
12eaa328 DM	272	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
ed6b0b45 DM	273	sllx %g3, 16, %g3
	274	or %g5, %g3, %g5
	275
	276	/* Window fixup? */
	277	rdpr %tl, %g2
	278	cmp %g2, 1
	279	bgu,pn %icc, winfix_mna
	280	rdpr %tpc, %g3
	281
	282	ba,pt %xcc, etrap
	283	rd %pc, %g7
	284	mov %l4, %o1
	285	mov %l5, %o2
	286	call sun4v_mna
	287	add %sp, PTREGS_OFF, %o0
	288	ba,a,pt %xcc, rtrap_clr_l6
	289
	290	/* Privileged Action. */
	291	sun4v_privact:
	292	ba,pt %xcc, etrap
	293	rd %pc, %g7
	294	call do_privact
	295	add %sp, PTREGS_OFF, %o0
	296	ba,a,pt %xcc, rtrap_clr_l6
	297
	298	/* Unaligned ldd float, tl0. */
	299	sun4v_lddfmna:
12eaa328 DM	300	ldxa [%g0] ASI_SCRATCHPAD, %g2
	301	ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	302	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	303	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
ed6b0b45 DM	304	sllx %g3, 16, %g3
	305	or %g5, %g3, %g5
	306	ba,pt %xcc, etrap
	307	rd %pc, %g7
	308	mov %l4, %o1
	309	mov %l5, %o2
	310	call handle_lddfmna
	311	add %sp, PTREGS_OFF, %o0
	312	ba,a,pt %xcc, rtrap_clr_l6
	313
	314	/* Unaligned std float, tl0. */
	315	sun4v_stdfmna:
12eaa328 DM	316	ldxa [%g0] ASI_SCRATCHPAD, %g2
	317	ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3
	318	ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4
	319	ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5
ed6b0b45 DM	320	sllx %g3, 16, %g3
	321	or %g5, %g3, %g5
	322	ba,pt %xcc, etrap
	323	rd %pc, %g7
	324	mov %l4, %o1
	325	mov %l5, %o2
	326	call handle_stdfmna
	327	add %sp, PTREGS_OFF, %o0
	328	ba,a,pt %xcc, rtrap_clr_l6
aa9143b9	329
d257d5da DM	330	#define BRANCH_ALWAYS 0x10680000
	331	#define NOP 0x01000000
	332	#define SUN4V_DO_PATCH(OLD, NEW) \
	333	sethi %hi(NEW), %g1; \
	334	or %g1, %lo(NEW), %g1; \
	335	sethi %hi(OLD), %g2; \
	336	or %g2, %lo(OLD), %g2; \
	337	sub %g1, %g2, %g1; \
	338	sethi %hi(BRANCH_ALWAYS), %g3; \
	339	srl %g1, 2, %g1; \
	340	or %g3, %lo(BRANCH_ALWAYS), %g3; \
	341	or %g3, %g1, %g3; \
	342	stw %g3, [%g2]; \
	343	sethi %hi(NOP), %g3; \
	344	or %g3, %lo(NOP), %g3; \
	345	stw %g3, [%g2 + 0x4]; \
	346	flush %g2;
	347
	348	.globl sun4v_patch_tlb_handlers
	349	.type sun4v_patch_tlb_handlers,#function
	350	sun4v_patch_tlb_handlers:
	351	SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss)
	352	SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss)
	353	SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss)
	354	SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss)
	355	SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot)
	356	SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot)
ed6b0b45 DM	357	SUN4V_DO_PATCH(tl0_iax, sun4v_iacc)
	358	SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1)
	359	SUN4V_DO_PATCH(tl0_dax, sun4v_dacc)
	360	SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1)
	361	SUN4V_DO_PATCH(tl0_mna, sun4v_mna)
	362	SUN4V_DO_PATCH(tl1_mna, sun4v_mna)
	363	SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna)
	364	SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna)
	365	SUN4V_DO_PATCH(tl0_privact, sun4v_privact)
d257d5da DM	366	retl
	367	nop
	368	.size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers