[deliverable/linux.git] / arch / sparc64 / kernel / ktlb.S

/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <linux/config.h>
#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/tsb.h>

	.text
	.align		32

kvmap_itlb:
	/* g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_IMMU, %g4

	/* sun4v_itlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_itlb_4v:

kvmap_itlb_nonlinear:
	/* Catch kernel NULL pointer calls.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_obp
	 nop

kvmap_itlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	mov		1, %g7
	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
	brgez,a,pn	%g5, kvmap_itlb_longpath
	 KTSB_STORE(%g1, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_itlb_load:

661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_itlb_load
	 mov		%g5, %g3

kvmap_itlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	nop
	.previous

	rdpr	%tpc, %g5
	ba,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_itlb_load
	 nop

kvmap_dtlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_dtlb_load
	 nop

	.align		32
kvmap_dtlb:
	/* %g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_DMMU, %g4

	/* sun4v_dtlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_dtlb_4v:
	brgez,pn	%g4, kvmap_dtlb_nonlinear
	 nop

	sethi		%hi(kern_linear_pte_xor), %g2
	ldx		[%g2 + %lo(kern_linear_pte_xor)], %g2

	.globl		kvmap_linear_patch
kvmap_linear_patch:
	ba,pt		%xcc, kvmap_dtlb_load
	 xor		%g2, %g4, %g5

kvmap_dtlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

	KTSB_LOCK_TAG(%g1, %g2, %g7)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	mov		1, %g7
	sllx		%g7, TSB_TAG_INVALID_BIT, %g7
	brgez,a,pn	%g5, kvmap_dtlb_longpath
	 KTSB_STORE(%g1, %g7)

	KTSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_dtlb_load:

661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_dtlb_load
	 mov		%g5, %g3

kvmap_dtlb_nonlinear:
	/* Catch kernel NULL pointer derefs.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
	sethi		%hi(MODULES_VADDR), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_longpath
	 mov		(VMALLOC_END >> 24), %g5
	sllx		%g5, 24, %g5
	cmp		%g4, %g5
	bgeu,pn		%xcc, kvmap_dtlb_longpath
	 nop

kvmap_check_obp:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_obp
	 nop
	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
	 nop

kvmap_dtlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	ldxa		[%g0] ASI_SCRATCHPAD, %g5
	.previous

	rdpr	%tl, %g3
	cmp	%g3, 1

661:	mov	TLB_TAG_ACCESS, %g4
	ldxa	[%g4] ASI_DMMU, %g5
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
	nop
	.previous

	be,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_DTLB, %g4
	ba,pt	%xcc, winfix_trampoline
	 nop
Commit	Line	Data
2a7e2990 DM	1	/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
	2	*
	3	* Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
	4	* Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
	5	* Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
	6	* Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
74bf4312	7	*/
2a7e2990 DM	8
	9	#include <linux/config.h>
	10	#include <asm/head.h>
	11	#include <asm/asi.h>
	12	#include <asm/page.h>
	13	#include <asm/pgtable.h>
74bf4312	14	#include <asm/tsb.h>
2a7e2990 DM	15
	16	.text
	17	.align 32
	18
74bf4312 DM	19	kvmap_itlb:
	20	/* g6: TAG TARGET */
	21	mov TLB_TAG_ACCESS, %g4
	22	ldxa [%g4] ASI_IMMU, %g4
	23
d257d5da DM	24	/* sun4v_itlb_miss branches here with the missing virtual
	25	* address already loaded into %g4
	26	*/
	27	kvmap_itlb_4v:
	28
74bf4312 DM	29	kvmap_itlb_nonlinear:
	30	/* Catch kernel NULL pointer calls. */
	31	sethi %hi(PAGE_SIZE), %g5
	32	cmp %g4, %g5
	33	bleu,pn %xcc, kvmap_dtlb_longpath
	34	nop
	35
	36	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
	37
	38	kvmap_itlb_tsb_miss:
2a7e2990 DM	39	sethi %hi(LOW_OBP_ADDRESS), %g5
2a7e2990 DM	40	cmp %g4, %g5
74bf4312	41	blu,pn %xcc, kvmap_itlb_vmalloc_addr
2a7e2990 DM	42	mov 0x1, %g5
	43	sllx %g5, 32, %g5
	44	cmp %g4, %g5
74bf4312	45	blu,pn %xcc, kvmap_itlb_obp
2a7e2990 DM	46	nop
2a7e2990 DM	47
74bf4312 DM	48	kvmap_itlb_vmalloc_addr:
	49	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
	50
459b6e62	51	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312 DM	52
	53	/* Load and check PTE. */
	54	ldxa [%g5] ASI_PHYS_USE_EC, %g5
8b234274 DM	55	mov 1, %g7
8b234274 DM	56	sllx %g7, TSB_TAG_INVALID_BIT, %g7
74bf4312	57	brgez,a,pn %g5, kvmap_itlb_longpath
8b234274	58	KTSB_STORE(%g1, %g7)
2a7e2990	59
517af332	60	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	61
	62	/* fallthrough to TLB load */
	63
	64	kvmap_itlb_load:
459b6e62 DM	65
459b6e62 DM	66	661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
2a7e2990	67	retry
459b6e62 DM	68	.section .sun4v_2insn_patch, "ax"
	69	.word 661b
	70	nop
	71	nop
	72	.previous
	73
	74	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	75	* instruction get nop'd out and we get here to branch
	76	* to the sun4v tlb load code. The registers are setup
	77	* as follows:
	78	*
	79	* %g4: vaddr
	80	* %g5: PTE
	81	* %g6: TAG
	82	*
	83	* The sun4v TLB load wants the PTE in %g3 so we fix that
	84	* up here.
	85	*/
	86	ba,pt %xcc, sun4v_itlb_load
	87	mov %g5, %g3
2a7e2990	88
74bf4312	89	kvmap_itlb_longpath:
45fec05f DM	90
45fec05f DM	91	661: rdpr %pstate, %g5
74bf4312	92	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	93	.section .sun4v_2insn_patch, "ax"
45fec05f	94	.word 661b
6cc200db	95	SET_GL(1)
45fec05f DM	96	nop
	97	.previous
	98
74bf4312 DM	99	rdpr %tpc, %g5
	100	ba,pt %xcc, sparc64_realfault_common
	101	mov FAULT_CODE_ITLB, %g4
	102
	103	kvmap_itlb_obp:
	104	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
	105
459b6e62	106	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	107
517af332	108	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	109
	110	ba,pt %xcc, kvmap_itlb_load
	111	nop
	112
	113	kvmap_dtlb_obp:
	114	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
	115
459b6e62	116	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	117
517af332	118	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	119
	120	ba,pt %xcc, kvmap_dtlb_load
	121	nop
c9c10830	122
2a7e2990	123	.align 32
74bf4312 DM	124	kvmap_dtlb:
	125	/* %g6: TAG TARGET */
	126	mov TLB_TAG_ACCESS, %g4
	127	ldxa [%g4] ASI_DMMU, %g4
d257d5da DM	128
	129	/* sun4v_dtlb_miss branches here with the missing virtual
	130	* address already loaded into %g4
	131	*/
	132	kvmap_dtlb_4v:
74bf4312	133	brgez,pn %g4, kvmap_dtlb_nonlinear
56425306 DM	134	nop
56425306 DM	135
c4bce90e DM	136	sethi %hi(kern_linear_pte_xor), %g2
c4bce90e DM	137	ldx [%g2 + %lo(kern_linear_pte_xor)], %g2
74bf4312	138
56425306 DM	139	.globl kvmap_linear_patch
56425306 DM	140	kvmap_linear_patch:
74bf4312	141	ba,pt %xcc, kvmap_dtlb_load
2a7e2990 DM	142	xor %g2, %g4, %g5
2a7e2990 DM	143
74bf4312 DM	144	kvmap_dtlb_vmalloc_addr:
	145	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	146
459b6e62	147	KTSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312 DM	148
	149	/* Load and check PTE. */
	150	ldxa [%g5] ASI_PHYS_USE_EC, %g5
8b234274 DM	151	mov 1, %g7
8b234274 DM	152	sllx %g7, TSB_TAG_INVALID_BIT, %g7
74bf4312	153	brgez,a,pn %g5, kvmap_dtlb_longpath
8b234274	154	KTSB_STORE(%g1, %g7)
74bf4312	155
517af332	156	KTSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	157
	158	/* fallthrough to TLB load */
	159
	160	kvmap_dtlb_load:
459b6e62 DM	161
459b6e62 DM	162	661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
74bf4312	163	retry
459b6e62 DM	164	.section .sun4v_2insn_patch, "ax"
	165	.word 661b
	166	nop
	167	nop
	168	.previous
	169
	170	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	171	* instruction get nop'd out and we get here to branch
	172	* to the sun4v tlb load code. The registers are setup
	173	* as follows:
	174	*
	175	* %g4: vaddr
	176	* %g5: PTE
	177	* %g6: TAG
	178	*
	179	* The sun4v TLB load wants the PTE in %g3 so we fix that
	180	* up here.
	181	*/
	182	ba,pt %xcc, sun4v_dtlb_load
	183	mov %g5, %g3
74bf4312 DM	184
	185	kvmap_dtlb_nonlinear:
	186	/* Catch kernel NULL pointer derefs. */
	187	sethi %hi(PAGE_SIZE), %g5
	188	cmp %g4, %g5
	189	bleu,pn %xcc, kvmap_dtlb_longpath
56425306	190	nop
56425306	191
74bf4312 DM	192	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	193
	194	kvmap_dtlb_tsbmiss:
2a7e2990 DM	195	sethi %hi(MODULES_VADDR), %g5
2a7e2990 DM	196	cmp %g4, %g5
74bf4312	197	blu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	198	mov (VMALLOC_END >> 24), %g5
	199	sllx %g5, 24, %g5
	200	cmp %g4, %g5
74bf4312	201	bgeu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	202	nop
	203
	204	kvmap_check_obp:
	205	sethi %hi(LOW_OBP_ADDRESS), %g5
	206	cmp %g4, %g5
74bf4312	207	blu,pn %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	208	mov 0x1, %g5
	209	sllx %g5, 32, %g5
	210	cmp %g4, %g5
74bf4312	211	blu,pn %xcc, kvmap_dtlb_obp
2a7e2990	212	nop
74bf4312	213	ba,pt %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	214	nop
2a7e2990 DM	215
74bf4312	216	kvmap_dtlb_longpath:
45fec05f DM	217
45fec05f DM	218	661: rdpr %pstate, %g5
74bf4312	219	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	220	.section .sun4v_2insn_patch, "ax"
45fec05f	221	.word 661b
8b234274 DM	222	SET_GL(1)
8b234274 DM	223	ldxa [%g0] ASI_SCRATCHPAD, %g5
45fec05f DM	224	.previous
45fec05f DM	225
459b6e62 DM	226	rdpr %tl, %g3
	227	cmp %g3, 1
	228
	229	661: mov TLB_TAG_ACCESS, %g4
74bf4312	230	ldxa [%g4] ASI_DMMU, %g5
459b6e62 DM	231	.section .sun4v_2insn_patch, "ax"
459b6e62 DM	232	.word 661b
8b234274	233	ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
459b6e62 DM	234	nop
	235	.previous
	236
74bf4312 DM	237	be,pt %xcc, sparc64_realfault_common
	238	mov FAULT_CODE_DTLB, %g4
	239	ba,pt %xcc, winfix_trampoline
	240	nop