[deliverable/linux.git] / arch / sparc / kernel / ktlb.S

/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/tsb.h>

	.text
	.align		32

kvmap_itlb:
	/* g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_IMMU, %g4

	/* sun4v_itlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_itlb_4v:

	/* Catch kernel NULL pointer calls.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_obp
	 nop

kvmap_itlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_itlb_load:

661:	stxa		%g5, [%g0] ASI_ITLB_DATA_IN
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_itlb_load
	 mov		%g5, %g3

kvmap_itlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	nop
	.previous

	rdpr	%tpc, %g5
	ba,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_itlb_load
	 nop

kvmap_dtlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_dtlb_load
	 nop

kvmap_linear_early:
	sethi		%hi(kern_linear_pte_xor), %g7
	ldx		[%g7 + %lo(kern_linear_pte_xor)], %g2
	ba,pt		%xcc, kvmap_dtlb_tsb4m_load
	 xor		%g2, %g4, %g5

	.align		32
kvmap_dtlb_tsb4m_load:
	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)
	ba,pt		%xcc, kvmap_dtlb_load
	 nop

kvmap_dtlb:
	/* %g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_DMMU, %g4

	/* sun4v_dtlb_miss branches here with the missing virtual
	 * address already loaded into %g4
	 */
kvmap_dtlb_4v:
	brgez,pn	%g4, kvmap_dtlb_nonlinear
	 nop

#ifdef CONFIG_DEBUG_PAGEALLOC
	/* Index through the base page size TSB even for linear
	 * mappings when using page allocation debugging.
	 */
	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#else
	/* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#endif
	/* Linear mapping TSB lookup failed.  Fallthrough to kernel
	 * page table based lookup.
	 */
	.globl		kvmap_linear_patch
kvmap_linear_patch:
	ba,a,pt		%xcc, kvmap_linear_early

kvmap_dtlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g7)
	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_dtlb_load:

661:	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
	retry
	.section	.sun4v_2insn_patch, "ax"
	.word		661b
	nop
	nop
	.previous

	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	 * instruction get nop'd out and we get here to branch
	 * to the sun4v tlb load code.  The registers are setup
	 * as follows:
	 *
	 * %g4: vaddr
	 * %g5: PTE
	 * %g6:	TAG
	 *
	 * The sun4v TLB load wants the PTE in %g3 so we fix that
	 * up here.
	 */
	ba,pt		%xcc, sun4v_dtlb_load
	 mov		%g5, %g3

#ifdef CONFIG_SPARSEMEM_VMEMMAP
kvmap_vmemmap:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	ba,a,pt		%xcc, kvmap_dtlb_load
#endif

kvmap_dtlb_nonlinear:
	/* Catch kernel NULL pointer derefs.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

#ifdef CONFIG_SPARSEMEM_VMEMMAP
	/* Do not use the TSB for vmemmap.  */
	sethi		%hi(VMEMMAP_BASE), %g5
	ldx		[%g5 + %lo(VMEMMAP_BASE)], %g5
	cmp		%g4,%g5
	bgeu,pn		%xcc, kvmap_vmemmap
	 nop
#endif

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
	sethi		%hi(MODULES_VADDR), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_longpath
	 sethi		%hi(VMALLOC_END), %g5
	ldx		[%g5 + %lo(VMALLOC_END)], %g5
	cmp		%g4, %g5
	bgeu,pn		%xcc, kvmap_dtlb_longpath
	 nop

kvmap_check_obp:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_obp
	 nop
	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
	 nop

kvmap_dtlb_longpath:

661:	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	SET_GL(1)
	ldxa		[%g0] ASI_SCRATCHPAD, %g5
	.previous

	rdpr	%tl, %g3
	cmp	%g3, 1

661:	mov	TLB_TAG_ACCESS, %g4
	ldxa	[%g4] ASI_DMMU, %g5
	.section .sun4v_2insn_patch, "ax"
	.word	661b
	ldx	[%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
	nop
	.previous

	be,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_DTLB, %g4
	ba,pt	%xcc, winfix_trampoline
	 nop
Commit	Line	Data
2a7e2990 DM	1	/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
2a7e2990 DM	2	*
bf4a7972	3	* Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
2a7e2990 DM	4	* Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
	5	* Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
	6	* Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
74bf4312	7	*/
2a7e2990	8
2a7e2990 DM	9	#include <asm/head.h>
	10	#include <asm/asi.h>
	11	#include <asm/page.h>
	12	#include <asm/pgtable.h>
74bf4312	13	#include <asm/tsb.h>
2a7e2990 DM	14
	15	.text
	16	.align 32
	17
74bf4312 DM	18	kvmap_itlb:
	19	/* g6: TAG TARGET */
	20	mov TLB_TAG_ACCESS, %g4
	21	ldxa [%g4] ASI_IMMU, %g4
	22
d257d5da DM	23	/* sun4v_itlb_miss branches here with the missing virtual
	24	* address already loaded into %g4
	25	*/
	26	kvmap_itlb_4v:
	27
74bf4312 DM	28	/* Catch kernel NULL pointer calls. */
	29	sethi %hi(PAGE_SIZE), %g5
	30	cmp %g4, %g5
1c2696cd	31	blu,pn %xcc, kvmap_itlb_longpath
74bf4312 DM	32	nop
	33
	34	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
	35
	36	kvmap_itlb_tsb_miss:
2a7e2990 DM	37	sethi %hi(LOW_OBP_ADDRESS), %g5
2a7e2990 DM	38	cmp %g4, %g5
74bf4312	39	blu,pn %xcc, kvmap_itlb_vmalloc_addr
2a7e2990 DM	40	mov 0x1, %g5
	41	sllx %g5, 32, %g5
	42	cmp %g4, %g5
74bf4312	43	blu,pn %xcc, kvmap_itlb_obp
2a7e2990 DM	44	nop
2a7e2990 DM	45
74bf4312 DM	46	kvmap_itlb_vmalloc_addr:
	47	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
	48
9076d0e7	49	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7	50	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	51
	52	/* fallthrough to TLB load */
	53
	54	kvmap_itlb_load:
459b6e62 DM	55
459b6e62 DM	56	661: stxa %g5, [%g0] ASI_ITLB_DATA_IN
2a7e2990	57	retry
459b6e62 DM	58	.section .sun4v_2insn_patch, "ax"
	59	.word 661b
	60	nop
	61	nop
	62	.previous
	63
	64	/* For sun4v the ASI_ITLB_DATA_IN store and the retry
	65	* instruction get nop'd out and we get here to branch
	66	* to the sun4v tlb load code. The registers are setup
	67	* as follows:
	68	*
	69	* %g4: vaddr
	70	* %g5: PTE
	71	* %g6: TAG
	72	*
	73	* The sun4v TLB load wants the PTE in %g3 so we fix that
	74	* up here.
	75	*/
	76	ba,pt %xcc, sun4v_itlb_load
	77	mov %g5, %g3
2a7e2990	78
74bf4312	79	kvmap_itlb_longpath:
45fec05f DM	80
45fec05f DM	81	661: rdpr %pstate, %g5
74bf4312	82	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	83	.section .sun4v_2insn_patch, "ax"
45fec05f	84	.word 661b
6cc200db	85	SET_GL(1)
45fec05f DM	86	nop
	87	.previous
	88
74bf4312 DM	89	rdpr %tpc, %g5
	90	ba,pt %xcc, sparc64_realfault_common
	91	mov FAULT_CODE_ITLB, %g4
	92
	93	kvmap_itlb_obp:
	94	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
	95
9076d0e7	96	TSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	97
9076d0e7	98	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	99
	100	ba,pt %xcc, kvmap_itlb_load
	101	nop
	102
	103	kvmap_dtlb_obp:
	104	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
	105
9076d0e7	106	TSB_LOCK_TAG(%g1, %g2, %g7)
74bf4312	107
9076d0e7	108	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	109
	110	ba,pt %xcc, kvmap_dtlb_load
	111	nop
c9c10830	112
0dd5b7b0 DM	113	kvmap_linear_early:
	114	sethi %hi(kern_linear_pte_xor), %g7
	115	ldx [%g7 + %lo(kern_linear_pte_xor)], %g2
	116	ba,pt %xcc, kvmap_dtlb_tsb4m_load
	117	xor %g2, %g4, %g5
	118
2a7e2990	119	.align 32
d7744a09	120	kvmap_dtlb_tsb4m_load:
9076d0e7 DM	121	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7 DM	122	TSB_WRITE(%g1, %g5, %g6)
d7744a09 DM	123	ba,pt %xcc, kvmap_dtlb_load
	124	nop
	125
74bf4312 DM	126	kvmap_dtlb:
	127	/* %g6: TAG TARGET */
	128	mov TLB_TAG_ACCESS, %g4
	129	ldxa [%g4] ASI_DMMU, %g4
d257d5da DM	130
	131	/* sun4v_dtlb_miss branches here with the missing virtual
	132	* address already loaded into %g4
	133	*/
	134	kvmap_dtlb_4v:
74bf4312	135	brgez,pn %g4, kvmap_dtlb_nonlinear
56425306 DM	136	nop
56425306 DM	137
d1acb421 DM	138	#ifdef CONFIG_DEBUG_PAGEALLOC
	139	/* Index through the base page size TSB even for linear
	140	* mappings when using page allocation debugging.
	141	*/
	142	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	143	#else
d7744a09 DM	144	/* Correct TAG_TARGET is already in %g6, check 4mb TSB. */
d7744a09 DM	145	KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
d1acb421	146	#endif
0dd5b7b0 DM	147	/* Linear mapping TSB lookup failed. Fallthrough to kernel
0dd5b7b0 DM	148	* page table based lookup.
9cc3a1ac	149	*/
56425306 DM	150	.globl kvmap_linear_patch
56425306 DM	151	kvmap_linear_patch:
0dd5b7b0	152	ba,a,pt %xcc, kvmap_linear_early
2a7e2990	153
74bf4312 DM	154	kvmap_dtlb_vmalloc_addr:
	155	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	156
9076d0e7	157	TSB_LOCK_TAG(%g1, %g2, %g7)
9076d0e7	158	TSB_WRITE(%g1, %g5, %g6)
74bf4312 DM	159
	160	/* fallthrough to TLB load */
	161
	162	kvmap_dtlb_load:
459b6e62 DM	163
459b6e62 DM	164	661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
74bf4312	165	retry
459b6e62 DM	166	.section .sun4v_2insn_patch, "ax"
	167	.word 661b
	168	nop
	169	nop
	170	.previous
	171
	172	/* For sun4v the ASI_DTLB_DATA_IN store and the retry
	173	* instruction get nop'd out and we get here to branch
	174	* to the sun4v tlb load code. The registers are setup
	175	* as follows:
	176	*
	177	* %g4: vaddr
	178	* %g5: PTE
	179	* %g6: TAG
	180	*
	181	* The sun4v TLB load wants the PTE in %g3 so we fix that
	182	* up here.
	183	*/
	184	ba,pt %xcc, sun4v_dtlb_load
	185	mov %g5, %g3
74bf4312	186
bf4a7972	187	#ifdef CONFIG_SPARSEMEM_VMEMMAP
46644c24	188	kvmap_vmemmap:
c06240c7 DM	189	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
c06240c7 DM	190	ba,a,pt %xcc, kvmap_dtlb_load
bf4a7972	191	#endif
46644c24	192
74bf4312 DM	193	kvmap_dtlb_nonlinear:
	194	/* Catch kernel NULL pointer derefs. */
	195	sethi %hi(PAGE_SIZE), %g5
	196	cmp %g4, %g5
	197	bleu,pn %xcc, kvmap_dtlb_longpath
56425306	198	nop
56425306	199
bf4a7972	200	#ifdef CONFIG_SPARSEMEM_VMEMMAP
46644c24	201	/* Do not use the TSB for vmemmap. */
bb4e6e85 DM	202	sethi %hi(VMEMMAP_BASE), %g5
bb4e6e85 DM	203	ldx [%g5 + %lo(VMEMMAP_BASE)], %g5
46644c24 DM	204	cmp %g4,%g5
	205	bgeu,pn %xcc, kvmap_vmemmap
	206	nop
bf4a7972	207	#endif
46644c24	208
74bf4312 DM	209	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	210
	211	kvmap_dtlb_tsbmiss:
2a7e2990 DM	212	sethi %hi(MODULES_VADDR), %g5
2a7e2990 DM	213	cmp %g4, %g5
74bf4312	214	blu,pn %xcc, kvmap_dtlb_longpath
bb4e6e85 DM	215	sethi %hi(VMALLOC_END), %g5
bb4e6e85 DM	216	ldx [%g5 + %lo(VMALLOC_END)], %g5
2a7e2990	217	cmp %g4, %g5
74bf4312	218	bgeu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	219	nop
	220
	221	kvmap_check_obp:
	222	sethi %hi(LOW_OBP_ADDRESS), %g5
	223	cmp %g4, %g5
74bf4312	224	blu,pn %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	225	mov 0x1, %g5
	226	sllx %g5, 32, %g5
	227	cmp %g4, %g5
74bf4312	228	blu,pn %xcc, kvmap_dtlb_obp
2a7e2990	229	nop
74bf4312	230	ba,pt %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	231	nop
2a7e2990 DM	232
74bf4312	233	kvmap_dtlb_longpath:
45fec05f DM	234
45fec05f DM	235	661: rdpr %pstate, %g5
74bf4312	236	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
df7d6aec	237	.section .sun4v_2insn_patch, "ax"
45fec05f	238	.word 661b
8b234274 DM	239	SET_GL(1)
8b234274 DM	240	ldxa [%g0] ASI_SCRATCHPAD, %g5
45fec05f DM	241	.previous
45fec05f DM	242
459b6e62 DM	243	rdpr %tl, %g3
	244	cmp %g3, 1
	245
	246	661: mov TLB_TAG_ACCESS, %g4
74bf4312	247	ldxa [%g4] ASI_DMMU, %g5
459b6e62 DM	248	.section .sun4v_2insn_patch, "ax"
459b6e62 DM	249	.word 661b
8b234274	250	ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
459b6e62 DM	251	nop
	252	.previous
	253
74bf4312 DM	254	be,pt %xcc, sparc64_realfault_common
	255	mov FAULT_CODE_DTLB, %g4
	256	ba,pt %xcc, winfix_trampoline
	257	nop