[deliverable/linux.git] / arch / sparc64 / kernel / ktlb.S

/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <linux/config.h>
#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/tsb.h>

	.text
	.align		32

	.globl		kvmap_itlb
kvmap_itlb:
	/* g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_IMMU, %g4

kvmap_itlb_nonlinear:
	/* Catch kernel NULL pointer calls.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_itlb_obp
	 nop

kvmap_itlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g4)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	brgez,a,pn	%g5, kvmap_itlb_longpath
	 stx		%g0, [%g1]

	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_itlb_load:
	stxa		%g5, [%g0] ASI_ITLB_DATA_IN	! Reload TLB
	retry

kvmap_itlb_longpath:
	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	rdpr	%tpc, %g5
	ba,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g4)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_itlb_load
	 nop

kvmap_dtlb_obp:
	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g4)

	TSB_WRITE(%g1, %g5, %g6)

	ba,pt		%xcc, kvmap_dtlb_load
	 nop

	.align		32
	.globl		kvmap_dtlb
kvmap_dtlb:
	/* %g6: TAG TARGET */
	mov		TLB_TAG_ACCESS, %g4
	ldxa		[%g4] ASI_DMMU, %g4
	brgez,pn	%g4, kvmap_dtlb_nonlinear
	 nop

#define KERN_HIGHBITS	((_PAGE_VALID|_PAGE_SZ4MB)^0xfffff80000000000)
#define KERN_LOWBITS	(_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_W)

	sethi		%uhi(KERN_HIGHBITS), %g2
	or		%g2, %ulo(KERN_HIGHBITS), %g2
	sllx		%g2, 32, %g2
	or		%g2, KERN_LOWBITS, %g2

#undef KERN_HIGHBITS
#undef KERN_LOWBITS

	.globl		kvmap_linear_patch
kvmap_linear_patch:
	ba,pt		%xcc, kvmap_dtlb_load
	 xor		%g2, %g4, %g5

kvmap_dtlb_vmalloc_addr:
	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

	TSB_LOCK_TAG(%g1, %g2, %g4)

	/* Load and check PTE.  */
	ldxa		[%g5] ASI_PHYS_USE_EC, %g5
	brgez,a,pn	%g5, kvmap_dtlb_longpath
	 stx		%g0, [%g1]

	TSB_WRITE(%g1, %g5, %g6)

	/* fallthrough to TLB load */

kvmap_dtlb_load:
	stxa		%g5, [%g0] ASI_DTLB_DATA_IN	! Reload TLB
	retry

kvmap_dtlb_nonlinear:
	/* Catch kernel NULL pointer derefs.  */
	sethi		%hi(PAGE_SIZE), %g5
	cmp		%g4, %g5
	bleu,pn		%xcc, kvmap_dtlb_longpath
	 nop

	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
	sethi		%hi(MODULES_VADDR), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_longpath
	 mov		(VMALLOC_END >> 24), %g5
	sllx		%g5, 24, %g5
	cmp		%g4, %g5
	bgeu,pn		%xcc, kvmap_dtlb_longpath
	 nop

kvmap_check_obp:
	sethi		%hi(LOW_OBP_ADDRESS), %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_vmalloc_addr
	 mov		0x1, %g5
	sllx		%g5, 32, %g5
	cmp		%g4, %g5
	blu,pn		%xcc, kvmap_dtlb_obp
	 nop
	ba,pt		%xcc, kvmap_dtlb_vmalloc_addr
	 nop

kvmap_dtlb_longpath:
	rdpr	%pstate, %g5
	wrpr	%g5, PSTATE_AG | PSTATE_MG, %pstate
	rdpr	%tl, %g4
	cmp	%g4, 1
	mov	TLB_TAG_ACCESS, %g4
	ldxa	[%g4] ASI_DMMU, %g5
	be,pt	%xcc, sparc64_realfault_common
	 mov	FAULT_CODE_DTLB, %g4
	ba,pt	%xcc, winfix_trampoline
	 nop
Commit	Line	Data
2a7e2990 DM	1	/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
	2	*
	3	* Copyright (C) 1995, 1997, 2005 David S. Miller <davem@davemloft.net>
	4	* Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de)
	5	* Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx)
	6	* Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
74bf4312	7	*/
2a7e2990 DM	8
	9	#include <linux/config.h>
	10	#include <asm/head.h>
	11	#include <asm/asi.h>
	12	#include <asm/page.h>
	13	#include <asm/pgtable.h>
74bf4312	14	#include <asm/tsb.h>
2a7e2990 DM	15
	16	.text
	17	.align 32
	18
74bf4312 DM	19	.globl kvmap_itlb
	20	kvmap_itlb:
	21	/* g6: TAG TARGET */
	22	mov TLB_TAG_ACCESS, %g4
	23	ldxa [%g4] ASI_IMMU, %g4
	24
	25	kvmap_itlb_nonlinear:
	26	/* Catch kernel NULL pointer calls. */
	27	sethi %hi(PAGE_SIZE), %g5
	28	cmp %g4, %g5
	29	bleu,pn %xcc, kvmap_dtlb_longpath
	30	nop
	31
	32	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)
	33
	34	kvmap_itlb_tsb_miss:
2a7e2990 DM	35	sethi %hi(LOW_OBP_ADDRESS), %g5
2a7e2990 DM	36	cmp %g4, %g5
74bf4312	37	blu,pn %xcc, kvmap_itlb_vmalloc_addr
2a7e2990 DM	38	mov 0x1, %g5
	39	sllx %g5, 32, %g5
	40	cmp %g4, %g5
74bf4312	41	blu,pn %xcc, kvmap_itlb_obp
2a7e2990 DM	42	nop
2a7e2990 DM	43
74bf4312 DM	44	kvmap_itlb_vmalloc_addr:
	45	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
	46
	47	TSB_LOCK_TAG(%g1, %g2, %g4)
	48
	49	/* Load and check PTE. */
	50	ldxa [%g5] ASI_PHYS_USE_EC, %g5
	51	brgez,a,pn %g5, kvmap_itlb_longpath
	52	stx %g0, [%g1]
2a7e2990	53
74bf4312 DM	54	TSB_WRITE(%g1, %g5, %g6)
	55
	56	/* fallthrough to TLB load */
	57
	58	kvmap_itlb_load:
	59	stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Reload TLB
2a7e2990 DM	60	retry
2a7e2990 DM	61
74bf4312 DM	62	kvmap_itlb_longpath:
	63	rdpr %pstate, %g5
	64	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
	65	rdpr %tpc, %g5
	66	ba,pt %xcc, sparc64_realfault_common
	67	mov FAULT_CODE_ITLB, %g4
	68
	69	kvmap_itlb_obp:
	70	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)
	71
	72	TSB_LOCK_TAG(%g1, %g2, %g4)
	73
	74	TSB_WRITE(%g1, %g5, %g6)
	75
	76	ba,pt %xcc, kvmap_itlb_load
	77	nop
	78
	79	kvmap_dtlb_obp:
	80	OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)
	81
	82	TSB_LOCK_TAG(%g1, %g2, %g4)
	83
	84	TSB_WRITE(%g1, %g5, %g6)
	85
	86	ba,pt %xcc, kvmap_dtlb_load
	87	nop
c9c10830	88
2a7e2990	89	.align 32
74bf4312 DM	90	.globl kvmap_dtlb
	91	kvmap_dtlb:
	92	/* %g6: TAG TARGET */
	93	mov TLB_TAG_ACCESS, %g4
	94	ldxa [%g4] ASI_DMMU, %g4
	95	brgez,pn %g4, kvmap_dtlb_nonlinear
56425306 DM	96	nop
56425306 DM	97
74bf4312 DM	98	#define KERN_HIGHBITS ((_PAGE_VALID\|_PAGE_SZ4MB)^0xfffff80000000000)
	99	#define KERN_LOWBITS (_PAGE_CP \| _PAGE_CV \| _PAGE_P \| _PAGE_W)
	100
	101	sethi %uhi(KERN_HIGHBITS), %g2
	102	or %g2, %ulo(KERN_HIGHBITS), %g2
	103	sllx %g2, 32, %g2
	104	or %g2, KERN_LOWBITS, %g2
	105
	106	#undef KERN_HIGHBITS
	107	#undef KERN_LOWBITS
	108
56425306 DM	109	.globl kvmap_linear_patch
56425306 DM	110	kvmap_linear_patch:
74bf4312	111	ba,pt %xcc, kvmap_dtlb_load
2a7e2990 DM	112	xor %g2, %g4, %g5
2a7e2990 DM	113
74bf4312 DM	114	kvmap_dtlb_vmalloc_addr:
	115	KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
	116
	117	TSB_LOCK_TAG(%g1, %g2, %g4)
	118
	119	/* Load and check PTE. */
	120	ldxa [%g5] ASI_PHYS_USE_EC, %g5
	121	brgez,a,pn %g5, kvmap_dtlb_longpath
	122	stx %g0, [%g1]
	123
	124	TSB_WRITE(%g1, %g5, %g6)
	125
	126	/* fallthrough to TLB load */
	127
	128	kvmap_dtlb_load:
	129	stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
	130	retry
	131
	132	kvmap_dtlb_nonlinear:
	133	/* Catch kernel NULL pointer derefs. */
	134	sethi %hi(PAGE_SIZE), %g5
	135	cmp %g4, %g5
	136	bleu,pn %xcc, kvmap_dtlb_longpath
56425306	137	nop
56425306	138
74bf4312 DM	139	KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
	140
	141	kvmap_dtlb_tsbmiss:
2a7e2990 DM	142	sethi %hi(MODULES_VADDR), %g5
2a7e2990 DM	143	cmp %g4, %g5
74bf4312	144	blu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	145	mov (VMALLOC_END >> 24), %g5
	146	sllx %g5, 24, %g5
	147	cmp %g4, %g5
74bf4312	148	bgeu,pn %xcc, kvmap_dtlb_longpath
2a7e2990 DM	149	nop
	150
	151	kvmap_check_obp:
	152	sethi %hi(LOW_OBP_ADDRESS), %g5
	153	cmp %g4, %g5
74bf4312	154	blu,pn %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	155	mov 0x1, %g5
	156	sllx %g5, 32, %g5
	157	cmp %g4, %g5
74bf4312	158	blu,pn %xcc, kvmap_dtlb_obp
2a7e2990	159	nop
74bf4312	160	ba,pt %xcc, kvmap_dtlb_vmalloc_addr
2a7e2990 DM	161	nop
2a7e2990 DM	162
74bf4312 DM	163	kvmap_dtlb_longpath:
	164	rdpr %pstate, %g5
	165	wrpr %g5, PSTATE_AG \| PSTATE_MG, %pstate
	166	rdpr %tl, %g4
	167	cmp %g4, 1
	168	mov TLB_TAG_ACCESS, %g4
	169	ldxa [%g4] ASI_DMMU, %g5
	170	be,pt %xcc, sparc64_realfault_common
	171	mov FAULT_CODE_DTLB, %g4
	172	ba,pt %xcc, winfix_trampoline
	173	nop