[deliverable/linux.git] / arch / sparc64 / lib / bitops.S

/* bitops.S: Sparc64 atomic bit operations.
 *
 * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
 */

#include <asm/asi.h>
#include <asm/backoff.h>

	.text

	/* On SMP we need to use memory barriers to ensure
	 * correct memory operation ordering, nop these out
	 * for uniprocessor.
	 */

#ifdef CONFIG_SMP
#define BITOP_PRE_BARRIER	membar #StoreLoad | #LoadLoad
#define BITOP_POST_BARRIER	\
	ba,pt	%xcc, 80b;	\
	membar #StoreLoad | #StoreStore

80:	retl
	 nop
#else
#define BITOP_PRE_BARRIER
#define BITOP_POST_BARRIER
#endif

	.globl	test_and_set_bit
	.type	test_and_set_bit,#function
test_and_set_bit:	/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	BITOP_PRE_BARRIER
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	or	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 and	%g7, %o2, %g2
	clr	%o0
	movrne	%g2, 1, %o0
	BITOP_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	test_and_set_bit, .-test_and_set_bit

	.globl	test_and_clear_bit
	.type	test_and_clear_bit,#function
test_and_clear_bit:	/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	BITOP_PRE_BARRIER
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	andn	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 and	%g7, %o2, %g2
	clr	%o0
	movrne	%g2, 1, %o0
	BITOP_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	test_and_clear_bit, .-test_and_clear_bit

	.globl	test_and_change_bit
	.type	test_and_change_bit,#function
test_and_change_bit:	/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	BITOP_PRE_BARRIER
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	xor	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 and	%g7, %o2, %g2
	clr	%o0
	movrne	%g2, 1, %o0
	BITOP_POST_BARRIER
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	test_and_change_bit, .-test_and_change_bit

	.globl	set_bit
	.type	set_bit,#function
set_bit:		/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	or	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	set_bit, .-set_bit

	.globl	clear_bit
	.type	clear_bit,#function
clear_bit:		/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	andn	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	clear_bit, .-clear_bit

	.globl	change_bit
	.type	change_bit,#function
change_bit:		/* %o0=nr, %o1=addr */
	BACKOFF_SETUP(%o3)
	srlx	%o0, 6, %g1
	mov	1, %o2
	sllx	%g1, 3, %g3
	and	%o0, 63, %g2
	sllx	%o2, %g2, %o2
	add	%o1, %g3, %o1
1:	ldx	[%o1], %g7
	xor	%g7, %o2, %g1
	casx	[%o1], %g7, %g1
	cmp	%g7, %g1
	bne,pn	%xcc, 2f
	 nop
	retl
	 nop
2:	BACKOFF_SPIN(%o3, %o4, 1b)
	.size	change_bit, .-change_bit
Commit	Line	Data
24f287e4	1	/* bitops.S: Sparc64 atomic bit operations.
1da177e4	2	*
24f287e4	3	* Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net)
1da177e4 LT	4	*/
1da177e4 LT	5
1da177e4	6	#include <asm/asi.h>
24f287e4	7	#include <asm/backoff.h>
1da177e4	8
b445e26c DM	9	.text
b445e26c DM	10
1da177e4 LT	11	/* On SMP we need to use memory barriers to ensure
	12	* correct memory operation ordering, nop these out
	13	* for uniprocessor.
	14	*/
b445e26c	15
1da177e4 LT	16	#ifdef CONFIG_SMP
1da177e4 LT	17	#define BITOP_PRE_BARRIER membar #StoreLoad \| #LoadLoad
b445e26c DM	18	#define BITOP_POST_BARRIER \
	19	ba,pt %xcc, 80b; \
	20	membar #StoreLoad \| #StoreStore
	21
	22	80: retl
	23	nop
1da177e4	24	#else
b445e26c DM	25	#define BITOP_PRE_BARRIER
b445e26c DM	26	#define BITOP_POST_BARRIER
1da177e4 LT	27	#endif
1da177e4 LT	28
1da177e4 LT	29	.globl test_and_set_bit
	30	.type test_and_set_bit,#function
	31	test_and_set_bit: /* %o0=nr, %o1=addr */
24f287e4	32	BACKOFF_SETUP(%o3)
1da177e4 LT	33	BITOP_PRE_BARRIER
	34	srlx %o0, 6, %g1
	35	mov 1, %o2
	36	sllx %g1, 3, %g3
	37	and %o0, 63, %g2
	38	sllx %o2, %g2, %o2
	39	add %o1, %g3, %o1
	40	1: ldx [%o1], %g7
	41	or %g7, %o2, %g1
	42	casx [%o1], %g7, %g1
	43	cmp %g7, %g1
24f287e4	44	bne,pn %xcc, 2f
1da177e4	45	and %g7, %o2, %g2
1da177e4	46	clr %o0
b445e26c DM	47	movrne %g2, 1, %o0
b445e26c DM	48	BITOP_POST_BARRIER
1da177e4	49	retl
b445e26c	50	nop
24f287e4	51	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4 LT	52	.size test_and_set_bit, .-test_and_set_bit
	53
	54	.globl test_and_clear_bit
	55	.type test_and_clear_bit,#function
	56	test_and_clear_bit: /* %o0=nr, %o1=addr */
24f287e4	57	BACKOFF_SETUP(%o3)
1da177e4 LT	58	BITOP_PRE_BARRIER
	59	srlx %o0, 6, %g1
	60	mov 1, %o2
	61	sllx %g1, 3, %g3
	62	and %o0, 63, %g2
	63	sllx %o2, %g2, %o2
	64	add %o1, %g3, %o1
	65	1: ldx [%o1], %g7
	66	andn %g7, %o2, %g1
	67	casx [%o1], %g7, %g1
	68	cmp %g7, %g1
24f287e4	69	bne,pn %xcc, 2f
1da177e4	70	and %g7, %o2, %g2
1da177e4	71	clr %o0
b445e26c DM	72	movrne %g2, 1, %o0
b445e26c DM	73	BITOP_POST_BARRIER
1da177e4	74	retl
b445e26c	75	nop
24f287e4	76	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4 LT	77	.size test_and_clear_bit, .-test_and_clear_bit
	78
	79	.globl test_and_change_bit
	80	.type test_and_change_bit,#function
	81	test_and_change_bit: /* %o0=nr, %o1=addr */
24f287e4	82	BACKOFF_SETUP(%o3)
1da177e4 LT	83	BITOP_PRE_BARRIER
	84	srlx %o0, 6, %g1
	85	mov 1, %o2
	86	sllx %g1, 3, %g3
	87	and %o0, 63, %g2
	88	sllx %o2, %g2, %o2
	89	add %o1, %g3, %o1
	90	1: ldx [%o1], %g7
	91	xor %g7, %o2, %g1
	92	casx [%o1], %g7, %g1
	93	cmp %g7, %g1
24f287e4	94	bne,pn %xcc, 2f
1da177e4	95	and %g7, %o2, %g2
1da177e4	96	clr %o0
b445e26c DM	97	movrne %g2, 1, %o0
b445e26c DM	98	BITOP_POST_BARRIER
1da177e4	99	retl
b445e26c	100	nop
24f287e4	101	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4 LT	102	.size test_and_change_bit, .-test_and_change_bit
	103
	104	.globl set_bit
	105	.type set_bit,#function
	106	set_bit: /* %o0=nr, %o1=addr */
24f287e4	107	BACKOFF_SETUP(%o3)
1da177e4 LT	108	srlx %o0, 6, %g1
	109	mov 1, %o2
	110	sllx %g1, 3, %g3
	111	and %o0, 63, %g2
	112	sllx %o2, %g2, %o2
	113	add %o1, %g3, %o1
	114	1: ldx [%o1], %g7
	115	or %g7, %o2, %g1
	116	casx [%o1], %g7, %g1
	117	cmp %g7, %g1
24f287e4	118	bne,pn %xcc, 2f
1da177e4 LT	119	nop
	120	retl
	121	nop
24f287e4	122	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4 LT	123	.size set_bit, .-set_bit
	124
	125	.globl clear_bit
	126	.type clear_bit,#function
	127	clear_bit: /* %o0=nr, %o1=addr */
24f287e4	128	BACKOFF_SETUP(%o3)
1da177e4 LT	129	srlx %o0, 6, %g1
	130	mov 1, %o2
	131	sllx %g1, 3, %g3
	132	and %o0, 63, %g2
	133	sllx %o2, %g2, %o2
	134	add %o1, %g3, %o1
	135	1: ldx [%o1], %g7
	136	andn %g7, %o2, %g1
	137	casx [%o1], %g7, %g1
	138	cmp %g7, %g1
24f287e4	139	bne,pn %xcc, 2f
1da177e4 LT	140	nop
	141	retl
	142	nop
24f287e4	143	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4 LT	144	.size clear_bit, .-clear_bit
	145
	146	.globl change_bit
	147	.type change_bit,#function
	148	change_bit: /* %o0=nr, %o1=addr */
24f287e4	149	BACKOFF_SETUP(%o3)
1da177e4 LT	150	srlx %o0, 6, %g1
	151	mov 1, %o2
	152	sllx %g1, 3, %g3
	153	and %o0, 63, %g2
	154	sllx %o2, %g2, %o2
	155	add %o1, %g3, %o1
	156	1: ldx [%o1], %g7
	157	xor %g7, %o2, %g1
	158	casx [%o1], %g7, %g1
	159	cmp %g7, %g1
24f287e4	160	bne,pn %xcc, 2f
1da177e4 LT	161	nop
	162	retl
	163	nop
24f287e4	164	2: BACKOFF_SPIN(%o3, %o4, 1b)
1da177e4	165	.size change_bit, .-change_bit