[deliverable/linux.git] / arch / powerpc / net / bpf_jit_asm.S

/* bpf_jit.S: Packet/header access helper functions
 * for PPC64 BPF compiler.
 *
 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; version 2
 * of the License.
 */

#include <asm/ppc_asm.h>
#include "bpf_jit.h"

/*
 * All of these routines are called directly from generated code,
 * whose register usage is:
 *
 * r3		skb
 * r4,r5	A,X
 * r6		*** address parameter to helper ***
 * r7-r10	scratch
 * r14		skb->data
 * r15		skb headlen
 * r16-31	M[]
 */

/*
 * To consider: These helpers are so small it could be better to just
 * generate them inline.  Inline code can do the simple headlen check
 * then branch directly to slow_path_XXX if required.  (In fact, could
 * load a spare GPR with the address of slow_path_generic and pass size
 * as an argument, making the call site a mtlr, li and bllr.)
 */
	.globl	sk_load_word
sk_load_word:
	PPC_LCMPI	r_addr, 0
	blt	bpf_slow_path_word_neg
	.globl	sk_load_word_positive_offset
sk_load_word_positive_offset:
	/* Are we accessing past headlen? */
	subi	r_scratch1, r_HL, 4
	PPC_LCMP	r_scratch1, r_addr
	blt	bpf_slow_path_word
	/* Nope, just hitting the header.  cr0 here is eq or gt! */
#ifdef __LITTLE_ENDIAN__
	lwbrx	r_A, r_D, r_addr
#else
	lwzx	r_A, r_D, r_addr
#endif
	blr	/* Return success, cr0 != LT */

	.globl	sk_load_half
sk_load_half:
	PPC_LCMPI	r_addr, 0
	blt	bpf_slow_path_half_neg
	.globl	sk_load_half_positive_offset
sk_load_half_positive_offset:
	subi	r_scratch1, r_HL, 2
	PPC_LCMP	r_scratch1, r_addr
	blt	bpf_slow_path_half
#ifdef __LITTLE_ENDIAN__
	lhbrx	r_A, r_D, r_addr
#else
	lhzx	r_A, r_D, r_addr
#endif
	blr

	.globl	sk_load_byte
sk_load_byte:
	PPC_LCMPI	r_addr, 0
	blt	bpf_slow_path_byte_neg
	.globl	sk_load_byte_positive_offset
sk_load_byte_positive_offset:
	PPC_LCMP	r_HL, r_addr
	ble	bpf_slow_path_byte
	lbzx	r_A, r_D, r_addr
	blr

/*
 * BPF_LDX | BPF_B | BPF_MSH: ldxb  4*([offset]&0xf)
 * r_addr is the offset value
 */
	.globl sk_load_byte_msh
sk_load_byte_msh:
	PPC_LCMPI	r_addr, 0
	blt	bpf_slow_path_byte_msh_neg
	.globl sk_load_byte_msh_positive_offset
sk_load_byte_msh_positive_offset:
	PPC_LCMP	r_HL, r_addr
	ble	bpf_slow_path_byte_msh
	lbzx	r_X, r_D, r_addr
	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
	blr

/* Call out to skb_copy_bits:
 * We'll need to back up our volatile regs first; we have
 * local variable space at r1+(BPF_PPC_STACK_BASIC).
 * Allocate a new stack frame here to remain ABI-compliant in
 * stashing LR.
 */
#define bpf_slow_path_common(SIZE)				\
	mflr	r0;						\
	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
	/* R3 goes in parameter space of caller's frame */	\
	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
	addi	r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ);		\
	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
	/* R3 = r_skb, as passed */				\
	mr	r4, r_addr;					\
	li	r6, SIZE;					\
	bl	skb_copy_bits;					\
	nop;							\
	/* R3 = 0 on success */					\
	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
	mtlr	r0;						\
	PPC_LCMPI	r3, 0;						\
	blt	bpf_error;	/* cr0 = LT */			\
	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
	/* Great success! */

bpf_slow_path_word:
	bpf_slow_path_common(4)
	/* Data value is on stack, and cr0 != LT */
	lwz	r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
	blr

bpf_slow_path_half:
	bpf_slow_path_common(2)
	lhz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
	blr

bpf_slow_path_byte:
	bpf_slow_path_common(1)
	lbz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
	blr

bpf_slow_path_byte_msh:
	bpf_slow_path_common(1)
	lbz	r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
	blr

/* Call out to bpf_internal_load_pointer_neg_helper:
 * We'll need to back up our volatile regs first; we have
 * local variable space at r1+(BPF_PPC_STACK_BASIC).
 * Allocate a new stack frame here to remain ABI-compliant in
 * stashing LR.
 */
#define sk_negative_common(SIZE)				\
	mflr	r0;						\
	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
	/* R3 goes in parameter space of caller's frame */	\
	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
	/* R3 = r_skb, as passed */				\
	mr	r4, r_addr;					\
	li	r5, SIZE;					\
	bl	bpf_internal_load_pointer_neg_helper;		\
	nop;							\
	/* R3 != 0 on success */				\
	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
	mtlr	r0;						\
	PPC_LCMPLI	r3, 0;						\
	beq	bpf_error_slow;	/* cr0 = EQ */			\
	mr	r_addr, r3;					\
	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
	/* Great success! */

bpf_slow_path_word_neg:
	lis     r_scratch1,-32	/* SKF_LL_OFF */
	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
	blt	bpf_error	/* cr0 = LT */
	.globl	sk_load_word_negative_offset
sk_load_word_negative_offset:
	sk_negative_common(4)
	lwz	r_A, 0(r_addr)
	blr

bpf_slow_path_half_neg:
	lis     r_scratch1,-32	/* SKF_LL_OFF */
	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
	blt	bpf_error	/* cr0 = LT */
	.globl	sk_load_half_negative_offset
sk_load_half_negative_offset:
	sk_negative_common(2)
	lhz	r_A, 0(r_addr)
	blr

bpf_slow_path_byte_neg:
	lis     r_scratch1,-32	/* SKF_LL_OFF */
	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
	blt	bpf_error	/* cr0 = LT */
	.globl	sk_load_byte_negative_offset
sk_load_byte_negative_offset:
	sk_negative_common(1)
	lbz	r_A, 0(r_addr)
	blr

bpf_slow_path_byte_msh_neg:
	lis     r_scratch1,-32	/* SKF_LL_OFF */
	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
	blt	bpf_error	/* cr0 = LT */
	.globl	sk_load_byte_msh_negative_offset
sk_load_byte_msh_negative_offset:
	sk_negative_common(1)
	lbz	r_X, 0(r_addr)
	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
	blr

bpf_error_slow:
	/* fabricate a cr0 = lt */
	li	r_scratch1, -1
	PPC_LCMPI	r_scratch1, 0
bpf_error:
	/* Entered with cr0 = lt */
	li	r3, 0
	/* Generated code will 'blt epilogue', returning 0. */
	blr
Commit	Line	Data
0ca87f05 ME	1	/* bpf_jit.S: Packet/header access helper functions
	2	* for PPC64 BPF compiler.
	3	*
	4	* Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License
	8	* as published by the Free Software Foundation; version 2
	9	* of the License.
	10	*/
	11
	12	#include <asm/ppc_asm.h>
	13	#include "bpf_jit.h"
	14
	15	/*
	16	* All of these routines are called directly from generated code,
	17	* whose register usage is:
	18	*
	19	* r3 skb
	20	* r4,r5 A,X
	21	* r6 * address parameter to helper *
	22	* r7-r10 scratch
	23	* r14 skb->data
	24	* r15 skb headlen
	25	* r16-31 M[]
	26	*/
	27
	28	/*
	29	* To consider: These helpers are so small it could be better to just
	30	* generate them inline. Inline code can do the simple headlen check
	31	* then branch directly to slow_path_XXX if required. (In fact, could
	32	* load a spare GPR with the address of slow_path_generic and pass size
	33	* as an argument, making the call site a mtlr, li and bllr.)
0ca87f05 ME	34	*/
	35	.globl sk_load_word
	36	sk_load_word:
09ca5ab2	37	PPC_LCMPI r_addr, 0
05be1824 JS	38	blt bpf_slow_path_word_neg
	39	.globl sk_load_word_positive_offset
	40	sk_load_word_positive_offset:
0ca87f05 ME	41	/* Are we accessing past headlen? */
0ca87f05 ME	42	subi r_scratch1, r_HL, 4
09ca5ab2	43	PPC_LCMP r_scratch1, r_addr
0ca87f05 ME	44	blt bpf_slow_path_word
0ca87f05 ME	45	/* Nope, just hitting the header. cr0 here is eq or gt! */
9c662cad PB	46	#ifdef __LITTLE_ENDIAN__
	47	lwbrx r_A, r_D, r_addr
	48	#else
0ca87f05	49	lwzx r_A, r_D, r_addr
9c662cad	50	#endif
0ca87f05 ME	51	blr /* Return success, cr0 != LT */
	52
	53	.globl sk_load_half
	54	sk_load_half:
09ca5ab2	55	PPC_LCMPI r_addr, 0
05be1824 JS	56	blt bpf_slow_path_half_neg
	57	.globl sk_load_half_positive_offset
	58	sk_load_half_positive_offset:
0ca87f05	59	subi r_scratch1, r_HL, 2
09ca5ab2	60	PPC_LCMP r_scratch1, r_addr
0ca87f05	61	blt bpf_slow_path_half
9c662cad PB	62	#ifdef __LITTLE_ENDIAN__
	63	lhbrx r_A, r_D, r_addr
	64	#else
0ca87f05	65	lhzx r_A, r_D, r_addr
9c662cad	66	#endif
0ca87f05 ME	67	blr
	68
	69	.globl sk_load_byte
	70	sk_load_byte:
09ca5ab2	71	PPC_LCMPI r_addr, 0
05be1824 JS	72	blt bpf_slow_path_byte_neg
	73	.globl sk_load_byte_positive_offset
	74	sk_load_byte_positive_offset:
09ca5ab2	75	PPC_LCMP r_HL, r_addr
0ca87f05 ME	76	ble bpf_slow_path_byte
	77	lbzx r_A, r_D, r_addr
	78	blr
	79
	80	/*
34805931	81	* BPF_LDX \| BPF_B \| BPF_MSH: ldxb 4*([offset]&0xf)
05be1824	82	* r_addr is the offset value
0ca87f05 ME	83	*/
	84	.globl sk_load_byte_msh
	85	sk_load_byte_msh:
09ca5ab2	86	PPC_LCMPI r_addr, 0
05be1824 JS	87	blt bpf_slow_path_byte_msh_neg
	88	.globl sk_load_byte_msh_positive_offset
	89	sk_load_byte_msh_positive_offset:
09ca5ab2	90	PPC_LCMP r_HL, r_addr
0ca87f05 ME	91	ble bpf_slow_path_byte_msh
	92	lbzx r_X, r_D, r_addr
	93	rlwinm r_X, r_X, 2, 32-4-2, 31-2
	94	blr
	95
0ca87f05 ME	96	/* Call out to skb_copy_bits:
	97	* We'll need to back up our volatile regs first; we have
	98	* local variable space at r1+(BPF_PPC_STACK_BASIC).
	99	* Allocate a new stack frame here to remain ABI-compliant in
	100	* stashing LR.
	101	*/
	102	#define bpf_slow_path_common(SIZE) \
	103	mflr r0; \
09ca5ab2	104	PPC_STL r0, PPC_LR_STKOFF(r1); \
0ca87f05	105	/* R3 goes in parameter space of caller's frame */ \
09ca5ab2 DK	106	PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
	107	PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
	108	PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
	109	addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \
	110	PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
0ca87f05 ME	111	/* R3 = r_skb, as passed */ \
	112	mr r4, r_addr; \
	113	li r6, SIZE; \
	114	bl skb_copy_bits; \
7784655a	115	nop; \
0ca87f05 ME	116	/* R3 = 0 on success */ \
0ca87f05 ME	117	addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
09ca5ab2 DK	118	PPC_LL r0, PPC_LR_STKOFF(r1); \
	119	PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
	120	PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
0ca87f05	121	mtlr r0; \
09ca5ab2	122	PPC_LCMPI r3, 0; \
0ca87f05	123	blt bpf_error; /* cr0 = LT */ \
09ca5ab2	124	PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
0ca87f05 ME	125	/* Great success! */
	126
	127	bpf_slow_path_word:
	128	bpf_slow_path_common(4)
	129	/* Data value is on stack, and cr0 != LT */
09ca5ab2	130	lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
0ca87f05 ME	131	blr
	132
	133	bpf_slow_path_half:
	134	bpf_slow_path_common(2)
	135	lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
	136	blr
	137
	138	bpf_slow_path_byte:
	139	bpf_slow_path_common(1)
	140	lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
	141	blr
	142
	143	bpf_slow_path_byte_msh:
	144	bpf_slow_path_common(1)
	145	lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
	146	rlwinm r_X, r_X, 2, 32-4-2, 31-2
	147	blr
05be1824 JS	148
	149	/* Call out to bpf_internal_load_pointer_neg_helper:
	150	* We'll need to back up our volatile regs first; we have
	151	* local variable space at r1+(BPF_PPC_STACK_BASIC).
	152	* Allocate a new stack frame here to remain ABI-compliant in
	153	* stashing LR.
	154	*/
	155	#define sk_negative_common(SIZE) \
	156	mflr r0; \
09ca5ab2	157	PPC_STL r0, PPC_LR_STKOFF(r1); \
05be1824	158	/* R3 goes in parameter space of caller's frame */ \
09ca5ab2 DK	159	PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
	160	PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
	161	PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
	162	PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
05be1824 JS	163	/* R3 = r_skb, as passed */ \
	164	mr r4, r_addr; \
	165	li r5, SIZE; \
	166	bl bpf_internal_load_pointer_neg_helper; \
7784655a	167	nop; \
05be1824 JS	168	/* R3 != 0 on success */ \
05be1824 JS	169	addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
09ca5ab2 DK	170	PPC_LL r0, PPC_LR_STKOFF(r1); \
	171	PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \
	172	PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \
05be1824	173	mtlr r0; \
09ca5ab2	174	PPC_LCMPLI r3, 0; \
05be1824 JS	175	beq bpf_error_slow; /* cr0 = EQ */ \
05be1824 JS	176	mr r_addr, r3; \
09ca5ab2	177	PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \
05be1824 JS	178	/* Great success! */
	179
	180	bpf_slow_path_word_neg:
	181	lis r_scratch1,-32 /* SKF_LL_OFF */
09ca5ab2	182	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
05be1824 JS	183	blt bpf_error /* cr0 = LT */
	184	.globl sk_load_word_negative_offset
	185	sk_load_word_negative_offset:
	186	sk_negative_common(4)
	187	lwz r_A, 0(r_addr)
	188	blr
	189
	190	bpf_slow_path_half_neg:
	191	lis r_scratch1,-32 /* SKF_LL_OFF */
09ca5ab2	192	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
05be1824 JS	193	blt bpf_error /* cr0 = LT */
	194	.globl sk_load_half_negative_offset
	195	sk_load_half_negative_offset:
	196	sk_negative_common(2)
	197	lhz r_A, 0(r_addr)
	198	blr
	199
	200	bpf_slow_path_byte_neg:
	201	lis r_scratch1,-32 /* SKF_LL_OFF */
09ca5ab2	202	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
05be1824 JS	203	blt bpf_error /* cr0 = LT */
	204	.globl sk_load_byte_negative_offset
	205	sk_load_byte_negative_offset:
	206	sk_negative_common(1)
	207	lbz r_A, 0(r_addr)
	208	blr
	209
	210	bpf_slow_path_byte_msh_neg:
	211	lis r_scratch1,-32 /* SKF_LL_OFF */
09ca5ab2	212	PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */
05be1824 JS	213	blt bpf_error /* cr0 = LT */
	214	.globl sk_load_byte_msh_negative_offset
	215	sk_load_byte_msh_negative_offset:
	216	sk_negative_common(1)
	217	lbz r_X, 0(r_addr)
	218	rlwinm r_X, r_X, 2, 32-4-2, 31-2
	219	blr
	220
	221	bpf_error_slow:
	222	/* fabricate a cr0 = lt */
	223	li r_scratch1, -1
09ca5ab2	224	PPC_LCMPI r_scratch1, 0
05be1824 JS	225	bpf_error:
	226	/* Entered with cr0 = lt */
	227	li r3, 0
	228	/* Generated code will 'blt epilogue', returning 0. */
	229	blr