Commit | Line | Data |
---|---|---|
0ca87f05 ME |
1 | /* bpf_jit.S: Packet/header access helper functions |
2 | * for PPC64 BPF compiler. | |
3 | * | |
4 | * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; version 2 | |
9 | * of the License. | |
10 | */ | |
11 | ||
12 | #include <asm/ppc_asm.h> | |
13 | #include "bpf_jit.h" | |
14 | ||
15 | /* | |
16 | * All of these routines are called directly from generated code, | |
17 | * whose register usage is: | |
18 | * | |
19 | * r3 skb | |
20 | * r4,r5 A,X | |
21 | * r6 *** address parameter to helper *** | |
22 | * r7-r10 scratch | |
23 | * r14 skb->data | |
24 | * r15 skb headlen | |
25 | * r16-31 M[] | |
26 | */ | |
27 | ||
28 | /* | |
29 | * To consider: These helpers are so small it could be better to just | |
30 | * generate them inline. Inline code can do the simple headlen check | |
31 | * then branch directly to slow_path_XXX if required. (In fact, could | |
32 | * load a spare GPR with the address of slow_path_generic and pass size | |
33 | * as an argument, making the call site a mtlr, li and bllr.) | |
0ca87f05 ME |
34 | */ |
35 | .globl sk_load_word | |
36 | sk_load_word: | |
37 | cmpdi r_addr, 0 | |
05be1824 JS |
38 | blt bpf_slow_path_word_neg |
39 | .globl sk_load_word_positive_offset | |
40 | sk_load_word_positive_offset: | |
0ca87f05 ME |
41 | /* Are we accessing past headlen? */ |
42 | subi r_scratch1, r_HL, 4 | |
43 | cmpd r_scratch1, r_addr | |
44 | blt bpf_slow_path_word | |
45 | /* Nope, just hitting the header. cr0 here is eq or gt! */ | |
46 | lwzx r_A, r_D, r_addr | |
47 | /* When big endian we don't need to byteswap. */ | |
48 | blr /* Return success, cr0 != LT */ | |
49 | ||
50 | .globl sk_load_half | |
51 | sk_load_half: | |
52 | cmpdi r_addr, 0 | |
05be1824 JS |
53 | blt bpf_slow_path_half_neg |
54 | .globl sk_load_half_positive_offset | |
55 | sk_load_half_positive_offset: | |
0ca87f05 ME |
56 | subi r_scratch1, r_HL, 2 |
57 | cmpd r_scratch1, r_addr | |
58 | blt bpf_slow_path_half | |
59 | lhzx r_A, r_D, r_addr | |
60 | blr | |
61 | ||
62 | .globl sk_load_byte | |
63 | sk_load_byte: | |
64 | cmpdi r_addr, 0 | |
05be1824 JS |
65 | blt bpf_slow_path_byte_neg |
66 | .globl sk_load_byte_positive_offset | |
67 | sk_load_byte_positive_offset: | |
0ca87f05 ME |
68 | cmpd r_HL, r_addr |
69 | ble bpf_slow_path_byte | |
70 | lbzx r_A, r_D, r_addr | |
71 | blr | |
72 | ||
73 | /* | |
74 | * BPF_S_LDX_B_MSH: ldxb 4*([offset]&0xf) | |
05be1824 | 75 | * r_addr is the offset value |
0ca87f05 ME |
76 | */ |
77 | .globl sk_load_byte_msh | |
78 | sk_load_byte_msh: | |
05be1824 JS |
79 | cmpdi r_addr, 0 |
80 | blt bpf_slow_path_byte_msh_neg | |
81 | .globl sk_load_byte_msh_positive_offset | |
82 | sk_load_byte_msh_positive_offset: | |
0ca87f05 ME |
83 | cmpd r_HL, r_addr |
84 | ble bpf_slow_path_byte_msh | |
85 | lbzx r_X, r_D, r_addr | |
86 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
87 | blr | |
88 | ||
0ca87f05 ME |
89 | /* Call out to skb_copy_bits: |
90 | * We'll need to back up our volatile regs first; we have | |
91 | * local variable space at r1+(BPF_PPC_STACK_BASIC). | |
92 | * Allocate a new stack frame here to remain ABI-compliant in | |
93 | * stashing LR. | |
94 | */ | |
95 | #define bpf_slow_path_common(SIZE) \ | |
96 | mflr r0; \ | |
97 | std r0, 16(r1); \ | |
98 | /* R3 goes in parameter space of caller's frame */ \ | |
99 | std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ | |
100 | std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ | |
101 | std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ | |
102 | addi r5, r1, BPF_PPC_STACK_BASIC+(2*8); \ | |
103 | stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ | |
104 | /* R3 = r_skb, as passed */ \ | |
105 | mr r4, r_addr; \ | |
106 | li r6, SIZE; \ | |
107 | bl skb_copy_bits; \ | |
7784655a | 108 | nop; \ |
0ca87f05 ME |
109 | /* R3 = 0 on success */ \ |
110 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ | |
111 | ld r0, 16(r1); \ | |
112 | ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ | |
113 | ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ | |
114 | mtlr r0; \ | |
115 | cmpdi r3, 0; \ | |
116 | blt bpf_error; /* cr0 = LT */ \ | |
117 | ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ | |
118 | /* Great success! */ | |
119 | ||
120 | bpf_slow_path_word: | |
121 | bpf_slow_path_common(4) | |
122 | /* Data value is on stack, and cr0 != LT */ | |
123 | lwz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
124 | blr | |
125 | ||
126 | bpf_slow_path_half: | |
127 | bpf_slow_path_common(2) | |
128 | lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
129 | blr | |
130 | ||
131 | bpf_slow_path_byte: | |
132 | bpf_slow_path_common(1) | |
133 | lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
134 | blr | |
135 | ||
136 | bpf_slow_path_byte_msh: | |
137 | bpf_slow_path_common(1) | |
138 | lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
139 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
140 | blr | |
05be1824 JS |
141 | |
142 | /* Call out to bpf_internal_load_pointer_neg_helper: | |
143 | * We'll need to back up our volatile regs first; we have | |
144 | * local variable space at r1+(BPF_PPC_STACK_BASIC). | |
145 | * Allocate a new stack frame here to remain ABI-compliant in | |
146 | * stashing LR. | |
147 | */ | |
148 | #define sk_negative_common(SIZE) \ | |
149 | mflr r0; \ | |
150 | std r0, 16(r1); \ | |
151 | /* R3 goes in parameter space of caller's frame */ \ | |
152 | std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ | |
153 | std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ | |
154 | std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ | |
155 | stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ | |
156 | /* R3 = r_skb, as passed */ \ | |
157 | mr r4, r_addr; \ | |
158 | li r5, SIZE; \ | |
159 | bl bpf_internal_load_pointer_neg_helper; \ | |
7784655a | 160 | nop; \ |
05be1824 JS |
161 | /* R3 != 0 on success */ \ |
162 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ | |
163 | ld r0, 16(r1); \ | |
164 | ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ | |
165 | ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ | |
166 | mtlr r0; \ | |
167 | cmpldi r3, 0; \ | |
168 | beq bpf_error_slow; /* cr0 = EQ */ \ | |
169 | mr r_addr, r3; \ | |
170 | ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ | |
171 | /* Great success! */ | |
172 | ||
173 | bpf_slow_path_word_neg: | |
174 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
175 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ | |
176 | blt bpf_error /* cr0 = LT */ | |
177 | .globl sk_load_word_negative_offset | |
178 | sk_load_word_negative_offset: | |
179 | sk_negative_common(4) | |
180 | lwz r_A, 0(r_addr) | |
181 | blr | |
182 | ||
183 | bpf_slow_path_half_neg: | |
184 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
185 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ | |
186 | blt bpf_error /* cr0 = LT */ | |
187 | .globl sk_load_half_negative_offset | |
188 | sk_load_half_negative_offset: | |
189 | sk_negative_common(2) | |
190 | lhz r_A, 0(r_addr) | |
191 | blr | |
192 | ||
193 | bpf_slow_path_byte_neg: | |
194 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
195 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ | |
196 | blt bpf_error /* cr0 = LT */ | |
197 | .globl sk_load_byte_negative_offset | |
198 | sk_load_byte_negative_offset: | |
199 | sk_negative_common(1) | |
200 | lbz r_A, 0(r_addr) | |
201 | blr | |
202 | ||
203 | bpf_slow_path_byte_msh_neg: | |
204 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
205 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ | |
206 | blt bpf_error /* cr0 = LT */ | |
207 | .globl sk_load_byte_msh_negative_offset | |
208 | sk_load_byte_msh_negative_offset: | |
209 | sk_negative_common(1) | |
210 | lbz r_X, 0(r_addr) | |
211 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
212 | blr | |
213 | ||
214 | bpf_error_slow: | |
215 | /* fabricate a cr0 = lt */ | |
216 | li r_scratch1, -1 | |
217 | cmpdi r_scratch1, 0 | |
218 | bpf_error: | |
219 | /* Entered with cr0 = lt */ | |
220 | li r3, 0 | |
221 | /* Generated code will 'blt epilogue', returning 0. */ | |
222 | blr |