Commit | Line | Data |
---|---|---|
0ca87f05 ME |
1 | /* bpf_jit.S: Packet/header access helper functions |
2 | * for PPC64 BPF compiler. | |
3 | * | |
4 | * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or | |
7 | * modify it under the terms of the GNU General Public License | |
8 | * as published by the Free Software Foundation; version 2 | |
9 | * of the License. | |
10 | */ | |
11 | ||
12 | #include <asm/ppc_asm.h> | |
13 | #include "bpf_jit.h" | |
14 | ||
15 | /* | |
16 | * All of these routines are called directly from generated code, | |
17 | * whose register usage is: | |
18 | * | |
19 | * r3 skb | |
20 | * r4,r5 A,X | |
21 | * r6 *** address parameter to helper *** | |
22 | * r7-r10 scratch | |
23 | * r14 skb->data | |
24 | * r15 skb headlen | |
25 | * r16-31 M[] | |
26 | */ | |
27 | ||
28 | /* | |
29 | * To consider: These helpers are so small it could be better to just | |
30 | * generate them inline. Inline code can do the simple headlen check | |
31 | * then branch directly to slow_path_XXX if required. (In fact, could | |
32 | * load a spare GPR with the address of slow_path_generic and pass size | |
33 | * as an argument, making the call site a mtlr, li and bllr.) | |
0ca87f05 ME |
34 | */ |
35 | .globl sk_load_word | |
36 | sk_load_word: | |
09ca5ab2 | 37 | PPC_LCMPI r_addr, 0 |
05be1824 JS |
38 | blt bpf_slow_path_word_neg |
39 | .globl sk_load_word_positive_offset | |
40 | sk_load_word_positive_offset: | |
0ca87f05 ME |
41 | /* Are we accessing past headlen? */ |
42 | subi r_scratch1, r_HL, 4 | |
09ca5ab2 | 43 | PPC_LCMP r_scratch1, r_addr |
0ca87f05 ME |
44 | blt bpf_slow_path_word |
45 | /* Nope, just hitting the header. cr0 here is eq or gt! */ | |
9c662cad PB |
46 | #ifdef __LITTLE_ENDIAN__ |
47 | lwbrx r_A, r_D, r_addr | |
48 | #else | |
0ca87f05 | 49 | lwzx r_A, r_D, r_addr |
9c662cad | 50 | #endif |
0ca87f05 ME |
51 | blr /* Return success, cr0 != LT */ |
52 | ||
53 | .globl sk_load_half | |
54 | sk_load_half: | |
09ca5ab2 | 55 | PPC_LCMPI r_addr, 0 |
05be1824 JS |
56 | blt bpf_slow_path_half_neg |
57 | .globl sk_load_half_positive_offset | |
58 | sk_load_half_positive_offset: | |
0ca87f05 | 59 | subi r_scratch1, r_HL, 2 |
09ca5ab2 | 60 | PPC_LCMP r_scratch1, r_addr |
0ca87f05 | 61 | blt bpf_slow_path_half |
9c662cad PB |
62 | #ifdef __LITTLE_ENDIAN__ |
63 | lhbrx r_A, r_D, r_addr | |
64 | #else | |
0ca87f05 | 65 | lhzx r_A, r_D, r_addr |
9c662cad | 66 | #endif |
0ca87f05 ME |
67 | blr |
68 | ||
69 | .globl sk_load_byte | |
70 | sk_load_byte: | |
09ca5ab2 | 71 | PPC_LCMPI r_addr, 0 |
05be1824 JS |
72 | blt bpf_slow_path_byte_neg |
73 | .globl sk_load_byte_positive_offset | |
74 | sk_load_byte_positive_offset: | |
09ca5ab2 | 75 | PPC_LCMP r_HL, r_addr |
0ca87f05 ME |
76 | ble bpf_slow_path_byte |
77 | lbzx r_A, r_D, r_addr | |
78 | blr | |
79 | ||
80 | /* | |
34805931 | 81 | * BPF_LDX | BPF_B | BPF_MSH: ldxb 4*([offset]&0xf) |
05be1824 | 82 | * r_addr is the offset value |
0ca87f05 ME |
83 | */ |
84 | .globl sk_load_byte_msh | |
85 | sk_load_byte_msh: | |
09ca5ab2 | 86 | PPC_LCMPI r_addr, 0 |
05be1824 JS |
87 | blt bpf_slow_path_byte_msh_neg |
88 | .globl sk_load_byte_msh_positive_offset | |
89 | sk_load_byte_msh_positive_offset: | |
09ca5ab2 | 90 | PPC_LCMP r_HL, r_addr |
0ca87f05 ME |
91 | ble bpf_slow_path_byte_msh |
92 | lbzx r_X, r_D, r_addr | |
93 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
94 | blr | |
95 | ||
0ca87f05 ME |
96 | /* Call out to skb_copy_bits: |
97 | * We'll need to back up our volatile regs first; we have | |
98 | * local variable space at r1+(BPF_PPC_STACK_BASIC). | |
99 | * Allocate a new stack frame here to remain ABI-compliant in | |
100 | * stashing LR. | |
101 | */ | |
102 | #define bpf_slow_path_common(SIZE) \ | |
103 | mflr r0; \ | |
09ca5ab2 | 104 | PPC_STL r0, PPC_LR_STKOFF(r1); \ |
0ca87f05 | 105 | /* R3 goes in parameter space of caller's frame */ \ |
09ca5ab2 DK |
106 | PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ |
107 | PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ | |
108 | PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ | |
109 | addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \ | |
110 | PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ | |
0ca87f05 ME |
111 | /* R3 = r_skb, as passed */ \ |
112 | mr r4, r_addr; \ | |
113 | li r6, SIZE; \ | |
114 | bl skb_copy_bits; \ | |
7784655a | 115 | nop; \ |
0ca87f05 ME |
116 | /* R3 = 0 on success */ \ |
117 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ | |
09ca5ab2 DK |
118 | PPC_LL r0, PPC_LR_STKOFF(r1); \ |
119 | PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ | |
120 | PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ | |
0ca87f05 | 121 | mtlr r0; \ |
09ca5ab2 | 122 | PPC_LCMPI r3, 0; \ |
0ca87f05 | 123 | blt bpf_error; /* cr0 = LT */ \ |
09ca5ab2 | 124 | PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ |
0ca87f05 ME |
125 | /* Great success! */ |
126 | ||
127 | bpf_slow_path_word: | |
128 | bpf_slow_path_common(4) | |
129 | /* Data value is on stack, and cr0 != LT */ | |
09ca5ab2 | 130 | lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1) |
0ca87f05 ME |
131 | blr |
132 | ||
133 | bpf_slow_path_half: | |
134 | bpf_slow_path_common(2) | |
135 | lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
136 | blr | |
137 | ||
138 | bpf_slow_path_byte: | |
139 | bpf_slow_path_common(1) | |
140 | lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
141 | blr | |
142 | ||
143 | bpf_slow_path_byte_msh: | |
144 | bpf_slow_path_common(1) | |
145 | lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1) | |
146 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
147 | blr | |
05be1824 JS |
148 | |
149 | /* Call out to bpf_internal_load_pointer_neg_helper: | |
150 | * We'll need to back up our volatile regs first; we have | |
151 | * local variable space at r1+(BPF_PPC_STACK_BASIC). | |
152 | * Allocate a new stack frame here to remain ABI-compliant in | |
153 | * stashing LR. | |
154 | */ | |
155 | #define sk_negative_common(SIZE) \ | |
156 | mflr r0; \ | |
09ca5ab2 | 157 | PPC_STL r0, PPC_LR_STKOFF(r1); \ |
05be1824 | 158 | /* R3 goes in parameter space of caller's frame */ \ |
09ca5ab2 DK |
159 | PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ |
160 | PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ | |
161 | PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ | |
162 | PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ | |
05be1824 JS |
163 | /* R3 = r_skb, as passed */ \ |
164 | mr r4, r_addr; \ | |
165 | li r5, SIZE; \ | |
166 | bl bpf_internal_load_pointer_neg_helper; \ | |
7784655a | 167 | nop; \ |
05be1824 JS |
168 | /* R3 != 0 on success */ \ |
169 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ | |
09ca5ab2 DK |
170 | PPC_LL r0, PPC_LR_STKOFF(r1); \ |
171 | PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ | |
172 | PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ | |
05be1824 | 173 | mtlr r0; \ |
09ca5ab2 | 174 | PPC_LCMPLI r3, 0; \ |
05be1824 JS |
175 | beq bpf_error_slow; /* cr0 = EQ */ \ |
176 | mr r_addr, r3; \ | |
09ca5ab2 | 177 | PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ |
05be1824 JS |
178 | /* Great success! */ |
179 | ||
180 | bpf_slow_path_word_neg: | |
181 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
09ca5ab2 | 182 | PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ |
05be1824 JS |
183 | blt bpf_error /* cr0 = LT */ |
184 | .globl sk_load_word_negative_offset | |
185 | sk_load_word_negative_offset: | |
186 | sk_negative_common(4) | |
187 | lwz r_A, 0(r_addr) | |
188 | blr | |
189 | ||
190 | bpf_slow_path_half_neg: | |
191 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
09ca5ab2 | 192 | PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ |
05be1824 JS |
193 | blt bpf_error /* cr0 = LT */ |
194 | .globl sk_load_half_negative_offset | |
195 | sk_load_half_negative_offset: | |
196 | sk_negative_common(2) | |
197 | lhz r_A, 0(r_addr) | |
198 | blr | |
199 | ||
200 | bpf_slow_path_byte_neg: | |
201 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
09ca5ab2 | 202 | PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ |
05be1824 JS |
203 | blt bpf_error /* cr0 = LT */ |
204 | .globl sk_load_byte_negative_offset | |
205 | sk_load_byte_negative_offset: | |
206 | sk_negative_common(1) | |
207 | lbz r_A, 0(r_addr) | |
208 | blr | |
209 | ||
210 | bpf_slow_path_byte_msh_neg: | |
211 | lis r_scratch1,-32 /* SKF_LL_OFF */ | |
09ca5ab2 | 212 | PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ |
05be1824 JS |
213 | blt bpf_error /* cr0 = LT */ |
214 | .globl sk_load_byte_msh_negative_offset | |
215 | sk_load_byte_msh_negative_offset: | |
216 | sk_negative_common(1) | |
217 | lbz r_X, 0(r_addr) | |
218 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 | |
219 | blr | |
220 | ||
221 | bpf_error_slow: | |
222 | /* fabricate a cr0 = lt */ | |
223 | li r_scratch1, -1 | |
09ca5ab2 | 224 | PPC_LCMPI r_scratch1, 0 |
05be1824 JS |
225 | bpf_error: |
226 | /* Entered with cr0 = lt */ | |
227 | li r3, 0 | |
228 | /* Generated code will 'blt epilogue', returning 0. */ | |
229 | blr |