Commit | Line | Data |
---|---|---|
0a14842f ED |
1 | /* bpf_jit.S : BPF JIT helper functions |
2 | * | |
3 | * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com) | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or | |
6 | * modify it under the terms of the GNU General Public License | |
7 | * as published by the Free Software Foundation; version 2 | |
8 | * of the License. | |
9 | */ | |
10 | #include <linux/linkage.h> | |
11 | #include <asm/dwarf2.h> | |
12 | ||
13 | /* | |
14 | * Calling convention : | |
62258278 | 15 | * rbx : skb pointer (callee saved) |
0a14842f | 16 | * esi : offset of byte(s) to fetch in skb (can be scratched) |
62258278 | 17 | * r10 : copy of skb->data |
0a14842f ED |
18 | * r9d : hlen = skb->len - skb->data_len |
19 | */ | |
62258278 | 20 | #define SKBDATA %r10 |
a998d434 | 21 | #define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */ |
62258278 AS |
22 | #define MAX_BPF_STACK (512 /* from filter.h */ + \ |
23 | 32 /* space for rbx,r13,r14,r15 */ + \ | |
24 | 8 /* space for skb_copy_bits */) | |
0a14842f ED |
25 | |
26 | sk_load_word: | |
27 | .globl sk_load_word | |
28 | ||
a998d434 JS |
29 | test %esi,%esi |
30 | js bpf_slow_path_word_neg | |
31 | ||
32 | sk_load_word_positive_offset: | |
33 | .globl sk_load_word_positive_offset | |
34 | ||
0a14842f ED |
35 | mov %r9d,%eax # hlen |
36 | sub %esi,%eax # hlen - offset | |
37 | cmp $3,%eax | |
38 | jle bpf_slow_path_word | |
39 | mov (SKBDATA,%rsi),%eax | |
40 | bswap %eax /* ntohl() */ | |
41 | ret | |
42 | ||
0a14842f ED |
43 | sk_load_half: |
44 | .globl sk_load_half | |
45 | ||
a998d434 JS |
46 | test %esi,%esi |
47 | js bpf_slow_path_half_neg | |
48 | ||
49 | sk_load_half_positive_offset: | |
50 | .globl sk_load_half_positive_offset | |
51 | ||
0a14842f ED |
52 | mov %r9d,%eax |
53 | sub %esi,%eax # hlen - offset | |
54 | cmp $1,%eax | |
55 | jle bpf_slow_path_half | |
56 | movzwl (SKBDATA,%rsi),%eax | |
57 | rol $8,%ax # ntohs() | |
58 | ret | |
59 | ||
0a14842f ED |
60 | sk_load_byte: |
61 | .globl sk_load_byte | |
62 | ||
a998d434 JS |
63 | test %esi,%esi |
64 | js bpf_slow_path_byte_neg | |
65 | ||
66 | sk_load_byte_positive_offset: | |
67 | .globl sk_load_byte_positive_offset | |
68 | ||
0a14842f ED |
69 | cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ |
70 | jle bpf_slow_path_byte | |
71 | movzbl (SKBDATA,%rsi),%eax | |
72 | ret | |
73 | ||
0a14842f ED |
74 | /* rsi contains offset and can be scratched */ |
75 | #define bpf_slow_path_common(LEN) \ | |
62258278 | 76 | mov %rbx, %rdi; /* arg1 == skb */ \ |
0a14842f ED |
77 | push %r9; \ |
78 | push SKBDATA; \ | |
79 | /* rsi already has offset */ \ | |
80 | mov $LEN,%ecx; /* len */ \ | |
62258278 | 81 | lea - MAX_BPF_STACK + 32(%rbp),%rdx; \ |
0a14842f ED |
82 | call skb_copy_bits; \ |
83 | test %eax,%eax; \ | |
84 | pop SKBDATA; \ | |
62258278 | 85 | pop %r9; |
0a14842f ED |
86 | |
87 | ||
88 | bpf_slow_path_word: | |
89 | bpf_slow_path_common(4) | |
90 | js bpf_error | |
62258278 | 91 | mov - MAX_BPF_STACK + 32(%rbp),%eax |
0a14842f ED |
92 | bswap %eax |
93 | ret | |
94 | ||
95 | bpf_slow_path_half: | |
96 | bpf_slow_path_common(2) | |
97 | js bpf_error | |
62258278 | 98 | mov - MAX_BPF_STACK + 32(%rbp),%ax |
0a14842f ED |
99 | rol $8,%ax |
100 | movzwl %ax,%eax | |
101 | ret | |
102 | ||
103 | bpf_slow_path_byte: | |
104 | bpf_slow_path_common(1) | |
105 | js bpf_error | |
62258278 | 106 | movzbl - MAX_BPF_STACK + 32(%rbp),%eax |
0a14842f | 107 | ret |
a998d434 JS |
108 | |
109 | #define sk_negative_common(SIZE) \ | |
62258278 | 110 | mov %rbx, %rdi; /* arg1 == skb */ \ |
a998d434 JS |
111 | push %r9; \ |
112 | push SKBDATA; \ | |
113 | /* rsi already has offset */ \ | |
fdfaf64e | 114 | mov $SIZE,%edx; /* size */ \ |
a998d434 JS |
115 | call bpf_internal_load_pointer_neg_helper; \ |
116 | test %rax,%rax; \ | |
117 | pop SKBDATA; \ | |
118 | pop %r9; \ | |
a998d434 JS |
119 | jz bpf_error |
120 | ||
a998d434 JS |
121 | bpf_slow_path_word_neg: |
122 | cmp SKF_MAX_NEG_OFF, %esi /* test range */ | |
123 | jl bpf_error /* offset lower -> error */ | |
124 | sk_load_word_negative_offset: | |
125 | .globl sk_load_word_negative_offset | |
126 | sk_negative_common(4) | |
127 | mov (%rax), %eax | |
128 | bswap %eax | |
129 | ret | |
130 | ||
131 | bpf_slow_path_half_neg: | |
132 | cmp SKF_MAX_NEG_OFF, %esi | |
133 | jl bpf_error | |
134 | sk_load_half_negative_offset: | |
135 | .globl sk_load_half_negative_offset | |
136 | sk_negative_common(2) | |
137 | mov (%rax),%ax | |
138 | rol $8,%ax | |
139 | movzwl %ax,%eax | |
140 | ret | |
141 | ||
142 | bpf_slow_path_byte_neg: | |
143 | cmp SKF_MAX_NEG_OFF, %esi | |
144 | jl bpf_error | |
145 | sk_load_byte_negative_offset: | |
146 | .globl sk_load_byte_negative_offset | |
147 | sk_negative_common(1) | |
148 | movzbl (%rax), %eax | |
149 | ret | |
150 | ||
a998d434 JS |
151 | bpf_error: |
152 | # force a return 0 from jit handler | |
62258278 AS |
153 | xor %eax,%eax |
154 | mov - MAX_BPF_STACK(%rbp),%rbx | |
155 | mov - MAX_BPF_STACK + 8(%rbp),%r13 | |
156 | mov - MAX_BPF_STACK + 16(%rbp),%r14 | |
157 | mov - MAX_BPF_STACK + 24(%rbp),%r15 | |
a998d434 JS |
158 | leaveq |
159 | ret |