Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net

[deliverable/linux.git] / arch / x86 / net / bpf_jit_comp.c

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index be2e7a2b10d7169b39b0849703b305ebb52c8933..ec5214f39aa802ed923d10315a1672547493262d 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -315,6 +315,26 @@ static void emit_bpf_tail_call(u8 **pprog)
        *pprog = prog;
 }
 
+
+static void emit_load_skb_data_hlen(u8 **pprog)
+{
+       u8 *prog = *pprog;
+       int cnt = 0;
+
+       /* r9d = skb->len - skb->data_len (headlen)
+        * r10 = skb->data
+        */
+       /* mov %r9d, off32(%rdi) */
+       EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len));
+
+       /* sub %r9d, off32(%rdi) */
+       EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len));
+
+       /* mov %r10, off32(%rdi) */
+       EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data));
+       *pprog = prog;
+}
+
 static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                  int oldproglen, struct jit_context *ctx)
 {
@@ -329,36 +349,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
 
        emit_prologue(&prog);
 
-       if (seen_ld_abs) {
-               /* r9d : skb->len - skb->data_len (headlen)
-                * r10 : skb->data
-                */
-               if (is_imm8(offsetof(struct sk_buff, len)))
-                       /* mov %r9d, off8(%rdi) */
-                       EMIT4(0x44, 0x8b, 0x4f,
-                             offsetof(struct sk_buff, len));
-               else
-                       /* mov %r9d, off32(%rdi) */
-                       EMIT3_off32(0x44, 0x8b, 0x8f,
-                                   offsetof(struct sk_buff, len));
-
-               if (is_imm8(offsetof(struct sk_buff, data_len)))
-                       /* sub %r9d, off8(%rdi) */
-                       EMIT4(0x44, 0x2b, 0x4f,
-                             offsetof(struct sk_buff, data_len));
-               else
-                       EMIT3_off32(0x44, 0x2b, 0x8f,
-                                   offsetof(struct sk_buff, data_len));
-
-               if (is_imm8(offsetof(struct sk_buff, data)))
-                       /* mov %r10, off8(%rdi) */
-                       EMIT4(0x4c, 0x8b, 0x57,
-                             offsetof(struct sk_buff, data));
-               else
-                       /* mov %r10, off32(%rdi) */
-                       EMIT3_off32(0x4c, 0x8b, 0x97,
-                                   offsetof(struct sk_buff, data));
-       }
+       if (seen_ld_abs)
+               emit_load_skb_data_hlen(&prog);
 
        for (i = 0; i < insn_cnt; i++, insn++) {
                const s32 imm32 = insn->imm;
@@ -367,6 +359,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
                u8 b1 = 0, b2 = 0, b3 = 0;
                s64 jmp_offset;
                u8 jmp_cond;
+               bool reload_skb_data;
                int ilen;
                u8 *func;
 
@@ -818,12 +811,18 @@ xadd:                     if (is_imm8(insn->off))
                        func = (u8 *) __bpf_call_base + imm32;
                        jmp_offset = func - (image + addrs[i]);
                        if (seen_ld_abs) {
-                               EMIT2(0x41, 0x52); /* push %r10 */
-                               EMIT2(0x41, 0x51); /* push %r9 */
-                               /* need to adjust jmp offset, since
-                                * pop %r9, pop %r10 take 4 bytes after call insn
-                                */
-                               jmp_offset += 4;
+                               reload_skb_data = bpf_helper_changes_skb_data(func);
+                               if (reload_skb_data) {
+                                       EMIT1(0x57); /* push %rdi */
+                                       jmp_offset += 22; /* pop, mov, sub, mov */
+                               } else {
+                                       EMIT2(0x41, 0x52); /* push %r10 */
+                                       EMIT2(0x41, 0x51); /* push %r9 */
+                                       /* need to adjust jmp offset, since
+                                        * pop %r9, pop %r10 take 4 bytes after call insn
+                                        */
+                                       jmp_offset += 4;
+                               }
                        }
                        if (!imm32 || !is_simm32(jmp_offset)) {
                                pr_err("unsupported bpf func %d addr %p image %p\n",
@@ -832,8 +831,13 @@ xadd:                      if (is_imm8(insn->off))
                        }
                        EMIT1_off32(0xE8, jmp_offset);
                        if (seen_ld_abs) {
-                               EMIT2(0x41, 0x59); /* pop %r9 */
-                               EMIT2(0x41, 0x5A); /* pop %r10 */
+                               if (reload_skb_data) {
+                                       EMIT1(0x5F); /* pop %rdi */
+                                       emit_load_skb_data_hlen(&prog);
+                               } else {
+                                       EMIT2(0x41, 0x59); /* pop %r9 */
+                                       EMIT2(0x41, 0x5A); /* pop %r10 */
+                               }
                        }
                        break;
 
@@ -1099,7 +1103,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
        }
 
        if (bpf_jit_enable > 1)
-               bpf_jit_dump(prog->len, proglen, 0, image);
+               bpf_jit_dump(prog->len, proglen, pass + 1, image);
 
        if (image) {
                bpf_flush_icache(header, image + proglen);