arch/powerpc/net/bpf_jit_64.S

   1 /* bpf_jit.S: Packet/header access helper functions
   2  * for PPC64 BPF compiler.
   3  *
   4  * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; version 2
   9  * of the License.
  10  */
  11
  12 #include <asm/ppc_asm.h>
  13 #include "bpf_jit.h"
  14
  15 /*
  16  * All of these routines are called directly from generated code,
  17  * whose register usage is:
  18  *
  19  * r3           skb
  20  * r4,r5        A,X
  21  * r6           *** address parameter to helper ***
  22  * r7-r10       scratch
  23  * r14          skb->data
  24  * r15          skb headlen
  25  * r16-31       M[]
  26  */
  27
  28 /*
  29  * To consider: These helpers are so small it could be better to just
  30  * generate them inline.  Inline code can do the simple headlen check
  31  * then branch directly to slow_path_XXX if required.  (In fact, could
  32  * load a spare GPR with the address of slow_path_generic and pass size
  33  * as an argument, making the call site a mtlr, li and bllr.)
  34  */
  35         .globl  sk_load_word
  36 sk_load_word:
  37         cmpdi   r_addr, 0
  38         blt     bpf_slow_path_word_neg
  39         .globl  sk_load_word_positive_offset
  40 sk_load_word_positive_offset:
  41         /* Are we accessing past headlen? */
  42         subi    r_scratch1, r_HL, 4
  43         cmpd    r_scratch1, r_addr
  44         blt     bpf_slow_path_word
  45         /* Nope, just hitting the header.  cr0 here is eq or gt! */
  46         lwzx    r_A, r_D, r_addr
  47         /* When big endian we don't need to byteswap. */
  48         blr     /* Return success, cr0 != LT */
  49
  50         .globl  sk_load_half
  51 sk_load_half:
  52         cmpdi   r_addr, 0
  53         blt     bpf_slow_path_half_neg
  54         .globl  sk_load_half_positive_offset
  55 sk_load_half_positive_offset:
  56         subi    r_scratch1, r_HL, 2
  57         cmpd    r_scratch1, r_addr
  58         blt     bpf_slow_path_half
  59         lhzx    r_A, r_D, r_addr
  60         blr
  61
  62         .globl  sk_load_byte
  63 sk_load_byte:
  64         cmpdi   r_addr, 0
  65         blt     bpf_slow_path_byte_neg
  66         .globl  sk_load_byte_positive_offset
  67 sk_load_byte_positive_offset:
  68         cmpd    r_HL, r_addr
  69         ble     bpf_slow_path_byte
  70         lbzx    r_A, r_D, r_addr
  71         blr
  72
  73 /*
  74  * BPF_S_LDX_B_MSH: ldxb  4*([offset]&0xf)
  75  * r_addr is the offset value
  76  */
  77         .globl sk_load_byte_msh
  78 sk_load_byte_msh:
  79         cmpdi   r_addr, 0
  80         blt     bpf_slow_path_byte_msh_neg
  81         .globl sk_load_byte_msh_positive_offset
  82 sk_load_byte_msh_positive_offset:
  83         cmpd    r_HL, r_addr
  84         ble     bpf_slow_path_byte_msh
  85         lbzx    r_X, r_D, r_addr
  86         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
  87         blr
  88
  89 /* Call out to skb_copy_bits:
  90  * We'll need to back up our volatile regs first; we have
  91  * local variable space at r1+(BPF_PPC_STACK_BASIC).
  92  * Allocate a new stack frame here to remain ABI-compliant in
  93  * stashing LR.
  94  */
  95 #define bpf_slow_path_common(SIZE)                              \
  96         mflr    r0;                                             \
  97         std     r0, 16(r1);                                     \
  98         /* R3 goes in parameter space of caller's frame */      \
  99         std     r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 100         std     r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 101         std     r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 102         addi    r5, r1, BPF_PPC_STACK_BASIC+(2*8);              \
 103         stdu    r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 104         /* R3 = r_skb, as passed */                             \
 105         mr      r4, r_addr;                                     \
 106         li      r6, SIZE;                                       \
 107         bl      skb_copy_bits;                                  \
 108         /* R3 = 0 on success */                                 \
 109         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 110         ld      r0, 16(r1);                                     \
 111         ld      r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 112         ld      r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 113         mtlr    r0;                                             \
 114         cmpdi   r3, 0;                                          \
 115         blt     bpf_error;      /* cr0 = LT */                  \
 116         ld      r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 117         /* Great success! */
 118
 119 bpf_slow_path_word:
 120         bpf_slow_path_common(4)
 121         /* Data value is on stack, and cr0 != LT */
 122         lwz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 123         blr
 124
 125 bpf_slow_path_half:
 126         bpf_slow_path_common(2)
 127         lhz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 128         blr
 129
 130 bpf_slow_path_byte:
 131         bpf_slow_path_common(1)
 132         lbz     r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
 133         blr
 134
 135 bpf_slow_path_byte_msh:
 136         bpf_slow_path_common(1)
 137         lbz     r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
 138         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 139         blr
 140
 141 /* Call out to bpf_internal_load_pointer_neg_helper:
 142  * We'll need to back up our volatile regs first; we have
 143  * local variable space at r1+(BPF_PPC_STACK_BASIC).
 144  * Allocate a new stack frame here to remain ABI-compliant in
 145  * stashing LR.
 146  */
 147 #define sk_negative_common(SIZE)                                \
 148         mflr    r0;                                             \
 149         std     r0, 16(r1);                                     \
 150         /* R3 goes in parameter space of caller's frame */      \
 151         std     r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 152         std     r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 153         std     r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 154         stdu    r1, -BPF_PPC_SLOWPATH_FRAME(r1);                \
 155         /* R3 = r_skb, as passed */                             \
 156         mr      r4, r_addr;                                     \
 157         li      r5, SIZE;                                       \
 158         bl      bpf_internal_load_pointer_neg_helper;           \
 159         /* R3 != 0 on success */                                \
 160         addi    r1, r1, BPF_PPC_SLOWPATH_FRAME;                 \
 161         ld      r0, 16(r1);                                     \
 162         ld      r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1);           \
 163         ld      r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1);           \
 164         mtlr    r0;                                             \
 165         cmpldi  r3, 0;                                          \
 166         beq     bpf_error_slow; /* cr0 = EQ */                  \
 167         mr      r_addr, r3;                                     \
 168         ld      r_skb, (BPF_PPC_STACKFRAME+48)(r1);             \
 169         /* Great success! */
 170
 171 bpf_slow_path_word_neg:
 172         lis     r_scratch1,-32  /* SKF_LL_OFF */
 173         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 174         blt     bpf_error       /* cr0 = LT */
 175         .globl  sk_load_word_negative_offset
 176 sk_load_word_negative_offset:
 177         sk_negative_common(4)
 178         lwz     r_A, 0(r_addr)
 179         blr
 180
 181 bpf_slow_path_half_neg:
 182         lis     r_scratch1,-32  /* SKF_LL_OFF */
 183         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 184         blt     bpf_error       /* cr0 = LT */
 185         .globl  sk_load_half_negative_offset
 186 sk_load_half_negative_offset:
 187         sk_negative_common(2)
 188         lhz     r_A, 0(r_addr)
 189         blr
 190
 191 bpf_slow_path_byte_neg:
 192         lis     r_scratch1,-32  /* SKF_LL_OFF */
 193         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 194         blt     bpf_error       /* cr0 = LT */
 195         .globl  sk_load_byte_negative_offset
 196 sk_load_byte_negative_offset:
 197         sk_negative_common(1)
 198         lbz     r_A, 0(r_addr)
 199         blr
 200
 201 bpf_slow_path_byte_msh_neg:
 202         lis     r_scratch1,-32  /* SKF_LL_OFF */
 203         cmpd    r_addr, r_scratch1      /* addr < SKF_* */
 204         blt     bpf_error       /* cr0 = LT */
 205         .globl  sk_load_byte_msh_negative_offset
 206 sk_load_byte_msh_negative_offset:
 207         sk_negative_common(1)
 208         lbz     r_X, 0(r_addr)
 209         rlwinm  r_X, r_X, 2, 32-4-2, 31-2
 210         blr
 211
 212 bpf_error_slow:
 213         /* fabricate a cr0 = lt */
 214         li      r_scratch1, -1
 215         cmpdi   r_scratch1, 0
 216 bpf_error:
 217         /* Entered with cr0 = lt */
 218         li      r3, 0
 219         /* Generated code will 'blt epilogue', returning 0. */
 220         blr