arch/arm64/kernel/entry-ftrace.S

   1 /*
   2  * arch/arm64/kernel/entry-ftrace.S
   3  *
   4  * Copyright (C) 2013 Linaro Limited
   5  * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/linkage.h>
  13 #include <asm/ftrace.h>
  14 #include <asm/insn.h>
  15
  16 /*
  17  * Gcc with -pg will put the following code in the beginning of each function:
  18  *      mov x0, x30
  19  *      bl _mcount
  20  *      [function's body ...]
  21  * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
  22  * ftrace is enabled.
  23  *
  24  * Please note that x0 as an argument will not be used here because we can
  25  * get lr(x30) of instrumented function at any time by winding up call stack
  26  * as long as the kernel is compiled without -fomit-frame-pointer.
  27  * (or CONFIG_FRAME_POINTER, this is forced on arm64)
  28  *
  29  * stack layout after mcount_enter in _mcount():
  30  *
  31  * current sp/fp =>  0:+-----+
  32  * in _mcount()        | x29 | -> instrumented function's fp
  33  *                     +-----+
  34  *                     | x30 | -> _mcount()'s lr (= instrumented function's pc)
  35  * old sp       => +16:+-----+
  36  * when instrumented   |     |
  37  * function calls      | ... |
  38  * _mcount()           |     |
  39  *                     |     |
  40  * instrumented => +xx:+-----+
  41  * function's fp       | x29 | -> parent's fp
  42  *                     +-----+
  43  *                     | x30 | -> instrumented function's lr (= parent's pc)
  44  *                     +-----+
  45  *                     | ... |
  46  */
  47
  48         .macro mcount_enter
  49         stp     x29, x30, [sp, #-16]!
  50         mov     x29, sp
  51         .endm
  52
  53         .macro mcount_exit
  54         ldp     x29, x30, [sp], #16
  55         ret
  56         .endm
  57
  58         .macro mcount_adjust_addr rd, rn
  59         sub     \rd, \rn, #AARCH64_INSN_SIZE
  60         .endm
  61
  62         /* for instrumented function's parent */
  63         .macro mcount_get_parent_fp reg
  64         ldr     \reg, [x29]
  65         ldr     \reg, [\reg]
  66         .endm
  67
  68         /* for instrumented function */
  69         .macro mcount_get_pc0 reg
  70         mcount_adjust_addr      \reg, x30
  71         .endm
  72
  73         .macro mcount_get_pc reg
  74         ldr     \reg, [x29, #8]
  75         mcount_adjust_addr      \reg, \reg
  76         .endm
  77
  78         .macro mcount_get_lr reg
  79         ldr     \reg, [x29]
  80         ldr     \reg, [\reg, #8]
  81         mcount_adjust_addr      \reg, \reg
  82         .endm
  83
  84         .macro mcount_get_lr_addr reg
  85         ldr     \reg, [x29]
  86         add     \reg, \reg, #8
  87         .endm
  88
  89 #ifndef CONFIG_DYNAMIC_FTRACE
  90 /*
  91  * void _mcount(unsigned long return_address)
  92  * @return_address: return address to instrumented function
  93  *
  94  * This function makes calls, if enabled, to:
  95  *     - tracer function to probe instrumented function's entry,
  96  *     - ftrace_graph_caller to set up an exit hook
  97  */
  98 ENTRY(_mcount)
  99 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 100         ldr     x0, =ftrace_trace_stop
 101         ldr     x0, [x0]                // if ftrace_trace_stop
 102         ret                             //   return;
 103 #endif
 104         mcount_enter
 105
 106         ldr     x0, =ftrace_trace_function
 107         ldr     x2, [x0]
 108         adr     x0, ftrace_stub
 109         cmp     x0, x2                  // if (ftrace_trace_function
 110         b.eq    skip_ftrace_call        //     != ftrace_stub) {
 111
 112         mcount_get_pc   x0              //       function's pc
 113         mcount_get_lr   x1              //       function's lr (= parent's pc)
 114         blr     x2                      //   (*ftrace_trace_function)(pc, lr);
 115
 116 #ifndef CONFIG_FUNCTION_GRAPH_TRACER
 117 skip_ftrace_call:                       //   return;
 118         mcount_exit                     // }
 119 #else
 120         mcount_exit                     //   return;
 121                                         // }
 122 skip_ftrace_call:
 123         ldr     x1, =ftrace_graph_return
 124         ldr     x2, [x1]                //   if ((ftrace_graph_return
 125         cmp     x0, x2                  //        != ftrace_stub)
 126         b.ne    ftrace_graph_caller
 127
 128         ldr     x1, =ftrace_graph_entry //     || (ftrace_graph_entry
 129         ldr     x2, [x1]                //        != ftrace_graph_entry_stub))
 130         ldr     x0, =ftrace_graph_entry_stub
 131         cmp     x0, x2
 132         b.ne    ftrace_graph_caller     //     ftrace_graph_caller();
 133
 134         mcount_exit
 135 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 136 ENDPROC(_mcount)
 137
 138 #else /* CONFIG_DYNAMIC_FTRACE */
 139 /*
 140  * _mcount() is used to build the kernel with -pg option, but all the branch
 141  * instructions to _mcount() are replaced to NOP initially at kernel start up,
 142  * and later on, NOP to branch to ftrace_caller() when enabled or branch to
 143  * NOP when disabled per-function base.
 144  */
 145 ENTRY(_mcount)
 146         ret
 147 ENDPROC(_mcount)
 148
 149 /*
 150  * void ftrace_caller(unsigned long return_address)
 151  * @return_address: return address to instrumented function
 152  *
 153  * This function is a counterpart of _mcount() in 'static' ftrace, and
 154  * makes calls to:
 155  *     - tracer function to probe instrumented function's entry,
 156  *     - ftrace_graph_caller to set up an exit hook
 157  */
 158 ENTRY(ftrace_caller)
 159         mcount_enter
 160
 161         mcount_get_pc0  x0              //     function's pc
 162         mcount_get_lr   x1              //     function's lr
 163
 164         .global ftrace_call
 165 ftrace_call:                            // tracer(pc, lr);
 166         nop                             // This will be replaced with "bl xxx"
 167                                         // where xxx can be any kind of tracer.
 168
 169 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 170         .global ftrace_graph_call
 171 ftrace_graph_call:                      // ftrace_graph_caller();
 172         nop                             // If enabled, this will be replaced
 173                                         // "b ftrace_graph_caller"
 174 #endif
 175
 176         mcount_exit
 177 ENDPROC(ftrace_caller)
 178 #endif /* CONFIG_DYNAMIC_FTRACE */
 179
 180 ENTRY(ftrace_stub)
 181         ret
 182 ENDPROC(ftrace_stub)
 183
 184 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 185 /*
 186  * void ftrace_graph_caller(void)
 187  *
 188  * Called from _mcount() or ftrace_caller() when function_graph tracer is
 189  * selected.
 190  * This function w/ prepare_ftrace_return() fakes link register's value on
 191  * the call stack in order to intercept instrumented function's return path
 192  * and run return_to_handler() later on its exit.
 193  */
 194 ENTRY(ftrace_graph_caller)
 195         mcount_get_lr_addr        x0    //     pointer to function's saved lr
 196         mcount_get_pc             x1    //     function's pc
 197         mcount_get_parent_fp      x2    //     parent's fp
 198         bl      prepare_ftrace_return   // prepare_ftrace_return(&lr, pc, fp)
 199
 200         mcount_exit
 201 ENDPROC(ftrace_graph_caller)
 202
 203 /*
 204  * void return_to_handler(void)
 205  *
 206  * Run ftrace_return_to_handler() before going back to parent.
 207  * @fp is checked against the value passed by ftrace_graph_caller()
 208  * only when CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST is enabled.
 209  */
 210 ENTRY(return_to_handler)
 211         str     x0, [sp, #-16]!
 212         mov     x0, x29                 //     parent's fp
 213         bl      ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
 214         mov     x30, x0                 // restore the original return address
 215         ldr     x0, [sp], #16
 216         ret
 217 END(return_to_handler)
 218 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */