include/rseq/arch/mips.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 MIPS Tech LLC */
   3 /* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * Author: Paul Burton <paul.burton@mips.com>
   7  */
   8
   9 #ifndef _RSEQ_RSEQ_H
  10 #error "Never use <rseq/arch/mips.h> directly; include <rseq/rseq.h> instead."
  11 #endif
  12
  13 #include <asm/byteorder.h>
  14
  15 /*
  16  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  17  * are not part of the public API.
  18  */
  19
  20 #if (RSEQ_BITS_PER_LONG != 64) && (RSEQ_BITS_PER_LONG != 32)
  21 # error unsupported RSEQ_BITS_PER_LONG
  22 #endif
  23
  24 /*
  25  * RSEQ_SIG uses the break instruction. The instruction pattern is:
  26  *
  27  * On MIPS:
  28  *      0350000d        break     0x350
  29  *
  30  * On nanoMIPS:
  31  *      00100350        break     0x350
  32  *
  33  * On microMIPS:
  34  *      0000d407        break     0x350
  35  *
  36  * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
  37  * halfwords, so the signature halfwords need to be swapped accordingly for
  38  * little-endian.
  39  */
  40 #if defined(__nanomips__)
  41 # ifdef __MIPSEL__
  42 #  define RSEQ_SIG      0x03500010
  43 # else
  44 #  define RSEQ_SIG      0x00100350
  45 # endif
  46 #elif defined(__mips_micromips)
  47 # ifdef __MIPSEL__
  48 #  define RSEQ_SIG      0xd4070000
  49 # else
  50 #  define RSEQ_SIG      0x0000d407
  51 # endif
  52 #elif defined(__mips__)
  53 # define RSEQ_SIG       0x0350000d
  54 #else
  55 /* Unknown MIPS architecture. */
  56 #endif
  57
  58 /*
  59  * Refer to the Linux kernel memory model (LKMM) for documentation of
  60  * the memory barriers.
  61  */
  62
  63 /* CPU memory barrier. */
  64 #define rseq_smp_mb()   __asm__ __volatile__ ("sync" ::: "memory")
  65 /* CPU read memory barrier */
  66 #define rseq_smp_rmb()  rseq_smp_mb()
  67 /* CPU write memory barrier */
  68 #define rseq_smp_wmb()  rseq_smp_mb()
  69
  70 /* Acquire: One-way permeable barrier. */
  71 #define rseq_smp_load_acquire(p)                                        \
  72 __extension__ ({                                                        \
  73         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  74         rseq_smp_mb();                                                  \
  75         ____p1;                                                         \
  76 })
  77
  78 /* Acquire barrier after control dependency. */
  79 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  80
  81 /* Release: One-way permeable barrier. */
  82 #define rseq_smp_store_release(p, v)                                    \
  83 do {                                                                    \
  84         rseq_smp_mb();                                                  \
  85         RSEQ_WRITE_ONCE(*(p), v);                                       \
  86 } while (0)
  87
  88 /*
  89  * Helper macros to define and access a variable of long integer type.
  90  * Only used internally in rseq headers.
  91  */
  92 #if RSEQ_BITS_PER_LONG == 64
  93 # define RSEQ_ASM_LONG                  ".dword"
  94 # define RSEQ_ASM_LONG_LA               "dla"
  95 # define RSEQ_ASM_LONG_L                "ld"
  96 # define RSEQ_ASM_LONG_S                "sd"
  97 # define RSEQ_ASM_LONG_ADDI             "daddiu"
  98 #else
  99 # define RSEQ_ASM_LONG                  ".word"
 100 # define RSEQ_ASM_LONG_LA               "la"
 101 # define RSEQ_ASM_LONG_L                "lw"
 102 # define RSEQ_ASM_LONG_S                "sw"
 103 # define RSEQ_ASM_LONG_ADDI             "addiu"
 104 #endif
 105
 106 /*
 107  * Helper macros to define a variable of pointer type stored in a 64-bit
 108  * integer. Only used internally in rseq headers.
 109  */
 110 #if RSEQ_BITS_PER_LONG == 64
 111 # define RSEQ_ASM_U64_PTR(x)            ".dword " x
 112 #else
 113 # if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN)
 114 #  define RSEQ_ASM_U64_PTR(x)           ".word 0x0, " x
 115 # else
 116 #  define RSEQ_ASM_U64_PTR(x)           ".word " x ", 0x0"
 117 # endif
 118 #endif
 119
 120 #define RSEQ_ASM_U32(x)                 ".word " x
 121
 122 /* Common architecture support macros. */
 123 #include "rseq/arch/generic/common.h"
 124
 125 /* Only used in RSEQ_ASM_DEFINE_ABORT. */
 126 #define __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
 127                                 table_label, version, flags, \
 128                                 start_ip, post_commit_offset, abort_ip) \
 129                 ".balign 32\n\t" \
 130                 __rseq_str(table_label) ":\n\t" \
 131                 RSEQ_ASM_U32(__rseq_str(version)) "\n\t" \
 132                 RSEQ_ASM_U32(__rseq_str(flags)) "\n\t" \
 133                 RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t" \
 134                 RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t" \
 135                 RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t" \
 136                 RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t" \
 137                 __rseq_str(label) ":\n\t" \
 138                 teardown \
 139                 "b %l[" __rseq_str(abort_label) "]\n\t"
 140
 141 /*
 142  * Define a critical section abort handler.
 143  *
 144  *  @label:
 145  *    Local label to the abort handler.
 146  *  @teardown:
 147  *    Sequence of instructions to run on abort.
 148  *  @abort_label:
 149  *    C label to jump to at the end of the sequence.
 150  *  @table_label:
 151  *    Local label to the critical section descriptor copy placed near
 152  *    the program counter. This is done for performance reasons because
 153  *    computing this address is faster than accessing the program data.
 154  *
 155  * The purpose of @start_ip, @post_commit_ip, and @abort_ip are
 156  * documented in RSEQ_ASM_DEFINE_TABLE.
 157  */
 158 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
 159                               table_label, start_ip, post_commit_ip, abort_ip) \
 160         __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
 161                                 table_label, 0x0, 0x0, start_ip, \
 162                                 (post_commit_ip) - (start_ip), abort_ip)
 163
 164 /*
 165  * Define a critical section teardown handler.
 166  *
 167  *  @label:
 168  *    Local label to the teardown handler.
 169  *  @teardown:
 170  *    Sequence of instructions to run on teardown.
 171  *  @target_label:
 172  *    C label to jump to at the end of the sequence.
 173  */
 174 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
 175                 __rseq_str(label) ":\n\t" \
 176                 teardown \
 177                 "b %l[" __rseq_str(target_label) "]\n\t"
 178
 179 /*
 180  * Store the address of the critical section descriptor structure at
 181  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 182  * is the beginning of the sequence of consecutive assembly instructions.
 183  *
 184  *  @label:
 185  *    Local label to the beginning of the sequence of consecutive assembly
 186  *    instructions.
 187  *  @cs_label:
 188  *    Source local label to the critical section descriptor structure.
 189  *  @rseq_cs:
 190  *    Destination pointer where to store the address of the critical
 191  *    section descriptor structure.
 192  */
 193 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
 194                 RSEQ_INJECT_ASM(1) \
 195                 RSEQ_ASM_LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
 196                 RSEQ_ASM_LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
 197                 __rseq_str(label) ":\n\t"
 198
 199 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 200 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
 201                 RSEQ_INJECT_ASM(2) \
 202                 "lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
 203                 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
 204
 205 /* Per-cpu-id indexing. */
 206
 207 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 208 #define RSEQ_TEMPLATE_MO_RELAXED
 209 #include "rseq/arch/mips/bits.h"
 210 #undef RSEQ_TEMPLATE_MO_RELAXED
 211
 212 #define RSEQ_TEMPLATE_MO_RELEASE
 213 #include "rseq/arch/mips/bits.h"
 214 #undef RSEQ_TEMPLATE_MO_RELEASE
 215 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 216
 217 /* Per-mm-cid indexing. */
 218
 219 #define RSEQ_TEMPLATE_INDEX_MM_CID
 220 #define RSEQ_TEMPLATE_MO_RELAXED
 221 #include "rseq/arch/mips/bits.h"
 222 #undef RSEQ_TEMPLATE_MO_RELAXED
 223
 224 #define RSEQ_TEMPLATE_MO_RELEASE
 225 #include "rseq/arch/mips/bits.h"
 226 #undef RSEQ_TEMPLATE_MO_RELEASE
 227 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 228
 229 /* APIs which are not indexed. */
 230
 231 #define RSEQ_TEMPLATE_INDEX_NONE
 232 #define RSEQ_TEMPLATE_MO_RELAXED
 233 #include "rseq/arch/mips/bits.h"
 234 #undef RSEQ_TEMPLATE_MO_RELAXED
 235 #undef RSEQ_TEMPLATE_INDEX_NONE