Commit | Line | Data |
---|---|---|
d9f5ab7b JB |
1 | #ifndef _ASM_X86_JUMP_LABEL_H |
2 | #define _ASM_X86_JUMP_LABEL_H | |
3 | ||
c2845433 AL |
4 | #ifndef HAVE_JUMP_LABEL |
5 | /* | |
6 | * For better or for worse, if jump labels (the gcc extension) are missing, | |
7 | * then the entire static branch patching infrastructure is compiled out. | |
8 | * If that happens, the code in here will malfunction. Raise a compiler | |
9 | * error instead. | |
10 | * | |
11 | * In theory, jump labels and the static branch patching infrastructure | |
12 | * could be decoupled to fix this. | |
13 | */ | |
14 | #error asm/jump_label.h included on a non-jump-label kernel | |
15 | #endif | |
16 | ||
d9f5ab7b JB |
17 | #define JUMP_LABEL_NOP_SIZE 5 |
18 | ||
c3c7f14a SR |
19 | #ifdef CONFIG_X86_64 |
20 | # define STATIC_KEY_INIT_NOP P6_NOP5_ATOMIC | |
21 | #else | |
22 | # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC | |
23 | #endif | |
d430d3d7 | 24 | |
2671c3e4 AL |
25 | #include <asm/asm.h> |
26 | #include <asm/nops.h> | |
27 | ||
28 | #ifndef __ASSEMBLY__ | |
29 | ||
30 | #include <linux/stringify.h> | |
31 | #include <linux/types.h> | |
32 | ||
11276d53 | 33 | static __always_inline bool arch_static_branch(struct static_key *key, bool branch) |
d430d3d7 | 34 | { |
3f0116c3 | 35 | asm_volatile_goto("1:" |
c3c7f14a | 36 | ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" |
d430d3d7 | 37 | ".pushsection __jump_table, \"aw\" \n\t" |
ef647894 | 38 | _ASM_ALIGN "\n\t" |
d420acd8 | 39 | _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" |
d430d3d7 | 40 | ".popsection \n\t" |
d420acd8 | 41 | : : "i" (key), "i" (branch) : : l_yes); |
11276d53 PZ |
42 | |
43 | return false; | |
44 | l_yes: | |
45 | return true; | |
46 | } | |
47 | ||
48 | static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) | |
49 | { | |
50 | asm_volatile_goto("1:" | |
51 | ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" | |
52 | "2:\n\t" | |
53 | ".pushsection __jump_table, \"aw\" \n\t" | |
54 | _ASM_ALIGN "\n\t" | |
d420acd8 | 55 | _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" |
11276d53 | 56 | ".popsection \n\t" |
d420acd8 | 57 | : : "i" (key), "i" (branch) : : l_yes); |
11276d53 | 58 | |
d430d3d7 JB |
59 | return false; |
60 | l_yes: | |
61 | return true; | |
62 | } | |
d9f5ab7b | 63 | |
d9f5ab7b | 64 | #ifdef CONFIG_X86_64 |
d9f5ab7b | 65 | typedef u64 jump_label_t; |
d9f5ab7b | 66 | #else |
d9f5ab7b | 67 | typedef u32 jump_label_t; |
95fccd46 | 68 | #endif |
d9f5ab7b JB |
69 | |
70 | struct jump_entry { | |
71 | jump_label_t code; | |
72 | jump_label_t target; | |
73 | jump_label_t key; | |
74 | }; | |
75 | ||
2671c3e4 AL |
76 | #else /* __ASSEMBLY__ */ |
77 | ||
78 | .macro STATIC_JUMP_IF_TRUE target, key, def | |
79 | .Lstatic_jump_\@: | |
80 | .if \def | |
81 | /* Equivalent to "jmp.d32 \target" */ | |
82 | .byte 0xe9 | |
83 | .long \target - .Lstatic_jump_after_\@ | |
84 | .Lstatic_jump_after_\@: | |
85 | .else | |
86 | .byte STATIC_KEY_INIT_NOP | |
87 | .endif | |
88 | .pushsection __jump_table, "aw" | |
89 | _ASM_ALIGN | |
90 | _ASM_PTR .Lstatic_jump_\@, \target, \key | |
91 | .popsection | |
92 | .endm | |
93 | ||
94 | .macro STATIC_JUMP_IF_FALSE target, key, def | |
95 | .Lstatic_jump_\@: | |
96 | .if \def | |
97 | .byte STATIC_KEY_INIT_NOP | |
98 | .else | |
99 | /* Equivalent to "jmp.d32 \target" */ | |
100 | .byte 0xe9 | |
101 | .long \target - .Lstatic_jump_after_\@ | |
102 | .Lstatic_jump_after_\@: | |
103 | .endif | |
104 | .pushsection __jump_table, "aw" | |
105 | _ASM_ALIGN | |
106 | _ASM_PTR .Lstatic_jump_\@, \target, \key + 1 | |
107 | .popsection | |
108 | .endm | |
109 | ||
110 | #endif /* __ASSEMBLY__ */ | |
111 | ||
d9f5ab7b | 112 | #endif |