2 * Copyright (C) 2014 ARM Limited
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/perf_event.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/sysctl.h>
17 #include <asm/alternative.h>
18 #include <asm/cpufeature.h>
20 #include <asm/opcodes.h>
21 #include <asm/sysreg.h>
22 #include <asm/system_misc.h>
23 #include <asm/traps.h>
24 #include <asm/uaccess.h>
25 #include <asm/cpufeature.h>
27 #define CREATE_TRACE_POINTS
28 #include "trace-events-emulation.h"
31 * The runtime support for deprecated instruction support can be in one of
32 * following three states -
35 * 1 = emulate (software emulation)
36 * 2 = hw (supported in hardware)
38 enum insn_emulation_mode
{
44 enum legacy_insn_status
{
49 struct insn_emulation_ops
{
51 enum legacy_insn_status status
;
52 struct undef_hook
*hooks
;
53 int (*set_hw_mode
)(bool enable
);
56 struct insn_emulation
{
57 struct list_head node
;
58 struct insn_emulation_ops
*ops
;
64 static LIST_HEAD(insn_emulation
);
65 static int nr_insn_emulated __initdata
;
66 static DEFINE_RAW_SPINLOCK(insn_emulation_lock
);
68 static void register_emulation_hooks(struct insn_emulation_ops
*ops
)
70 struct undef_hook
*hook
;
74 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
75 register_undef_hook(hook
);
77 pr_notice("Registered %s emulation handler\n", ops
->name
);
80 static void remove_emulation_hooks(struct insn_emulation_ops
*ops
)
82 struct undef_hook
*hook
;
86 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
87 unregister_undef_hook(hook
);
89 pr_notice("Removed %s emulation handler\n", ops
->name
);
92 static void enable_insn_hw_mode(void *data
)
94 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
95 if (insn
->ops
->set_hw_mode
)
96 insn
->ops
->set_hw_mode(true);
99 static void disable_insn_hw_mode(void *data
)
101 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
102 if (insn
->ops
->set_hw_mode
)
103 insn
->ops
->set_hw_mode(false);
106 /* Run set_hw_mode(mode) on all active CPUs */
107 static int run_all_cpu_set_hw_mode(struct insn_emulation
*insn
, bool enable
)
109 if (!insn
->ops
->set_hw_mode
)
112 on_each_cpu(enable_insn_hw_mode
, (void *)insn
, true);
114 on_each_cpu(disable_insn_hw_mode
, (void *)insn
, true);
119 * Run set_hw_mode for all insns on a starting CPU.
121 * 0 - If all the hooks ran successfully.
122 * -EINVAL - At least one hook is not supported by the CPU.
124 static int run_all_insn_set_hw_mode(unsigned int cpu
)
128 struct insn_emulation
*insn
;
130 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
131 list_for_each_entry(insn
, &insn_emulation
, node
) {
132 bool enable
= (insn
->current_mode
== INSN_HW
);
133 if (insn
->ops
->set_hw_mode
&& insn
->ops
->set_hw_mode(enable
)) {
134 pr_warn("CPU[%u] cannot support the emulation of %s",
135 cpu
, insn
->ops
->name
);
139 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
143 static int update_insn_emulation_mode(struct insn_emulation
*insn
,
144 enum insn_emulation_mode prev
)
149 case INSN_UNDEF
: /* Nothing to be done */
152 remove_emulation_hooks(insn
->ops
);
155 if (!run_all_cpu_set_hw_mode(insn
, false))
156 pr_notice("Disabled %s support\n", insn
->ops
->name
);
160 switch (insn
->current_mode
) {
164 register_emulation_hooks(insn
->ops
);
167 ret
= run_all_cpu_set_hw_mode(insn
, true);
169 pr_notice("Enabled %s support\n", insn
->ops
->name
);
176 static void __init
register_insn_emulation(struct insn_emulation_ops
*ops
)
179 struct insn_emulation
*insn
;
181 insn
= kzalloc(sizeof(*insn
), GFP_KERNEL
);
183 insn
->min
= INSN_UNDEF
;
185 switch (ops
->status
) {
186 case INSN_DEPRECATED
:
187 insn
->current_mode
= INSN_EMULATE
;
188 /* Disable the HW mode if it was turned on at early boot time */
189 run_all_cpu_set_hw_mode(insn
, false);
193 insn
->current_mode
= INSN_UNDEF
;
194 insn
->max
= INSN_EMULATE
;
198 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
199 list_add(&insn
->node
, &insn_emulation
);
201 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
203 /* Register any handlers if required */
204 update_insn_emulation_mode(insn
, INSN_UNDEF
);
207 static int emulation_proc_handler(struct ctl_table
*table
, int write
,
208 void __user
*buffer
, size_t *lenp
,
212 struct insn_emulation
*insn
= (struct insn_emulation
*) table
->data
;
213 enum insn_emulation_mode prev_mode
= insn
->current_mode
;
215 table
->data
= &insn
->current_mode
;
216 ret
= proc_dointvec_minmax(table
, write
, buffer
, lenp
, ppos
);
218 if (ret
|| !write
|| prev_mode
== insn
->current_mode
)
221 ret
= update_insn_emulation_mode(insn
, prev_mode
);
223 /* Mode change failed, revert to previous mode. */
224 insn
->current_mode
= prev_mode
;
225 update_insn_emulation_mode(insn
, INSN_UNDEF
);
232 static struct ctl_table ctl_abi
[] = {
240 static void __init
register_insn_emulation_sysctl(struct ctl_table
*table
)
244 struct insn_emulation
*insn
;
245 struct ctl_table
*insns_sysctl
, *sysctl
;
247 insns_sysctl
= kzalloc(sizeof(*sysctl
) * (nr_insn_emulated
+ 1),
250 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
251 list_for_each_entry(insn
, &insn_emulation
, node
) {
252 sysctl
= &insns_sysctl
[i
];
255 sysctl
->maxlen
= sizeof(int);
257 sysctl
->procname
= insn
->ops
->name
;
259 sysctl
->extra1
= &insn
->min
;
260 sysctl
->extra2
= &insn
->max
;
261 sysctl
->proc_handler
= emulation_proc_handler
;
264 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
266 table
->child
= insns_sysctl
;
267 register_sysctl_table(table
);
271 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
274 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
275 * Where: Rt = destination
281 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
283 #define __user_swpX_asm(data, addr, res, temp, B) \
284 __asm__ __volatile__( \
285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
287 "0: ldxr"B" %w2, [%3]\n" \
288 "1: stxr"B" %w0, %w1, [%3]\n" \
295 " .pushsection .fixup,\"ax\"\n" \
297 "4: mov %w0, %w5\n" \
300 _ASM_EXTABLE(0b, 4b) \
301 _ASM_EXTABLE(1b, 4b) \
302 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
304 : "=&r" (res), "+r" (data), "=&r" (temp) \
305 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
308 #define __user_swp_asm(data, addr, res, temp) \
309 __user_swpX_asm(data, addr, res, temp, "")
310 #define __user_swpb_asm(data, addr, res, temp) \
311 __user_swpX_asm(data, addr, res, temp, "b")
314 * Bit 22 of the instruction encoding distinguishes between
315 * the SWP and SWPB variants (bit set means SWPB).
317 #define TYPE_SWPB (1 << 22)
319 static int emulate_swpX(unsigned int address
, unsigned int *data
,
322 unsigned int res
= 0;
324 if ((type
!= TYPE_SWPB
) && (address
& 0x3)) {
325 /* SWP to unaligned address not permitted */
326 pr_debug("SWP instruction on unaligned pointer!\n");
333 if (type
== TYPE_SWPB
)
334 __user_swpb_asm(*data
, address
, res
, temp
);
336 __user_swp_asm(*data
, address
, res
, temp
);
338 if (likely(res
!= -EAGAIN
) || signal_pending(current
))
347 #define ARM_OPCODE_CONDITION_UNCOND 0xf
349 static unsigned int __kprobes
aarch32_check_condition(u32 opcode
, u32 psr
)
351 u32 cc_bits
= opcode
>> 28;
353 if (cc_bits
!= ARM_OPCODE_CONDITION_UNCOND
) {
354 if ((*aarch32_opcode_cond_checks
[cc_bits
])(psr
))
355 return ARM_OPCODE_CONDTEST_PASS
;
357 return ARM_OPCODE_CONDTEST_FAIL
;
359 return ARM_OPCODE_CONDTEST_UNCOND
;
363 * swp_handler logs the id of calling process, dissects the instruction, sanity
364 * checks the memory location, calls emulate_swpX for the actual operation and
365 * deals with fixup/error handling before returning
367 static int swp_handler(struct pt_regs
*regs
, u32 instr
)
369 u32 destreg
, data
, type
, address
= 0;
370 int rn
, rt2
, res
= 0;
372 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
374 type
= instr
& TYPE_SWPB
;
376 switch (aarch32_check_condition(instr
, regs
->pstate
)) {
377 case ARM_OPCODE_CONDTEST_PASS
:
379 case ARM_OPCODE_CONDTEST_FAIL
:
380 /* Condition failed - return to next instruction */
382 case ARM_OPCODE_CONDTEST_UNCOND
:
383 /* If unconditional encoding - not a SWP, undef */
389 rn
= aarch32_insn_extract_reg_num(instr
, A32_RN_OFFSET
);
390 rt2
= aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
);
392 address
= (u32
)regs
->user_regs
.regs
[rn
];
393 data
= (u32
)regs
->user_regs
.regs
[rt2
];
394 destreg
= aarch32_insn_extract_reg_num(instr
, A32_RT_OFFSET
);
396 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
397 rn
, address
, destreg
,
398 aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
), data
);
400 /* Check access in reasonable access range for both SWP and SWPB */
401 if (!access_ok(VERIFY_WRITE
, (address
& ~3), 4)) {
402 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
407 res
= emulate_swpX(address
, &data
, type
);
411 regs
->user_regs
.regs
[destreg
] = data
;
414 if (type
== TYPE_SWPB
)
415 trace_instruction_emulation("swpb", regs
->pc
);
417 trace_instruction_emulation("swp", regs
->pc
);
419 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
420 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
426 pr_debug("SWP{B} emulation: access caused memory abort!\n");
427 arm64_notify_segfault(regs
, address
);
433 * Only emulate SWP/SWPB executed in ARM state/User mode.
434 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
436 static struct undef_hook swp_hooks
[] = {
438 .instr_mask
= 0x0fb00ff0,
439 .instr_val
= 0x01000090,
440 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
441 .pstate_val
= COMPAT_PSR_MODE_USR
,
447 static struct insn_emulation_ops swp_ops
= {
449 .status
= INSN_OBSOLETE
,
454 static int cp15barrier_handler(struct pt_regs
*regs
, u32 instr
)
456 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
458 switch (aarch32_check_condition(instr
, regs
->pstate
)) {
459 case ARM_OPCODE_CONDTEST_PASS
:
461 case ARM_OPCODE_CONDTEST_FAIL
:
462 /* Condition failed - return to next instruction */
464 case ARM_OPCODE_CONDTEST_UNCOND
:
465 /* If unconditional encoding - not a barrier instruction */
471 switch (aarch32_insn_mcr_extract_crm(instr
)) {
474 * dmb - mcr p15, 0, Rt, c7, c10, 5
475 * dsb - mcr p15, 0, Rt, c7, c10, 4
477 if (aarch32_insn_mcr_extract_opc2(instr
) == 5) {
479 trace_instruction_emulation(
480 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs
->pc
);
483 trace_instruction_emulation(
484 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs
->pc
);
489 * isb - mcr p15, 0, Rt, c7, c5, 4
491 * Taking an exception or returning from one acts as an
492 * instruction barrier. So no explicit barrier needed here.
494 trace_instruction_emulation(
495 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs
->pc
);
500 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
501 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
507 static int cp15_barrier_set_hw_mode(bool enable
)
510 config_sctlr_el1(0, SCTLR_EL1_CP15BEN
);
512 config_sctlr_el1(SCTLR_EL1_CP15BEN
, 0);
516 static struct undef_hook cp15_barrier_hooks
[] = {
518 .instr_mask
= 0x0fff0fdf,
519 .instr_val
= 0x0e070f9a,
520 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
521 .pstate_val
= COMPAT_PSR_MODE_USR
,
522 .fn
= cp15barrier_handler
,
525 .instr_mask
= 0x0fff0fff,
526 .instr_val
= 0x0e070f95,
527 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
528 .pstate_val
= COMPAT_PSR_MODE_USR
,
529 .fn
= cp15barrier_handler
,
534 static struct insn_emulation_ops cp15_barrier_ops
= {
535 .name
= "cp15_barrier",
536 .status
= INSN_DEPRECATED
,
537 .hooks
= cp15_barrier_hooks
,
538 .set_hw_mode
= cp15_barrier_set_hw_mode
,
541 static int setend_set_hw_mode(bool enable
)
543 if (!cpu_supports_mixed_endian_el0())
547 config_sctlr_el1(SCTLR_EL1_SED
, 0);
549 config_sctlr_el1(0, SCTLR_EL1_SED
);
553 static int compat_setend_handler(struct pt_regs
*regs
, u32 big_endian
)
557 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
561 regs
->pstate
|= COMPAT_PSR_E_BIT
;
564 regs
->pstate
&= ~COMPAT_PSR_E_BIT
;
567 trace_instruction_emulation(insn
, regs
->pc
);
568 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
569 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
574 static int a32_setend_handler(struct pt_regs
*regs
, u32 instr
)
576 int rc
= compat_setend_handler(regs
, (instr
>> 9) & 1);
581 static int t16_setend_handler(struct pt_regs
*regs
, u32 instr
)
583 int rc
= compat_setend_handler(regs
, (instr
>> 3) & 1);
588 static struct undef_hook setend_hooks
[] = {
590 .instr_mask
= 0xfffffdff,
591 .instr_val
= 0xf1010000,
592 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
593 .pstate_val
= COMPAT_PSR_MODE_USR
,
594 .fn
= a32_setend_handler
,
598 .instr_mask
= 0x0000fff7,
599 .instr_val
= 0x0000b650,
600 .pstate_mask
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_MASK
),
601 .pstate_val
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_USR
),
602 .fn
= t16_setend_handler
,
607 static struct insn_emulation_ops setend_ops
= {
609 .status
= INSN_DEPRECATED
,
610 .hooks
= setend_hooks
,
611 .set_hw_mode
= setend_set_hw_mode
,
615 * Invoked as late_initcall, since not needed before init spawned.
617 static int __init
armv8_deprecated_init(void)
619 if (IS_ENABLED(CONFIG_SWP_EMULATION
))
620 register_insn_emulation(&swp_ops
);
622 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION
))
623 register_insn_emulation(&cp15_barrier_ops
);
625 if (IS_ENABLED(CONFIG_SETEND_EMULATION
)) {
626 if(system_supports_mixed_endian_el0())
627 register_insn_emulation(&setend_ops
);
629 pr_info("setend instruction emulation is not supported on the system");
632 cpuhp_setup_state_nocalls(CPUHP_AP_ARM64_ISNDEP_STARTING
,
633 "AP_ARM64_ISNDEP_STARTING",
634 run_all_insn_set_hw_mode
, NULL
);
635 register_insn_emulation_sysctl(ctl_abi
);
640 late_initcall(armv8_deprecated_init
);