2 * Copyright (C) 2014 ARM Limited
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <linux/init.h>
11 #include <linux/list.h>
12 #include <linux/perf_event.h>
13 #include <linux/sched.h>
14 #include <linux/slab.h>
15 #include <linux/sysctl.h>
17 #include <asm/alternative.h>
18 #include <asm/cpufeature.h>
20 #include <asm/opcodes.h>
21 #include <asm/sysreg.h>
22 #include <asm/system_misc.h>
23 #include <asm/traps.h>
24 #include <asm/uaccess.h>
25 #include <asm/cpufeature.h>
27 #define CREATE_TRACE_POINTS
28 #include "trace-events-emulation.h"
31 * The runtime support for deprecated instruction support can be in one of
32 * following three states -
35 * 1 = emulate (software emulation)
36 * 2 = hw (supported in hardware)
38 enum insn_emulation_mode
{
44 enum legacy_insn_status
{
49 struct insn_emulation_ops
{
51 enum legacy_insn_status status
;
52 struct undef_hook
*hooks
;
53 int (*set_hw_mode
)(bool enable
);
56 struct insn_emulation
{
57 struct list_head node
;
58 struct insn_emulation_ops
*ops
;
64 static LIST_HEAD(insn_emulation
);
65 static int nr_insn_emulated __initdata
;
66 static DEFINE_RAW_SPINLOCK(insn_emulation_lock
);
68 static void register_emulation_hooks(struct insn_emulation_ops
*ops
)
70 struct undef_hook
*hook
;
74 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
75 register_undef_hook(hook
);
77 pr_notice("Registered %s emulation handler\n", ops
->name
);
80 static void remove_emulation_hooks(struct insn_emulation_ops
*ops
)
82 struct undef_hook
*hook
;
86 for (hook
= ops
->hooks
; hook
->instr_mask
; hook
++)
87 unregister_undef_hook(hook
);
89 pr_notice("Removed %s emulation handler\n", ops
->name
);
92 static void enable_insn_hw_mode(void *data
)
94 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
95 if (insn
->ops
->set_hw_mode
)
96 insn
->ops
->set_hw_mode(true);
99 static void disable_insn_hw_mode(void *data
)
101 struct insn_emulation
*insn
= (struct insn_emulation
*)data
;
102 if (insn
->ops
->set_hw_mode
)
103 insn
->ops
->set_hw_mode(false);
106 /* Run set_hw_mode(mode) on all active CPUs */
107 static int run_all_cpu_set_hw_mode(struct insn_emulation
*insn
, bool enable
)
109 if (!insn
->ops
->set_hw_mode
)
112 on_each_cpu(enable_insn_hw_mode
, (void *)insn
, true);
114 on_each_cpu(disable_insn_hw_mode
, (void *)insn
, true);
119 * Run set_hw_mode for all insns on a starting CPU.
121 * 0 - If all the hooks ran successfully.
122 * -EINVAL - At least one hook is not supported by the CPU.
124 static int run_all_insn_set_hw_mode(unsigned long cpu
)
128 struct insn_emulation
*insn
;
130 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
131 list_for_each_entry(insn
, &insn_emulation
, node
) {
132 bool enable
= (insn
->current_mode
== INSN_HW
);
133 if (insn
->ops
->set_hw_mode
&& insn
->ops
->set_hw_mode(enable
)) {
134 pr_warn("CPU[%ld] cannot support the emulation of %s",
135 cpu
, insn
->ops
->name
);
139 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
143 static int update_insn_emulation_mode(struct insn_emulation
*insn
,
144 enum insn_emulation_mode prev
)
149 case INSN_UNDEF
: /* Nothing to be done */
152 remove_emulation_hooks(insn
->ops
);
155 if (!run_all_cpu_set_hw_mode(insn
, false))
156 pr_notice("Disabled %s support\n", insn
->ops
->name
);
160 switch (insn
->current_mode
) {
164 register_emulation_hooks(insn
->ops
);
167 ret
= run_all_cpu_set_hw_mode(insn
, true);
169 pr_notice("Enabled %s support\n", insn
->ops
->name
);
176 static void __init
register_insn_emulation(struct insn_emulation_ops
*ops
)
179 struct insn_emulation
*insn
;
181 insn
= kzalloc(sizeof(*insn
), GFP_KERNEL
);
183 insn
->min
= INSN_UNDEF
;
185 switch (ops
->status
) {
186 case INSN_DEPRECATED
:
187 insn
->current_mode
= INSN_EMULATE
;
188 /* Disable the HW mode if it was turned on at early boot time */
189 run_all_cpu_set_hw_mode(insn
, false);
193 insn
->current_mode
= INSN_UNDEF
;
194 insn
->max
= INSN_EMULATE
;
198 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
199 list_add(&insn
->node
, &insn_emulation
);
201 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
203 /* Register any handlers if required */
204 update_insn_emulation_mode(insn
, INSN_UNDEF
);
207 static int emulation_proc_handler(struct ctl_table
*table
, int write
,
208 void __user
*buffer
, size_t *lenp
,
212 struct insn_emulation
*insn
= (struct insn_emulation
*) table
->data
;
213 enum insn_emulation_mode prev_mode
= insn
->current_mode
;
215 table
->data
= &insn
->current_mode
;
216 ret
= proc_dointvec_minmax(table
, write
, buffer
, lenp
, ppos
);
218 if (ret
|| !write
|| prev_mode
== insn
->current_mode
)
221 ret
= update_insn_emulation_mode(insn
, prev_mode
);
223 /* Mode change failed, revert to previous mode. */
224 insn
->current_mode
= prev_mode
;
225 update_insn_emulation_mode(insn
, INSN_UNDEF
);
232 static struct ctl_table ctl_abi
[] = {
240 static void __init
register_insn_emulation_sysctl(struct ctl_table
*table
)
244 struct insn_emulation
*insn
;
245 struct ctl_table
*insns_sysctl
, *sysctl
;
247 insns_sysctl
= kzalloc(sizeof(*sysctl
) * (nr_insn_emulated
+ 1),
250 raw_spin_lock_irqsave(&insn_emulation_lock
, flags
);
251 list_for_each_entry(insn
, &insn_emulation
, node
) {
252 sysctl
= &insns_sysctl
[i
];
255 sysctl
->maxlen
= sizeof(int);
257 sysctl
->procname
= insn
->ops
->name
;
259 sysctl
->extra1
= &insn
->min
;
260 sysctl
->extra2
= &insn
->max
;
261 sysctl
->proc_handler
= emulation_proc_handler
;
264 raw_spin_unlock_irqrestore(&insn_emulation_lock
, flags
);
266 table
->child
= insns_sysctl
;
267 register_sysctl_table(table
);
271 * Implement emulation of the SWP/SWPB instructions using load-exclusive and
274 * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
275 * Where: Rt = destination
281 * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
283 #define __user_swpX_asm(data, addr, res, temp, B) \
284 __asm__ __volatile__( \
285 ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \
287 "0: ldxr"B" %w2, [%3]\n" \
288 "1: stxr"B" %w0, %w1, [%3]\n" \
295 " .pushsection .fixup,\"ax\"\n" \
297 "4: mov %w0, %w5\n" \
300 _ASM_EXTABLE(0b, 4b) \
301 _ASM_EXTABLE(1b, 4b) \
302 ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \
304 : "=&r" (res), "+r" (data), "=&r" (temp) \
305 : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \
308 #define __user_swp_asm(data, addr, res, temp) \
309 __user_swpX_asm(data, addr, res, temp, "")
310 #define __user_swpb_asm(data, addr, res, temp) \
311 __user_swpX_asm(data, addr, res, temp, "b")
314 * Bit 22 of the instruction encoding distinguishes between
315 * the SWP and SWPB variants (bit set means SWPB).
317 #define TYPE_SWPB (1 << 22)
320 * Set up process info to signal segmentation fault - called on access error.
322 static void set_segfault(struct pt_regs
*regs
, unsigned long addr
)
326 down_read(¤t
->mm
->mmap_sem
);
327 if (find_vma(current
->mm
, addr
) == NULL
)
328 info
.si_code
= SEGV_MAPERR
;
330 info
.si_code
= SEGV_ACCERR
;
331 up_read(¤t
->mm
->mmap_sem
);
333 info
.si_signo
= SIGSEGV
;
335 info
.si_addr
= (void *) instruction_pointer(regs
);
337 pr_debug("SWP{B} emulation: access caused memory abort!\n");
338 arm64_notify_die("Illegal memory access", regs
, &info
, 0);
341 static int emulate_swpX(unsigned int address
, unsigned int *data
,
344 unsigned int res
= 0;
346 if ((type
!= TYPE_SWPB
) && (address
& 0x3)) {
347 /* SWP to unaligned address not permitted */
348 pr_debug("SWP instruction on unaligned pointer!\n");
355 if (type
== TYPE_SWPB
)
356 __user_swpb_asm(*data
, address
, res
, temp
);
358 __user_swp_asm(*data
, address
, res
, temp
);
360 if (likely(res
!= -EAGAIN
) || signal_pending(current
))
370 * swp_handler logs the id of calling process, dissects the instruction, sanity
371 * checks the memory location, calls emulate_swpX for the actual operation and
372 * deals with fixup/error handling before returning
374 static int swp_handler(struct pt_regs
*regs
, u32 instr
)
376 u32 destreg
, data
, type
, address
= 0;
377 int rn
, rt2
, res
= 0;
379 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
381 type
= instr
& TYPE_SWPB
;
383 switch (arm_check_condition(instr
, regs
->pstate
)) {
384 case ARM_OPCODE_CONDTEST_PASS
:
386 case ARM_OPCODE_CONDTEST_FAIL
:
387 /* Condition failed - return to next instruction */
389 case ARM_OPCODE_CONDTEST_UNCOND
:
390 /* If unconditional encoding - not a SWP, undef */
396 rn
= aarch32_insn_extract_reg_num(instr
, A32_RN_OFFSET
);
397 rt2
= aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
);
399 address
= (u32
)regs
->user_regs
.regs
[rn
];
400 data
= (u32
)regs
->user_regs
.regs
[rt2
];
401 destreg
= aarch32_insn_extract_reg_num(instr
, A32_RT_OFFSET
);
403 pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
404 rn
, address
, destreg
,
405 aarch32_insn_extract_reg_num(instr
, A32_RT2_OFFSET
), data
);
407 /* Check access in reasonable access range for both SWP and SWPB */
408 if (!access_ok(VERIFY_WRITE
, (address
& ~3), 4)) {
409 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
414 res
= emulate_swpX(address
, &data
, type
);
418 regs
->user_regs
.regs
[destreg
] = data
;
421 if (type
== TYPE_SWPB
)
422 trace_instruction_emulation("swpb", regs
->pc
);
424 trace_instruction_emulation("swp", regs
->pc
);
426 pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
427 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
433 set_segfault(regs
, address
);
439 * Only emulate SWP/SWPB executed in ARM state/User mode.
440 * The kernel must be SWP free and SWP{B} does not exist in Thumb.
442 static struct undef_hook swp_hooks
[] = {
444 .instr_mask
= 0x0fb00ff0,
445 .instr_val
= 0x01000090,
446 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
447 .pstate_val
= COMPAT_PSR_MODE_USR
,
453 static struct insn_emulation_ops swp_ops
= {
455 .status
= INSN_OBSOLETE
,
460 static int cp15barrier_handler(struct pt_regs
*regs
, u32 instr
)
462 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
464 switch (arm_check_condition(instr
, regs
->pstate
)) {
465 case ARM_OPCODE_CONDTEST_PASS
:
467 case ARM_OPCODE_CONDTEST_FAIL
:
468 /* Condition failed - return to next instruction */
470 case ARM_OPCODE_CONDTEST_UNCOND
:
471 /* If unconditional encoding - not a barrier instruction */
477 switch (aarch32_insn_mcr_extract_crm(instr
)) {
480 * dmb - mcr p15, 0, Rt, c7, c10, 5
481 * dsb - mcr p15, 0, Rt, c7, c10, 4
483 if (aarch32_insn_mcr_extract_opc2(instr
) == 5) {
485 trace_instruction_emulation(
486 "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs
->pc
);
489 trace_instruction_emulation(
490 "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs
->pc
);
495 * isb - mcr p15, 0, Rt, c7, c5, 4
497 * Taking an exception or returning from one acts as an
498 * instruction barrier. So no explicit barrier needed here.
500 trace_instruction_emulation(
501 "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs
->pc
);
506 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
507 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
513 static int cp15_barrier_set_hw_mode(bool enable
)
516 config_sctlr_el1(0, SCTLR_EL1_CP15BEN
);
518 config_sctlr_el1(SCTLR_EL1_CP15BEN
, 0);
522 static struct undef_hook cp15_barrier_hooks
[] = {
524 .instr_mask
= 0x0fff0fdf,
525 .instr_val
= 0x0e070f9a,
526 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
527 .pstate_val
= COMPAT_PSR_MODE_USR
,
528 .fn
= cp15barrier_handler
,
531 .instr_mask
= 0x0fff0fff,
532 .instr_val
= 0x0e070f95,
533 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
534 .pstate_val
= COMPAT_PSR_MODE_USR
,
535 .fn
= cp15barrier_handler
,
540 static struct insn_emulation_ops cp15_barrier_ops
= {
541 .name
= "cp15_barrier",
542 .status
= INSN_DEPRECATED
,
543 .hooks
= cp15_barrier_hooks
,
544 .set_hw_mode
= cp15_barrier_set_hw_mode
,
547 static int setend_set_hw_mode(bool enable
)
549 if (!cpu_supports_mixed_endian_el0())
553 config_sctlr_el1(SCTLR_EL1_SED
, 0);
555 config_sctlr_el1(0, SCTLR_EL1_SED
);
559 static int compat_setend_handler(struct pt_regs
*regs
, u32 big_endian
)
563 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, regs
->pc
);
567 regs
->pstate
|= COMPAT_PSR_E_BIT
;
570 regs
->pstate
&= ~COMPAT_PSR_E_BIT
;
573 trace_instruction_emulation(insn
, regs
->pc
);
574 pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
575 current
->comm
, (unsigned long)current
->pid
, regs
->pc
);
580 static int a32_setend_handler(struct pt_regs
*regs
, u32 instr
)
582 int rc
= compat_setend_handler(regs
, (instr
>> 9) & 1);
587 static int t16_setend_handler(struct pt_regs
*regs
, u32 instr
)
589 int rc
= compat_setend_handler(regs
, (instr
>> 3) & 1);
594 static struct undef_hook setend_hooks
[] = {
596 .instr_mask
= 0xfffffdff,
597 .instr_val
= 0xf1010000,
598 .pstate_mask
= COMPAT_PSR_MODE_MASK
,
599 .pstate_val
= COMPAT_PSR_MODE_USR
,
600 .fn
= a32_setend_handler
,
604 .instr_mask
= 0x0000fff7,
605 .instr_val
= 0x0000b650,
606 .pstate_mask
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_MASK
),
607 .pstate_val
= (COMPAT_PSR_T_BIT
| COMPAT_PSR_MODE_USR
),
608 .fn
= t16_setend_handler
,
613 static struct insn_emulation_ops setend_ops
= {
615 .status
= INSN_DEPRECATED
,
616 .hooks
= setend_hooks
,
617 .set_hw_mode
= setend_set_hw_mode
,
620 static int insn_cpu_hotplug_notify(struct notifier_block
*b
,
621 unsigned long action
, void *hcpu
)
624 if ((action
& ~CPU_TASKS_FROZEN
) == CPU_STARTING
)
625 rc
= run_all_insn_set_hw_mode((unsigned long)hcpu
);
627 return notifier_from_errno(rc
);
630 static struct notifier_block insn_cpu_hotplug_notifier
= {
631 .notifier_call
= insn_cpu_hotplug_notify
,
635 * Invoked as late_initcall, since not needed before init spawned.
637 static int __init
armv8_deprecated_init(void)
639 if (IS_ENABLED(CONFIG_SWP_EMULATION
))
640 register_insn_emulation(&swp_ops
);
642 if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION
))
643 register_insn_emulation(&cp15_barrier_ops
);
645 if (IS_ENABLED(CONFIG_SETEND_EMULATION
)) {
646 if(system_supports_mixed_endian_el0())
647 register_insn_emulation(&setend_ops
);
649 pr_info("setend instruction emulation is not supported on the system");
652 register_cpu_notifier(&insn_cpu_hotplug_notifier
);
653 register_insn_emulation_sysctl(ctl_abi
);
658 late_initcall(armv8_deprecated_init
);