arch/arm64/kernel/armv8_deprecated.c

   1 /*
   2  *  Copyright (C) 2014 ARM Limited
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #include <linux/cpu.h>
  10 #include <linux/init.h>
  11 #include <linux/list.h>
  12 #include <linux/perf_event.h>
  13 #include <linux/sched.h>
  14 #include <linux/slab.h>
  15 #include <linux/sysctl.h>
  16
  17 #include <asm/insn.h>
  18 #include <asm/opcodes.h>
  19 #include <asm/system_misc.h>
  20 #include <asm/traps.h>
  21 #include <asm/uaccess.h>
  22
  23 /*
  24  * The runtime support for deprecated instruction support can be in one of
  25  * following three states -
  26  *
  27  * 0 = undef
  28  * 1 = emulate (software emulation)
  29  * 2 = hw (supported in hardware)
  30  */
  31 enum insn_emulation_mode {
  32         INSN_UNDEF,
  33         INSN_EMULATE,
  34         INSN_HW,
  35 };
  36
  37 enum legacy_insn_status {
  38         INSN_DEPRECATED,
  39         INSN_OBSOLETE,
  40 };
  41
  42 struct insn_emulation_ops {
  43         const char              *name;
  44         enum legacy_insn_status status;
  45         struct undef_hook       *hooks;
  46         int                     (*set_hw_mode)(bool enable);
  47 };
  48
  49 struct insn_emulation {
  50         struct list_head node;
  51         struct insn_emulation_ops *ops;
  52         int current_mode;
  53         int min;
  54         int max;
  55 };
  56
  57 static LIST_HEAD(insn_emulation);
  58 static int nr_insn_emulated;
  59 static DEFINE_RAW_SPINLOCK(insn_emulation_lock);
  60
  61 static void register_emulation_hooks(struct insn_emulation_ops *ops)
  62 {
  63         struct undef_hook *hook;
  64
  65         BUG_ON(!ops->hooks);
  66
  67         for (hook = ops->hooks; hook->instr_mask; hook++)
  68                 register_undef_hook(hook);
  69
  70         pr_notice("Registered %s emulation handler\n", ops->name);
  71 }
  72
  73 static void remove_emulation_hooks(struct insn_emulation_ops *ops)
  74 {
  75         struct undef_hook *hook;
  76
  77         BUG_ON(!ops->hooks);
  78
  79         for (hook = ops->hooks; hook->instr_mask; hook++)
  80                 unregister_undef_hook(hook);
  81
  82         pr_notice("Removed %s emulation handler\n", ops->name);
  83 }
  84
  85 static int update_insn_emulation_mode(struct insn_emulation *insn,
  86                                        enum insn_emulation_mode prev)
  87 {
  88         int ret = 0;
  89
  90         switch (prev) {
  91         case INSN_UNDEF: /* Nothing to be done */
  92                 break;
  93         case INSN_EMULATE:
  94                 remove_emulation_hooks(insn->ops);
  95                 break;
  96         case INSN_HW:
  97                 if (insn->ops->set_hw_mode) {
  98                         insn->ops->set_hw_mode(false);
  99                         pr_notice("Disabled %s support\n", insn->ops->name);
 100                 }
 101                 break;
 102         }
 103
 104         switch (insn->current_mode) {
 105         case INSN_UNDEF:
 106                 break;
 107         case INSN_EMULATE:
 108                 register_emulation_hooks(insn->ops);
 109                 break;
 110         case INSN_HW:
 111                 if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true))
 112                         pr_notice("Enabled %s support\n", insn->ops->name);
 113                 else
 114                         ret = -EINVAL;
 115                 break;
 116         }
 117
 118         return ret;
 119 }
 120
 121 static void register_insn_emulation(struct insn_emulation_ops *ops)
 122 {
 123         unsigned long flags;
 124         struct insn_emulation *insn;
 125
 126         insn = kzalloc(sizeof(*insn), GFP_KERNEL);
 127         insn->ops = ops;
 128         insn->min = INSN_UNDEF;
 129
 130         switch (ops->status) {
 131         case INSN_DEPRECATED:
 132                 insn->current_mode = INSN_EMULATE;
 133                 insn->max = INSN_HW;
 134                 break;
 135         case INSN_OBSOLETE:
 136                 insn->current_mode = INSN_UNDEF;
 137                 insn->max = INSN_EMULATE;
 138                 break;
 139         }
 140
 141         raw_spin_lock_irqsave(&insn_emulation_lock, flags);
 142         list_add(&insn->node, &insn_emulation);
 143         nr_insn_emulated++;
 144         raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
 145
 146         /* Register any handlers if required */
 147         update_insn_emulation_mode(insn, INSN_UNDEF);
 148 }
 149
 150 static int emulation_proc_handler(struct ctl_table *table, int write,
 151                                   void __user *buffer, size_t *lenp,
 152                                   loff_t *ppos)
 153 {
 154         int ret = 0;
 155         struct insn_emulation *insn = (struct insn_emulation *) table->data;
 156         enum insn_emulation_mode prev_mode = insn->current_mode;
 157
 158         table->data = &insn->current_mode;
 159         ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 160
 161         if (ret || !write || prev_mode == insn->current_mode)
 162                 goto ret;
 163
 164         ret = update_insn_emulation_mode(insn, prev_mode);
 165         if (!ret) {
 166                 /* Mode change failed, revert to previous mode. */
 167                 insn->current_mode = prev_mode;
 168                 update_insn_emulation_mode(insn, INSN_UNDEF);
 169         }
 170 ret:
 171         table->data = insn;
 172         return ret;
 173 }
 174
 175 static struct ctl_table ctl_abi[] = {
 176         {
 177                 .procname = "abi",
 178                 .mode = 0555,
 179         },
 180         { }
 181 };
 182
 183 static void register_insn_emulation_sysctl(struct ctl_table *table)
 184 {
 185         unsigned long flags;
 186         int i = 0;
 187         struct insn_emulation *insn;
 188         struct ctl_table *insns_sysctl, *sysctl;
 189
 190         insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1),
 191                               GFP_KERNEL);
 192
 193         raw_spin_lock_irqsave(&insn_emulation_lock, flags);
 194         list_for_each_entry(insn, &insn_emulation, node) {
 195                 sysctl = &insns_sysctl[i];
 196
 197                 sysctl->mode = 0644;
 198                 sysctl->maxlen = sizeof(int);
 199
 200                 sysctl->procname = insn->ops->name;
 201                 sysctl->data = insn;
 202                 sysctl->extra1 = &insn->min;
 203                 sysctl->extra2 = &insn->max;
 204                 sysctl->proc_handler = emulation_proc_handler;
 205                 i++;
 206         }
 207         raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
 208
 209         table->child = insns_sysctl;
 210         register_sysctl_table(table);
 211 }
 212
 213 /*
 214  *  Implement emulation of the SWP/SWPB instructions using load-exclusive and
 215  *  store-exclusive.
 216  *
 217  *  Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
 218  *  Where: Rt  = destination
 219  *         Rt2 = source
 220  *         Rn  = address
 221  */
 222
 223 /*
 224  * Error-checking SWP macros implemented using ldxr{b}/stxr{b}
 225  */
 226 #define __user_swpX_asm(data, addr, res, temp, B)               \
 227         __asm__ __volatile__(                                   \
 228         "       mov             %w2, %w1\n"                     \
 229         "0:     ldxr"B"         %w1, [%3]\n"                    \
 230         "1:     stxr"B"         %w0, %w2, [%3]\n"               \
 231         "       cbz             %w0, 2f\n"                      \
 232         "       mov             %w0, %w4\n"                     \
 233         "2:\n"                                                  \
 234         "       .pushsection     .fixup,\"ax\"\n"               \
 235         "       .align          2\n"                            \
 236         "3:     mov             %w0, %w5\n"                     \
 237         "       b               2b\n"                           \
 238         "       .popsection"                                    \
 239         "       .pushsection     __ex_table,\"a\"\n"            \
 240         "       .align          3\n"                            \
 241         "       .quad           0b, 3b\n"                       \
 242         "       .quad           1b, 3b\n"                       \
 243         "       .popsection"                                    \
 244         : "=&r" (res), "+r" (data), "=&r" (temp)                \
 245         : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)              \
 246         : "memory")
 247
 248 #define __user_swp_asm(data, addr, res, temp) \
 249         __user_swpX_asm(data, addr, res, temp, "")
 250 #define __user_swpb_asm(data, addr, res, temp) \
 251         __user_swpX_asm(data, addr, res, temp, "b")
 252
 253 /*
 254  * Bit 22 of the instruction encoding distinguishes between
 255  * the SWP and SWPB variants (bit set means SWPB).
 256  */
 257 #define TYPE_SWPB (1 << 22)
 258
 259 /*
 260  * Set up process info to signal segmentation fault - called on access error.
 261  */
 262 static void set_segfault(struct pt_regs *regs, unsigned long addr)
 263 {
 264         siginfo_t info;
 265
 266         down_read(&current->mm->mmap_sem);
 267         if (find_vma(current->mm, addr) == NULL)
 268                 info.si_code = SEGV_MAPERR;
 269         else
 270                 info.si_code = SEGV_ACCERR;
 271         up_read(&current->mm->mmap_sem);
 272
 273         info.si_signo = SIGSEGV;
 274         info.si_errno = 0;
 275         info.si_addr  = (void *) instruction_pointer(regs);
 276
 277         pr_debug("SWP{B} emulation: access caused memory abort!\n");
 278         arm64_notify_die("Illegal memory access", regs, &info, 0);
 279 }
 280
 281 static int emulate_swpX(unsigned int address, unsigned int *data,
 282                         unsigned int type)
 283 {
 284         unsigned int res = 0;
 285
 286         if ((type != TYPE_SWPB) && (address & 0x3)) {
 287                 /* SWP to unaligned address not permitted */
 288                 pr_debug("SWP instruction on unaligned pointer!\n");
 289                 return -EFAULT;
 290         }
 291
 292         while (1) {
 293                 unsigned long temp;
 294
 295                 if (type == TYPE_SWPB)
 296                         __user_swpb_asm(*data, address, res, temp);
 297                 else
 298                         __user_swp_asm(*data, address, res, temp);
 299
 300                 if (likely(res != -EAGAIN) || signal_pending(current))
 301                         break;
 302
 303                 cond_resched();
 304         }
 305
 306         return res;
 307 }
 308
 309 /*
 310  * swp_handler logs the id of calling process, dissects the instruction, sanity
 311  * checks the memory location, calls emulate_swpX for the actual operation and
 312  * deals with fixup/error handling before returning
 313  */
 314 static int swp_handler(struct pt_regs *regs, u32 instr)
 315 {
 316         u32 destreg, data, type, address = 0;
 317         int rn, rt2, res = 0;
 318
 319         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
 320
 321         type = instr & TYPE_SWPB;
 322
 323         switch (arm_check_condition(instr, regs->pstate)) {
 324         case ARM_OPCODE_CONDTEST_PASS:
 325                 break;
 326         case ARM_OPCODE_CONDTEST_FAIL:
 327                 /* Condition failed - return to next instruction */
 328                 goto ret;
 329         case ARM_OPCODE_CONDTEST_UNCOND:
 330                 /* If unconditional encoding - not a SWP, undef */
 331                 return -EFAULT;
 332         default:
 333                 return -EINVAL;
 334         }
 335
 336         rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET);
 337         rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET);
 338
 339         address = (u32)regs->user_regs.regs[rn];
 340         data    = (u32)regs->user_regs.regs[rt2];
 341         destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET);
 342
 343         pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
 344                 rn, address, destreg,
 345                 aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data);
 346
 347         /* Check access in reasonable access range for both SWP and SWPB */
 348         if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) {
 349                 pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n",
 350                         address);
 351                 goto fault;
 352         }
 353
 354         res = emulate_swpX(address, &data, type);
 355         if (res == -EFAULT)
 356                 goto fault;
 357         else if (res == 0)
 358                 regs->user_regs.regs[destreg] = data;
 359
 360 ret:
 361         pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n",
 362                         current->comm, (unsigned long)current->pid, regs->pc);
 363
 364         regs->pc += 4;
 365         return 0;
 366
 367 fault:
 368         set_segfault(regs, address);
 369
 370         return 0;
 371 }
 372
 373 /*
 374  * Only emulate SWP/SWPB executed in ARM state/User mode.
 375  * The kernel must be SWP free and SWP{B} does not exist in Thumb.
 376  */
 377 static struct undef_hook swp_hooks[] = {
 378         {
 379                 .instr_mask     = 0x0fb00ff0,
 380                 .instr_val      = 0x01000090,
 381                 .pstate_mask    = COMPAT_PSR_MODE_MASK,
 382                 .pstate_val     = COMPAT_PSR_MODE_USR,
 383                 .fn             = swp_handler
 384         },
 385         { }
 386 };
 387
 388 static struct insn_emulation_ops swp_ops = {
 389         .name = "swp",
 390         .status = INSN_OBSOLETE,
 391         .hooks = swp_hooks,
 392         .set_hw_mode = NULL,
 393 };
 394
 395 static int cp15barrier_handler(struct pt_regs *regs, u32 instr)
 396 {
 397         perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
 398
 399         switch (arm_check_condition(instr, regs->pstate)) {
 400         case ARM_OPCODE_CONDTEST_PASS:
 401                 break;
 402         case ARM_OPCODE_CONDTEST_FAIL:
 403                 /* Condition failed - return to next instruction */
 404                 goto ret;
 405         case ARM_OPCODE_CONDTEST_UNCOND:
 406                 /* If unconditional encoding - not a barrier instruction */
 407                 return -EFAULT;
 408         default:
 409                 return -EINVAL;
 410         }
 411
 412         switch (aarch32_insn_mcr_extract_crm(instr)) {
 413         case 10:
 414                 /*
 415                  * dmb - mcr p15, 0, Rt, c7, c10, 5
 416                  * dsb - mcr p15, 0, Rt, c7, c10, 4
 417                  */
 418                 if (aarch32_insn_mcr_extract_opc2(instr) == 5)
 419                         dmb(sy);
 420                 else
 421                         dsb(sy);
 422                 break;
 423         case 5:
 424                 /*
 425                  * isb - mcr p15, 0, Rt, c7, c5, 4
 426                  *
 427                  * Taking an exception or returning from one acts as an
 428                  * instruction barrier. So no explicit barrier needed here.
 429                  */
 430                 break;
 431         }
 432
 433 ret:
 434         pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n",
 435                         current->comm, (unsigned long)current->pid, regs->pc);
 436
 437         regs->pc += 4;
 438         return 0;
 439 }
 440
 441 #define SCTLR_EL1_CP15BEN (1 << 5)
 442
 443 static inline void config_sctlr_el1(u32 clear, u32 set)
 444 {
 445         u32 val;
 446
 447         asm volatile("mrs %0, sctlr_el1" : "=r" (val));
 448         val &= ~clear;
 449         val |= set;
 450         asm volatile("msr sctlr_el1, %0" : : "r" (val));
 451 }
 452
 453 static void enable_cp15_ben(void *info)
 454 {
 455         config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
 456 }
 457
 458 static void disable_cp15_ben(void *info)
 459 {
 460         config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
 461 }
 462
 463 static int cpu_hotplug_notify(struct notifier_block *b,
 464                               unsigned long action, void *hcpu)
 465 {
 466         switch (action) {
 467         case CPU_STARTING:
 468         case CPU_STARTING_FROZEN:
 469                 enable_cp15_ben(NULL);
 470                 return NOTIFY_DONE;
 471         case CPU_DYING:
 472         case CPU_DYING_FROZEN:
 473                 disable_cp15_ben(NULL);
 474                 return NOTIFY_DONE;
 475         }
 476
 477         return NOTIFY_OK;
 478 }
 479
 480 static struct notifier_block cpu_hotplug_notifier = {
 481         .notifier_call = cpu_hotplug_notify,
 482 };
 483
 484 static int cp15_barrier_set_hw_mode(bool enable)
 485 {
 486         if (enable) {
 487                 register_cpu_notifier(&cpu_hotplug_notifier);
 488                 on_each_cpu(enable_cp15_ben, NULL, true);
 489         } else {
 490                 unregister_cpu_notifier(&cpu_hotplug_notifier);
 491                 on_each_cpu(disable_cp15_ben, NULL, true);
 492         }
 493
 494         return true;
 495 }
 496
 497 static struct undef_hook cp15_barrier_hooks[] = {
 498         {
 499                 .instr_mask     = 0x0fff0fdf,
 500                 .instr_val      = 0x0e070f9a,
 501                 .pstate_mask    = COMPAT_PSR_MODE_MASK,
 502                 .pstate_val     = COMPAT_PSR_MODE_USR,
 503                 .fn             = cp15barrier_handler,
 504         },
 505         {
 506                 .instr_mask     = 0x0fff0fff,
 507                 .instr_val      = 0x0e070f95,
 508                 .pstate_mask    = COMPAT_PSR_MODE_MASK,
 509                 .pstate_val     = COMPAT_PSR_MODE_USR,
 510                 .fn             = cp15barrier_handler,
 511         },
 512         { }
 513 };
 514
 515 static struct insn_emulation_ops cp15_barrier_ops = {
 516         .name = "cp15_barrier",
 517         .status = INSN_DEPRECATED,
 518         .hooks = cp15_barrier_hooks,
 519         .set_hw_mode = cp15_barrier_set_hw_mode,
 520 };
 521
 522 /*
 523  * Invoked as late_initcall, since not needed before init spawned.
 524  */
 525 static int __init armv8_deprecated_init(void)
 526 {
 527         if (IS_ENABLED(CONFIG_SWP_EMULATION))
 528                 register_insn_emulation(&swp_ops);
 529
 530         if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
 531                 register_insn_emulation(&cp15_barrier_ops);
 532
 533         register_insn_emulation_sysctl(ctl_abi);
 534
 535         return 0;
 536 }
 537
 538 late_initcall(armv8_deprecated_init);