2 * unaligned.c: Unaligned load/store trap handling with special
3 * cases for the kernel to do them more quickly.
5 * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net)
6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
10 #include <linux/jiffies.h>
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
14 #include <linux/module.h>
16 #include <asm/ptrace.h>
17 #include <asm/pstate.h>
18 #include <asm/processor.h>
19 #include <asm/uaccess.h>
20 #include <linux/smp.h>
21 #include <linux/bitops.h>
22 #include <linux/perf_event.h>
23 #include <linux/ratelimit.h>
24 #include <linux/context_tracking.h>
25 #include <asm/fpumacro.h>
26 #include <asm/cacheflush.h>
31 load
, /* ld, ldd, ldh, ldsh */
32 store
, /* st, std, sth, stsh */
33 both
, /* Swap, ldstub, cas, ... */
39 static inline enum direction
decode_direction(unsigned int insn
)
41 unsigned long tmp
= (insn
>> 21) & 1;
46 switch ((insn
>>19)&0xf) {
55 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
56 static inline int decode_access_size(struct pt_regs
*regs
, unsigned int insn
)
60 tmp
= ((insn
>> 19) & 0xf);
61 if (tmp
== 11 || tmp
== 14) /* ldx/stx */
67 return 16; /* ldd/std - Although it is actually 8 */
71 printk("Impossible unaligned trap. insn=%08x\n", insn
);
72 die_if_kernel("Byte sized unaligned access?!?!", regs
);
74 /* GCC should never warn that control reaches the end
75 * of this function without returning a value because
76 * die_if_kernel() is marked with attribute 'noreturn'.
77 * Alas, some versions do...
84 static inline int decode_asi(unsigned int insn
, struct pt_regs
*regs
)
86 if (insn
& 0x800000) {
88 return (unsigned char)(regs
->tstate
>> 24); /* %asi */
90 return (unsigned char)(insn
>> 5); /* imm_asi */
95 /* 0x400000 = signed, 0 = unsigned */
96 static inline int decode_signedness(unsigned int insn
)
98 return (insn
& 0x400000);
101 static inline void maybe_flush_windows(unsigned int rs1
, unsigned int rs2
,
102 unsigned int rd
, int from_kernel
)
104 if (rs2
>= 16 || rs1
>= 16 || rd
>= 16) {
105 if (from_kernel
!= 0)
106 __asm__
__volatile__("flushw");
112 static inline long sign_extend_imm13(long imm
)
114 return imm
<< 51 >> 51;
117 static unsigned long fetch_reg(unsigned int reg
, struct pt_regs
*regs
)
119 unsigned long value
, fp
;
122 return (!reg
? 0 : regs
->u_regs
[reg
]);
124 fp
= regs
->u_regs
[UREG_FP
];
126 if (regs
->tstate
& TSTATE_PRIV
) {
127 struct reg_window
*win
;
128 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
129 value
= win
->locals
[reg
- 16];
130 } else if (!test_thread_64bit_stack(fp
)) {
131 struct reg_window32 __user
*win32
;
132 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)fp
));
133 get_user(value
, &win32
->locals
[reg
- 16]);
135 struct reg_window __user
*win
;
136 win
= (struct reg_window __user
*)(fp
+ STACK_BIAS
);
137 get_user(value
, &win
->locals
[reg
- 16]);
142 static unsigned long *fetch_reg_addr(unsigned int reg
, struct pt_regs
*regs
)
147 return ®s
->u_regs
[reg
];
149 fp
= regs
->u_regs
[UREG_FP
];
151 if (regs
->tstate
& TSTATE_PRIV
) {
152 struct reg_window
*win
;
153 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
154 return &win
->locals
[reg
- 16];
155 } else if (!test_thread_64bit_stack(fp
)) {
156 struct reg_window32
*win32
;
157 win32
= (struct reg_window32
*)((unsigned long)((u32
)fp
));
158 return (unsigned long *)&win32
->locals
[reg
- 16];
160 struct reg_window
*win
;
161 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
162 return &win
->locals
[reg
- 16];
166 unsigned long compute_effective_address(struct pt_regs
*regs
,
167 unsigned int insn
, unsigned int rd
)
169 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
170 unsigned int rs1
= (insn
>> 14) & 0x1f;
171 unsigned int rs2
= insn
& 0x1f;
175 maybe_flush_windows(rs1
, 0, rd
, from_kernel
);
176 addr
= (fetch_reg(rs1
, regs
) + sign_extend_imm13(insn
));
178 maybe_flush_windows(rs1
, rs2
, rd
, from_kernel
);
179 addr
= (fetch_reg(rs1
, regs
) + fetch_reg(rs2
, regs
));
182 if (!from_kernel
&& test_thread_flag(TIF_32BIT
))
188 /* This is just to make gcc think die_if_kernel does return... */
189 static void __used
unaligned_panic(char *str
, struct pt_regs
*regs
)
191 die_if_kernel(str
, regs
);
194 extern int do_int_load(unsigned long *dest_reg
, int size
,
195 unsigned long *saddr
, int is_signed
, int asi
);
197 extern int __do_int_store(unsigned long *dst_addr
, int size
,
198 unsigned long src_val
, int asi
);
200 static inline int do_int_store(int reg_num
, int size
, unsigned long *dst_addr
,
201 struct pt_regs
*regs
, int asi
, int orig_asi
)
203 unsigned long zero
= 0;
204 unsigned long *src_val_p
= &zero
;
205 unsigned long src_val
;
209 zero
= (((long)(reg_num
?
210 (unsigned)fetch_reg(reg_num
, regs
) : 0)) << 32) |
211 (unsigned)fetch_reg(reg_num
+ 1, regs
);
212 } else if (reg_num
) {
213 src_val_p
= fetch_reg_addr(reg_num
, regs
);
215 src_val
= *src_val_p
;
216 if (unlikely(asi
!= orig_asi
)) {
219 src_val
= swab16(src_val
);
222 src_val
= swab32(src_val
);
225 src_val
= swab64(src_val
);
233 return __do_int_store(dst_addr
, size
, src_val
, asi
);
236 static inline void advance(struct pt_regs
*regs
)
238 regs
->tpc
= regs
->tnpc
;
240 if (test_thread_flag(TIF_32BIT
)) {
241 regs
->tpc
&= 0xffffffff;
242 regs
->tnpc
&= 0xffffffff;
246 static inline int floating_point_load_or_store_p(unsigned int insn
)
248 return (insn
>> 24) & 1;
251 static inline int ok_for_kernel(unsigned int insn
)
253 return !floating_point_load_or_store_p(insn
);
256 static void kernel_mna_trap_fault(int fixup_tstate_asi
)
258 struct pt_regs
*regs
= current_thread_info()->kern_una_regs
;
259 unsigned int insn
= current_thread_info()->kern_una_insn
;
260 const struct exception_table_entry
*entry
;
262 entry
= search_exception_tables(regs
->tpc
);
264 unsigned long address
;
266 address
= compute_effective_address(regs
, insn
,
267 ((insn
>> 25) & 0x1f));
268 if (address
< PAGE_SIZE
) {
269 printk(KERN_ALERT
"Unable to handle kernel NULL "
270 "pointer dereference in mna handler");
272 printk(KERN_ALERT
"Unable to handle kernel paging "
273 "request in mna handler");
274 printk(KERN_ALERT
" at virtual address %016lx\n",address
);
275 printk(KERN_ALERT
"current->{active_,}mm->context = %016lx\n",
276 (current
->mm
? CTX_HWBITS(current
->mm
->context
) :
277 CTX_HWBITS(current
->active_mm
->context
)));
278 printk(KERN_ALERT
"current->{active_,}mm->pgd = %016lx\n",
279 (current
->mm
? (unsigned long) current
->mm
->pgd
:
280 (unsigned long) current
->active_mm
->pgd
));
281 die_if_kernel("Oops", regs
);
284 regs
->tpc
= entry
->fixup
;
285 regs
->tnpc
= regs
->tpc
+ 4;
287 if (fixup_tstate_asi
) {
288 regs
->tstate
&= ~TSTATE_ASI
;
289 regs
->tstate
|= (ASI_AIUS
<< 24UL);
293 static void log_unaligned(struct pt_regs
*regs
)
295 static DEFINE_RATELIMIT_STATE(ratelimit
, 5 * HZ
, 5);
297 if (__ratelimit(&ratelimit
)) {
298 printk("Kernel unaligned access at TPC[%lx] %pS\n",
299 regs
->tpc
, (void *) regs
->tpc
);
303 asmlinkage
void kernel_unaligned_trap(struct pt_regs
*regs
, unsigned int insn
)
305 enum direction dir
= decode_direction(insn
);
306 int size
= decode_access_size(regs
, insn
);
309 current_thread_info()->kern_una_regs
= regs
;
310 current_thread_info()->kern_una_insn
= insn
;
312 orig_asi
= asi
= decode_asi(insn
, regs
);
314 /* If this is a {get,put}_user() on an unaligned userspace pointer,
315 * just signal a fault and do not log the event.
317 if (asi
== ASI_AIUS
) {
318 kernel_mna_trap_fault(0);
324 if (!ok_for_kernel(insn
) || dir
== both
) {
325 printk("Unsupported unaligned load/store trap for kernel "
326 "at <%016lx>.\n", regs
->tpc
);
327 unaligned_panic("Kernel does fpu/atomic "
328 "unaligned load/store.", regs
);
330 kernel_mna_trap_fault(0);
332 unsigned long addr
, *reg_addr
;
335 addr
= compute_effective_address(regs
, insn
,
336 ((insn
>> 25) & 0x1f));
337 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, addr
);
351 reg_addr
= fetch_reg_addr(((insn
>>25)&0x1f), regs
);
352 err
= do_int_load(reg_addr
, size
,
353 (unsigned long *) addr
,
354 decode_signedness(insn
), asi
);
355 if (likely(!err
) && unlikely(asi
!= orig_asi
)) {
356 unsigned long val_in
= *reg_addr
;
359 val_in
= swab16(val_in
);
362 val_in
= swab32(val_in
);
365 val_in
= swab64(val_in
);
377 err
= do_int_store(((insn
>>25)&0x1f), size
,
378 (unsigned long *) addr
, regs
,
383 panic("Impossible kernel unaligned trap.");
387 kernel_mna_trap_fault(1);
393 int handle_popc(u32 insn
, struct pt_regs
*regs
)
395 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
396 int ret
, rd
= ((insn
>> 25) & 0x1f);
399 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
401 maybe_flush_windows(0, 0, rd
, from_kernel
);
402 value
= sign_extend_imm13(insn
);
404 maybe_flush_windows(0, insn
& 0x1f, rd
, from_kernel
);
405 value
= fetch_reg(insn
& 0x1f, regs
);
407 ret
= hweight64(value
);
410 regs
->u_regs
[rd
] = ret
;
412 unsigned long fp
= regs
->u_regs
[UREG_FP
];
414 if (!test_thread_64bit_stack(fp
)) {
415 struct reg_window32 __user
*win32
;
416 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)fp
));
417 put_user(ret
, &win32
->locals
[rd
- 16]);
419 struct reg_window __user
*win
;
420 win
= (struct reg_window __user
*)(fp
+ STACK_BIAS
);
421 put_user(ret
, &win
->locals
[rd
- 16]);
428 extern void do_fpother(struct pt_regs
*regs
);
429 extern void do_privact(struct pt_regs
*regs
);
430 extern void sun4v_data_access_exception(struct pt_regs
*regs
,
432 unsigned long type_ctx
);
434 int handle_ldf_stq(u32 insn
, struct pt_regs
*regs
)
436 unsigned long addr
= compute_effective_address(regs
, insn
, 0);
437 int freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
438 struct fpustate
*f
= FPUSTATE
;
439 int asi
= decode_asi(insn
, regs
);
440 int flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
442 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
444 save_and_clear_fpu();
445 current_thread_info()->xfsr
[0] &= ~0x1c000;
447 current_thread_info()->xfsr
[0] |= (6 << 14) /* invalid_fp_register */;
451 if (insn
& 0x200000) {
453 u64 first
= 0, second
= 0;
455 if (current_thread_info()->fpsaved
[0] & flag
) {
456 first
= *(u64
*)&f
->regs
[freg
];
457 second
= *(u64
*)&f
->regs
[freg
+2];
469 /* Need to convert endians */
470 u64 tmp
= __swab64p(&first
);
472 first
= __swab64p(&second
);
477 if (tlb_type
== hypervisor
)
478 sun4v_data_access_exception(regs
, addr
, 0);
480 spitfire_data_access_exception(regs
, 0, addr
);
483 if (put_user (first
>> 32, (u32 __user
*)addr
) ||
484 __put_user ((u32
)first
, (u32 __user
*)(addr
+ 4)) ||
485 __put_user (second
>> 32, (u32 __user
*)(addr
+ 8)) ||
486 __put_user ((u32
)second
, (u32 __user
*)(addr
+ 12))) {
487 if (tlb_type
== hypervisor
)
488 sun4v_data_access_exception(regs
, addr
, 0);
490 spitfire_data_access_exception(regs
, 0, addr
);
494 /* LDF, LDDF, LDQF */
495 u32 data
[4] __attribute__ ((aligned(8)));
502 } else if (asi
> ASI_SNFL
) {
503 if (tlb_type
== hypervisor
)
504 sun4v_data_access_exception(regs
, addr
, 0);
506 spitfire_data_access_exception(regs
, 0, addr
);
509 switch (insn
& 0x180000) {
510 case 0x000000: size
= 1; break;
511 case 0x100000: size
= 4; break;
512 default: size
= 2; break;
514 for (i
= 0; i
< size
; i
++)
517 err
= get_user (data
[0], (u32 __user
*) addr
);
519 for (i
= 1; i
< size
; i
++)
520 err
|= __get_user (data
[i
], (u32 __user
*)(addr
+ 4*i
));
522 if (err
&& !(asi
& 0x2 /* NF */)) {
523 if (tlb_type
== hypervisor
)
524 sun4v_data_access_exception(regs
, addr
, 0);
526 spitfire_data_access_exception(regs
, 0, addr
);
529 if (asi
& 0x8) /* Little */ {
533 case 1: data
[0] = le32_to_cpup(data
+ 0); break;
534 default:*(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 0));
536 case 4: tmp
= le64_to_cpup((u64
*)(data
+ 0));
537 *(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 2));
538 *(u64
*)(data
+ 2) = tmp
;
542 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
543 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
544 current_thread_info()->gsr
[0] = 0;
546 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
548 memset(f
->regs
, 0, 32*sizeof(u32
));
550 memset(f
->regs
+32, 0, 32*sizeof(u32
));
552 memcpy(f
->regs
+ freg
, data
, size
* 4);
553 current_thread_info()->fpsaved
[0] |= flag
;
559 void handle_ld_nf(u32 insn
, struct pt_regs
*regs
)
561 int rd
= ((insn
>> 25) & 0x1f);
562 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
565 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
567 maybe_flush_windows(0, 0, rd
, from_kernel
);
568 reg
= fetch_reg_addr(rd
, regs
);
569 if (from_kernel
|| rd
< 16) {
571 if ((insn
& 0x780000) == 0x180000)
573 } else if (!test_thread_64bit_stack(regs
->u_regs
[UREG_FP
])) {
574 put_user(0, (int __user
*) reg
);
575 if ((insn
& 0x780000) == 0x180000)
576 put_user(0, ((int __user
*) reg
) + 1);
578 put_user(0, (unsigned long __user
*) reg
);
579 if ((insn
& 0x780000) == 0x180000)
580 put_user(0, (unsigned long __user
*) reg
+ 1);
585 void handle_lddfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
587 enum ctx_state prev_state
= exception_enter();
588 unsigned long pc
= regs
->tpc
;
589 unsigned long tstate
= regs
->tstate
;
594 struct fpustate
*f
= FPUSTATE
;
596 if (tstate
& TSTATE_PRIV
)
597 die_if_kernel("lddfmna from kernel", regs
);
598 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, sfar
);
599 if (test_thread_flag(TIF_32BIT
))
601 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
602 int asi
= decode_asi(insn
, regs
);
606 if ((asi
> ASI_SNFL
) ||
610 err
= get_user(first
, (u32 __user
*)sfar
);
612 err
= get_user(second
, (u32 __user
*)(sfar
+ 4));
618 save_and_clear_fpu();
619 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
620 value
= (((u64
)first
) << 32) | second
;
621 if (asi
& 0x8) /* Little */
622 value
= __swab64p(&value
);
623 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
624 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
625 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
626 current_thread_info()->gsr
[0] = 0;
628 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
630 memset(f
->regs
, 0, 32*sizeof(u32
));
632 memset(f
->regs
+32, 0, 32*sizeof(u32
));
634 *(u64
*)(f
->regs
+ freg
) = value
;
635 current_thread_info()->fpsaved
[0] |= flag
;
638 if (tlb_type
== hypervisor
)
639 sun4v_data_access_exception(regs
, sfar
, sfsr
);
641 spitfire_data_access_exception(regs
, sfsr
, sfar
);
646 exception_exit(prev_state
);
649 void handle_stdfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
651 enum ctx_state prev_state
= exception_enter();
652 unsigned long pc
= regs
->tpc
;
653 unsigned long tstate
= regs
->tstate
;
658 struct fpustate
*f
= FPUSTATE
;
660 if (tstate
& TSTATE_PRIV
)
661 die_if_kernel("stdfmna from kernel", regs
);
662 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, sfar
);
663 if (test_thread_flag(TIF_32BIT
))
665 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
666 int asi
= decode_asi(insn
, regs
);
667 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
669 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
670 if ((asi
> ASI_SNFL
) ||
673 save_and_clear_fpu();
674 if (current_thread_info()->fpsaved
[0] & flag
)
675 value
= *(u64
*)&f
->regs
[freg
];
681 value
= __swab64p(&value
); break;
684 if (put_user (value
>> 32, (u32 __user
*) sfar
) ||
685 __put_user ((u32
)value
, (u32 __user
*)(sfar
+ 4)))
689 if (tlb_type
== hypervisor
)
690 sun4v_data_access_exception(regs
, sfar
, sfsr
);
692 spitfire_data_access_exception(regs
, sfsr
, sfar
);
697 exception_exit(prev_state
);