projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'upstream' of git://git.linux-mips.org/pub/scm/ralf/upstream-linus
[deliverable/linux.git] / arch / mips / mm / tlbex.c
CommitLineData
1da177e4
LT		 1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
70342287
RB		 8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
9 * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
41c594ab 10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
fd062c84 11 * Copyright (C) 2008, 2009 Cavium Networks, Inc.
113c62d9 12 * Copyright (C) 2011 MIPS Technologies, Inc.
41c594ab
RB		 13 *
14 * ... and the days got worse and worse and now you see
92a76f6d 15 * I've gone completely out of my mind.
41c594ab
RB		 16 *
17 * They're coming to take me a away haha
18 * they're coming to take me a away hoho hihi haha
19 * to the funny farm where code is beautiful all the time ...
20 *
21 * (Condolences to Napoleon XIV)
1da177e4
LT		 22 */
23
95affdda 24#include <linux/bug.h>
1da177e4
LT		 25#include <linux/kernel.h>
26#include <linux/types.h>
631330f5 27#include <linux/smp.h>
1da177e4 28#include <linux/string.h>
3d8bfdd0 29#include <linux/cache.h>
1da177e4 30
3d8bfdd0 31#include <asm/cacheflush.h>
69f24d17 32#include <asm/cpu-type.h>
3d8bfdd0 33#include <asm/pgtable.h>
1da177e4 34#include <asm/war.h>
3482d713 35#include <asm/uasm.h>
b81947c6 36#include <asm/setup.h>
e30ec452 37
a2d25e63 38static int mips_xpa_disabled;
c5b36783
SH		 39
40static int __init xpa_disable(char *s)
41{
42 mips_xpa_disabled = 1;
43
44 return 1;
45}
46
47__setup("noxpa", xpa_disable);
48
1ec56329
DD		 49/*
50 * TLB load/store/modify handlers.
51 *
52 * Only the fastpath gets synthesized at runtime, the slowpath for
53 * do_page_fault remains normal asm.
54 */
55extern void tlb_do_page_fault_0(void);
56extern void tlb_do_page_fault_1(void);
57
bf28607f
DD		 58struct work_registers {
59 int r1;
60 int r2;
61 int r3;
62};
63
64struct tlb_reg_save {
65 unsigned long a;
66 unsigned long b;
67} ____cacheline_aligned_in_smp;
68
69static struct tlb_reg_save handler_reg_save[NR_CPUS];
1ec56329 70
aeffdbba 71static inline int r45k_bvahwbug(void)
1da177e4
LT		 72{
73 /* XXX: We should probe for the presence of this bug, but we don't. */
74 return 0;
75}
76
aeffdbba 77static inline int r4k_250MHZhwbug(void)
1da177e4
LT		 78{
79 /* XXX: We should probe for the presence of this bug, but we don't. */
80 return 0;
81}
82
aeffdbba 83static inline int __maybe_unused bcm1250_m3_war(void)
1da177e4
LT		 84{
85 return BCM1250_M3_WAR;
86}
87
aeffdbba 88static inline int __maybe_unused r10000_llsc_war(void)
1da177e4
LT		 89{
90 return R10000_LLSC_WAR;
91}
92
cc33ae43
DD		 93static int use_bbit_insns(void)
94{
95 switch (current_cpu_type()) {
96 case CPU_CAVIUM_OCTEON:
97 case CPU_CAVIUM_OCTEON_PLUS:
98 case CPU_CAVIUM_OCTEON2:
4723b20a 99 case CPU_CAVIUM_OCTEON3:
cc33ae43
DD		 100 return 1;
101 default:
102 return 0;
103 }
104}
105
2c8c53e2
DD		 106static int use_lwx_insns(void)
107{
108 switch (current_cpu_type()) {
109 case CPU_CAVIUM_OCTEON2:
4723b20a 110 case CPU_CAVIUM_OCTEON3:
2c8c53e2
DD		 111 return 1;
112 default:
113 return 0;
114 }
115}
116#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \
117 CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0
118static bool scratchpad_available(void)
119{
120 return true;
121}
122static int scratchpad_offset(int i)
123{
124 /*
125 * CVMSEG starts at address -32768 and extends for
126 * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines.
127 */
128 i += 1; /* Kernel use starts at the top and works down. */
129 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;
130}
131#else
132static bool scratchpad_available(void)
133{
134 return false;
135}
136static int scratchpad_offset(int i)
137{
138 BUG();
e1c87d2a
DD		 139 /* Really unreachable, but evidently some GCC want this. */
140 return 0;
2c8c53e2
DD		 141}
142#endif
8df5beac
MR		 143/*
144 * Found by experiment: At least some revisions of the 4kc throw under
145 * some circumstances a machine check exception, triggered by invalid
146 * values in the index register. Delaying the tlbp instruction until
147 * after the next branch, plus adding an additional nop in front of
148 * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
149 * why; it's not an issue caused by the core RTL.
150 *
151 */
078a55fc 152static int m4kc_tlbp_war(void)
8df5beac
MR		 153{
154 return (current_cpu_data.processor_id & 0xffff00) ==
155 (PRID_COMP_MIPS | PRID_IMP_4KC);
156}
157
e30ec452 158/* Handle labels (which must be positive integers). */
1da177e4 159enum label_id {
e30ec452 160 label_second_part = 1,
1da177e4
LT		 161 label_leave,
162 label_vmalloc,
163 label_vmalloc_done,
02a54177
RB		 164 label_tlbw_hazard_0,
165 label_split = label_tlbw_hazard_0 + 8,
6dd9344c
DD		 166 label_tlbl_goaround1,
167 label_tlbl_goaround2,
1da177e4
LT		 168 label_nopage_tlbl,
169 label_nopage_tlbs,
170 label_nopage_tlbm,
171 label_smp_pgtable_change,
172 label_r3000_write_probe_fail,
1ec56329 173 label_large_segbits_fault,
aa1762f4 174#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 175 label_tlb_huge_update,
176#endif
1da177e4
LT		 177};
178
e30ec452
TS		 179UASM_L_LA(_second_part)
180UASM_L_LA(_leave)
e30ec452
TS		 181UASM_L_LA(_vmalloc)
182UASM_L_LA(_vmalloc_done)
02a54177 183/* _tlbw_hazard_x is handled differently. */
e30ec452 184UASM_L_LA(_split)
6dd9344c
DD		 185UASM_L_LA(_tlbl_goaround1)
186UASM_L_LA(_tlbl_goaround2)
e30ec452
TS		 187UASM_L_LA(_nopage_tlbl)
188UASM_L_LA(_nopage_tlbs)
189UASM_L_LA(_nopage_tlbm)
190UASM_L_LA(_smp_pgtable_change)
191UASM_L_LA(_r3000_write_probe_fail)
1ec56329 192UASM_L_LA(_large_segbits_fault)
aa1762f4 193#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 194UASM_L_LA(_tlb_huge_update)
195#endif
656be92f 196
078a55fc 197static int hazard_instance;
02a54177 198
078a55fc 199static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
02a54177
RB		 200{
201 switch (instance) {
202 case 0 ... 7:
203 uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
204 return;
205 default:
206 BUG();
207 }
208}
209
078a55fc 210static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
02a54177
RB		 211{
212 switch (instance) {
213 case 0 ... 7:
214 uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
215 break;
216 default:
217 BUG();
218 }
219}
220
92b1e6a6 221/*
a2c763e0
RB		 222 * pgtable bits are assigned dynamically depending on processor feature
223 * and statically based on kernel configuration. This spits out the actual
70342287 224 * values the kernel is using. Required to make sense from disassembled
a2c763e0 225 * TLB exception handlers.
92b1e6a6 226 */
a2c763e0
RB		 227static void output_pgtable_bits_defines(void)
228{
229#define pr_define(fmt, ...) \
230 pr_debug("#define " fmt, ##__VA_ARGS__)
231
232 pr_debug("#include <asm/asm.h>\n");
233 pr_debug("#include <asm/regdef.h>\n");
234 pr_debug("\n");
235
236 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
780602d7 237 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
a2c763e0
RB		 238 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
239 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
240 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
970d032f 241#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
a2c763e0
RB		 242 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
243#endif
a2c763e0 244#ifdef _PAGE_NO_EXEC_SHIFT
780602d7 245 if (cpu_has_rixi)
a2c763e0 246 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
be0c37c9 247#endif
a2c763e0
RB		 248 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
249 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
250 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
251 pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
252 pr_debug("\n");
253}
254
255static inline void dump_handler(const char *symbol, const u32 *handler, int count)
92b1e6a6
FBH		 256{
257 int i;
258
a2c763e0
RB		 259 pr_debug("LEAF(%s)\n", symbol);
260
92b1e6a6
FBH		 261 pr_debug("\t.set push\n");
262 pr_debug("\t.set noreorder\n");
263
264 for (i = 0; i < count; i++)
a2c763e0 265 pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]);
92b1e6a6 266
a2c763e0
RB		 267 pr_debug("\t.set\tpop\n");
268
269 pr_debug("\tEND(%s)\n", symbol);
92b1e6a6
FBH		 270}
271
1da177e4
LT		 272/* The only general purpose registers allowed in TLB handlers. */
273#define K0 26
274#define K1 27
275
276/* Some CP0 registers */
41c594ab
RB		 277#define C0_INDEX 0, 0
278#define C0_ENTRYLO0 2, 0
279#define C0_TCBIND 2, 2
280#define C0_ENTRYLO1 3, 0
281#define C0_CONTEXT 4, 0
fd062c84 282#define C0_PAGEMASK 5, 0
380cd582
HC		 283#define C0_PWBASE 5, 5
284#define C0_PWFIELD 5, 6
285#define C0_PWSIZE 5, 7
286#define C0_PWCTL 6, 6
41c594ab 287#define C0_BADVADDR 8, 0
380cd582 288#define C0_PGD 9, 7
41c594ab
RB		 289#define C0_ENTRYHI 10, 0
290#define C0_EPC 14, 0
291#define C0_XCONTEXT 20, 0
1da177e4 292
875d43e7 293#ifdef CONFIG_64BIT
e30ec452 294# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
1da177e4 295#else
e30ec452 296# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
1da177e4
LT		 297#endif
298
299/* The worst case length of the handler is around 18 instructions for
300 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
301 * Maximum space available is 32 instructions for R3000 and 64
302 * instructions for R4000.
303 *
304 * We deliberately chose a buffer size of 128, so we won't scribble
305 * over anything important on overflow before we panic.
306 */
078a55fc 307static u32 tlb_handler[128];
1da177e4
LT		 308
309/* simply assume worst case size for labels and relocs */
078a55fc
PG		 310static struct uasm_label labels[128];
311static struct uasm_reloc relocs[128];
1da177e4 312
078a55fc 313static int check_for_high_segbits;
00bf1c69 314static bool fill_includes_sw_bits;
3d8bfdd0 315
078a55fc 316static unsigned int kscratch_used_mask;
3d8bfdd0 317
7777b939
J		 318static inline int __maybe_unused c0_kscratch(void)
319{
320 switch (current_cpu_type()) {
321 case CPU_XLP:
322 case CPU_XLR:
323 return 22;
324 default:
325 return 31;
326 }
327}
328
078a55fc 329static int allocate_kscratch(void)
3d8bfdd0
DD		 330{
331 int r;
332 unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
333
334 r = ffs(a);
335
336 if (r == 0)
337 return -1;
338
339 r--; /* make it zero based */
340
341 kscratch_used_mask |= (1 << r);
342
343 return r;
344}
345
078a55fc
PG		 346static int scratch_reg;
347static int pgd_reg;
2c8c53e2
DD		 348enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
349
078a55fc 350static struct work_registers build_get_work_registers(u32 **p)
bf28607f
DD		 351{
352 struct work_registers r;
353
0e6ecc1a 354 if (scratch_reg >= 0) {
bf28607f 355 /* Save in CPU local C0_KScratch? */
7777b939 356 UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
bf28607f
DD		 357 r.r1 = K0;
358 r.r2 = K1;
359 r.r3 = 1;
360 return r;
361 }
362
363 if (num_possible_cpus() > 1) {
bf28607f 364 /* Get smp_processor_id */
c2377a42
J		 365 UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
366 UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
bf28607f
DD		 367
368 /* handler_reg_save index in K0 */
369 UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
370
371 UASM_i_LA(p, K1, (long)&handler_reg_save);
372 UASM_i_ADDU(p, K0, K0, K1);
373 } else {
374 UASM_i_LA(p, K0, (long)&handler_reg_save);
375 }
376 /* K0 now points to save area, save $1 and $2 */
377 UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
378 UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
379
380 r.r1 = K1;
381 r.r2 = 1;
382 r.r3 = 2;
383 return r;
384}
385
078a55fc 386static void build_restore_work_registers(u32 **p)
bf28607f 387{
0e6ecc1a 388 if (scratch_reg >= 0) {
7777b939 389 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
bf28607f
DD		 390 return;
391 }
392 /* K0 already points to save area, restore $1 and $2 */
393 UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
394 UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
395}
396
2c8c53e2 397#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
3d8bfdd0 398
82622284
DD		 399/*
400 * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
401 * we cannot do r3000 under these circumstances.
3d8bfdd0
DD		 402 *
403 * Declare pgd_current here instead of including mmu_context.h to avoid type
404 * conflicts for tlbmiss_handler_setup_pgd
82622284 405 */
3d8bfdd0 406extern unsigned long pgd_current[];
82622284 407
1da177e4
LT		 408/*
409 * The R3000 TLB handler is simple.
410 */
078a55fc 411static void build_r3000_tlb_refill_handler(void)
1da177e4
LT		 412{
413 long pgdc = (long)pgd_current;
414 u32 *p;
415
416 memset(tlb_handler, 0, sizeof(tlb_handler));
417 p = tlb_handler;
418
e30ec452
TS		 419 uasm_i_mfc0(&p, K0, C0_BADVADDR);
420 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
421 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
422 uasm_i_srl(&p, K0, K0, 22); /* load delay */
423 uasm_i_sll(&p, K0, K0, 2);
424 uasm_i_addu(&p, K1, K1, K0);
425 uasm_i_mfc0(&p, K0, C0_CONTEXT);
426 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
427 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
428 uasm_i_addu(&p, K1, K1, K0);
429 uasm_i_lw(&p, K0, 0, K1);
430 uasm_i_nop(&p); /* load delay */
431 uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
432 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
433 uasm_i_tlbwr(&p); /* cp0 delay */
434 uasm_i_jr(&p, K1);
435 uasm_i_rfe(&p); /* branch delay */
1da177e4
LT		 436
437 if (p > tlb_handler + 32)
438 panic("TLB refill handler space exceeded");
439
e30ec452
TS		 440 pr_debug("Wrote TLB refill handler (%u instructions).\n",
441 (unsigned int)(p - tlb_handler));
1da177e4 442
91b05e67 443 memcpy((void *)ebase, tlb_handler, 0x80);
1062080a 444 local_flush_icache_range(ebase, ebase + 0x80);
92b1e6a6 445
a2c763e0 446 dump_handler("r3000_tlb_refill", (u32 *)ebase, 32);
1da177e4 447}
82622284 448#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
1da177e4
LT		 449
450/*
451 * The R4000 TLB handler is much more complicated. We have two
452 * consecutive handler areas with 32 instructions space each.
453 * Since they aren't used at the same time, we can overflow in the
454 * other one.To keep things simple, we first assume linear space,
455 * then we relocate it to the final handler layout as needed.
456 */
078a55fc 457static u32 final_handler[64];
1da177e4
LT		 458
459/*
460 * Hazards
461 *
462 * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
463 * 2. A timing hazard exists for the TLBP instruction.
464 *
70342287
RB		 465 * stalling_instruction
466 * TLBP
1da177e4
LT		 467 *
468 * The JTLB is being read for the TLBP throughout the stall generated by the
469 * previous instruction. This is not really correct as the stalling instruction
470 * can modify the address used to access the JTLB. The failure symptom is that
471 * the TLBP instruction will use an address created for the stalling instruction
472 * and not the address held in C0_ENHI and thus report the wrong results.
473 *
474 * The software work-around is to not allow the instruction preceding the TLBP
475 * to stall - make it an NOP or some other instruction guaranteed not to stall.
476 *
70342287 477 * Errata 2 will not be fixed. This errata is also on the R5000.
1da177e4
LT		 478 *
479 * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
480 */
078a55fc 481static void __maybe_unused build_tlb_probe_entry(u32 **p)
1da177e4 482{
10cc3529 483 switch (current_cpu_type()) {
326e2e1a 484 /* Found by experiment: R4600 v2.0/R4700 needs this, too. */
f5b4d956 485 case CPU_R4600:
326e2e1a 486 case CPU_R4700:
1da177e4 487 case CPU_R5000:
1da177e4 488 case CPU_NEVADA:
e30ec452
TS		 489 uasm_i_nop(p);
490 uasm_i_tlbp(p);
1da177e4
LT		 491 break;
492
493 default:
e30ec452 494 uasm_i_tlbp(p);
1da177e4
LT		 495 break;
496 }
497}
498
499/*
500 * Write random or indexed TLB entry, and care about the hazards from
25985edc 501 * the preceding mtc0 and for the following eret.
1da177e4
LT		 502 */
503enum tlb_write_entry { tlb_random, tlb_indexed };
504
078a55fc
PG		 505static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
506 struct uasm_reloc **r,
507 enum tlb_write_entry wmode)
1da177e4
LT		 508{
509 void(*tlbw)(u32 **) = NULL;
510
511 switch (wmode) {
e30ec452
TS		 512 case tlb_random: tlbw = uasm_i_tlbwr; break;
513 case tlb_indexed: tlbw = uasm_i_tlbwi; break;
1da177e4
LT		 514 }
515
9eaffa84
RB		 516 if (cpu_has_mips_r2_r6) {
517 if (cpu_has_mips_r2_exec_hazard)
41f0e4d0 518 uasm_i_ehb(p);
161548bf
RB		 519 tlbw(p);
520 return;
521 }
522
10cc3529 523 switch (current_cpu_type()) {
1da177e4
LT		 524 case CPU_R4000PC:
525 case CPU_R4000SC:
526 case CPU_R4000MC:
527 case CPU_R4400PC:
528 case CPU_R4400SC:
529 case CPU_R4400MC:
530 /*
531 * This branch uses up a mtc0 hazard nop slot and saves
532 * two nops after the tlbw instruction.
533 */
02a54177 534 uasm_bgezl_hazard(p, r, hazard_instance);
1da177e4 535 tlbw(p);
02a54177
RB		 536 uasm_bgezl_label(l, p, hazard_instance);
537 hazard_instance++;
e30ec452 538 uasm_i_nop(p);
1da177e4
LT		 539 break;
540
541 case CPU_R4600:
542 case CPU_R4700:
e30ec452 543 uasm_i_nop(p);
2c93e12c 544 tlbw(p);
e30ec452 545 uasm_i_nop(p);
2c93e12c
MR		 546 break;
547
359187d6 548 case CPU_R5000:
359187d6
RB		 549 case CPU_NEVADA:
550 uasm_i_nop(p); /* QED specifies 2 nops hazard */
551 uasm_i_nop(p); /* QED specifies 2 nops hazard */
552 tlbw(p);
553 break;
554
2c93e12c 555 case CPU_R4300:
1da177e4
LT		 556 case CPU_5KC:
557 case CPU_TX49XX:
bdf21b18 558 case CPU_PR4450:
efa0f81c 559 case CPU_XLR:
e30ec452 560 uasm_i_nop(p);
1da177e4
LT		 561 tlbw(p);
562 break;
563
564 case CPU_R10000:
565 case CPU_R12000:
44d921b2 566 case CPU_R14000:
30577391 567 case CPU_R16000:
1da177e4 568 case CPU_4KC:
b1ec4c8e 569 case CPU_4KEC:
113c62d9 570 case CPU_M14KC:
f8fa4811 571 case CPU_M14KEC:
1da177e4 572 case CPU_SB1:
93ce2f52 573 case CPU_SB1A:
1da177e4
LT		 574 case CPU_4KSC:
575 case CPU_20KC:
576 case CPU_25KF:
602977b0
KC		 577 case CPU_BMIPS32:
578 case CPU_BMIPS3300:
579 case CPU_BMIPS4350:
580 case CPU_BMIPS4380:
581 case CPU_BMIPS5000:
2a21c730 582 case CPU_LOONGSON2:
c579d310 583 case CPU_LOONGSON3:
a644b277 584 case CPU_R5500:
8df5beac 585 if (m4kc_tlbp_war())
e30ec452 586 uasm_i_nop(p);
2f794d09 587 case CPU_ALCHEMY:
1da177e4
LT		 588 tlbw(p);
589 break;
590
1da177e4 591 case CPU_RM7000:
e30ec452
TS		 592 uasm_i_nop(p);
593 uasm_i_nop(p);
594 uasm_i_nop(p);
595 uasm_i_nop(p);
1da177e4
LT		 596 tlbw(p);
597 break;
598
1da177e4
LT		 599 case CPU_VR4111:
600 case CPU_VR4121:
601 case CPU_VR4122:
602 case CPU_VR4181:
603 case CPU_VR4181A:
e30ec452
TS		 604 uasm_i_nop(p);
605 uasm_i_nop(p);
1da177e4 606 tlbw(p);
e30ec452
TS		 607 uasm_i_nop(p);
608 uasm_i_nop(p);
1da177e4
LT		 609 break;
610
611 case CPU_VR4131:
612 case CPU_VR4133:
7623debf 613 case CPU_R5432:
e30ec452
TS		 614 uasm_i_nop(p);
615 uasm_i_nop(p);
1da177e4
LT		 616 tlbw(p);
617 break;
618
83ccf69d
LPC		 619 case CPU_JZRISC:
620 tlbw(p);
621 uasm_i_nop(p);
622 break;
623
1da177e4
LT		 624 default:
625 panic("No TLB refill handler yet (CPU type: %d)",
d7b12056 626 current_cpu_type());
1da177e4
LT		 627 break;
628 }
629}
630
078a55fc
PG		 631static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
632 unsigned int reg)
fd062c84 633{
2caa89b4
PB		 634 if (_PAGE_GLOBAL_SHIFT == 0) {
635 /* pte_t is already in EntryLo format */
636 return;
637 }
638
00bf1c69
PB		 639 if (cpu_has_rixi && _PAGE_NO_EXEC) {
640 if (fill_includes_sw_bits) {
641 UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
642 } else {
643 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
644 UASM_i_ROTR(p, reg, reg,
645 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
646 }
6dd9344c 647 } else {
34adb28d 648#ifdef CONFIG_PHYS_ADDR_T_64BIT
3be6022c 649 uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
6dd9344c
DD		 650#else
651 UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
652#endif
653 }
654}
fd062c84 655
aa1762f4 656#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84 657
078a55fc
PG		 658static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
659 unsigned int tmp, enum label_id lid,
660 int restore_scratch)
6dd9344c 661{
2c8c53e2
DD		 662 if (restore_scratch) {
663 /* Reset default page size */
664 if (PM_DEFAULT_MASK >> 16) {
665 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
666 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
667 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
668 uasm_il_b(p, r, lid);
669 } else if (PM_DEFAULT_MASK) {
670 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
671 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
672 uasm_il_b(p, r, lid);
673 } else {
674 uasm_i_mtc0(p, 0, C0_PAGEMASK);
675 uasm_il_b(p, r, lid);
676 }
0e6ecc1a 677 if (scratch_reg >= 0)
7777b939 678 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
2c8c53e2
DD		 679 else
680 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
fd062c84 681 } else {
2c8c53e2
DD		 682 /* Reset default page size */
683 if (PM_DEFAULT_MASK >> 16) {
684 uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
685 uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
686 uasm_il_b(p, r, lid);
687 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
688 } else if (PM_DEFAULT_MASK) {
689 uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
690 uasm_il_b(p, r, lid);
691 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
692 } else {
693 uasm_il_b(p, r, lid);
694 uasm_i_mtc0(p, 0, C0_PAGEMASK);
695 }
fd062c84
DD		 696 }
697}
698
078a55fc
PG		 699static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
700 struct uasm_reloc **r,
701 unsigned int tmp,
702 enum tlb_write_entry wmode,
703 int restore_scratch)
6dd9344c
DD		 704{
705 /* Set huge page tlb entry size */
706 uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
707 uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
708 uasm_i_mtc0(p, tmp, C0_PAGEMASK);
709
710 build_tlb_write_entry(p, l, r, wmode);
711
2c8c53e2 712 build_restore_pagemask(p, r, tmp, label_leave, restore_scratch);
6dd9344c
DD		 713}
714
fd062c84
DD		 715/*
716 * Check if Huge PTE is present, if so then jump to LABEL.
717 */
078a55fc 718static void
fd062c84 719build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
078a55fc 720 unsigned int pmd, int lid)
fd062c84
DD		 721{
722 UASM_i_LW(p, tmp, 0, pmd);
cc33ae43
DD		 723 if (use_bbit_insns()) {
724 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
725 } else {
726 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
727 uasm_il_bnez(p, r, tmp, lid);
728 }
fd062c84
DD		 729}
730
078a55fc
PG		 731static void build_huge_update_entries(u32 **p, unsigned int pte,
732 unsigned int tmp)
fd062c84
DD		 733{
734 int small_sequence;
735
736 /*
737 * A huge PTE describes an area the size of the
738 * configured huge page size. This is twice the
739 * of the large TLB entry size we intend to use.
740 * A TLB entry half the size of the configured
741 * huge page size is configured into entrylo0
742 * and entrylo1 to cover the contiguous huge PTE
743 * address space.
744 */
745 small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
746
70342287 747 /* We can clobber tmp. It isn't used after this.*/
fd062c84
DD		 748 if (!small_sequence)
749 uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
750
6dd9344c 751 build_convert_pte_to_entrylo(p, pte);
9b8c3891 752 UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
fd062c84
DD		 753 /* convert to entrylo1 */
754 if (small_sequence)
755 UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
756 else
757 UASM_i_ADDU(p, pte, pte, tmp);
758
9b8c3891 759 UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
fd062c84
DD		 760}
761
078a55fc
PG		 762static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
763 struct uasm_label **l,
764 unsigned int pte,
765 unsigned int ptr)
fd062c84
DD		 766{
767#ifdef CONFIG_SMP
768 UASM_i_SC(p, pte, 0, ptr);
769 uasm_il_beqz(p, r, pte, label_tlb_huge_update);
770 UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
771#else
772 UASM_i_SW(p, pte, 0, ptr);
773#endif
774 build_huge_update_entries(p, pte, ptr);
2c8c53e2 775 build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
fd062c84 776}
aa1762f4 777#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
fd062c84 778
875d43e7 779#ifdef CONFIG_64BIT
1da177e4
LT		 780/*
781 * TMP and PTR are scratch.
782 * TMP will be clobbered, PTR will hold the pmd entry.
783 */
078a55fc 784static void
e30ec452 785build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1da177e4
LT		 786 unsigned int tmp, unsigned int ptr)
787{
82622284 788#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1da177e4 789 long pgdc = (long)pgd_current;
82622284 790#endif
1da177e4
LT		 791 /*
792 * The vmalloc handling is not in the hotpath.
793 */
e30ec452 794 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
1ec56329
DD		 795
796 if (check_for_high_segbits) {
797 /*
798 * The kernel currently implicitely assumes that the
799 * MIPS SEGBITS parameter for the processor is
800 * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
801 * allocate virtual addresses outside the maximum
802 * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
803 * that doesn't prevent user code from accessing the
804 * higher xuseg addresses. Here, we make sure that
805 * everything but the lower xuseg addresses goes down
806 * the module_alloc/vmalloc path.
807 */
808 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
809 uasm_il_bnez(p, r, ptr, label_vmalloc);
810 } else {
811 uasm_il_bltz(p, r, tmp, label_vmalloc);
812 }
e30ec452 813 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
1da177e4 814
3d8bfdd0
DD		 815 if (pgd_reg != -1) {
816 /* pgd is in pgd_reg */
380cd582
HC		 817 if (cpu_has_ldpte)
818 UASM_i_MFC0(p, ptr, C0_PWBASE);
819 else
820 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
3d8bfdd0 821 } else {
f4ae17aa 822#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
3d8bfdd0
DD		 823 /*
824 * &pgd << 11 stored in CONTEXT [23..63].
825 */
826 UASM_i_MFC0(p, ptr, C0_CONTEXT);
827
828 /* Clear lower 23 bits of context. */
829 uasm_i_dins(p, ptr, 0, 0, 23);
830
70342287 831 /* 1 0 1 0 1 << 6 xkphys cached */
3d8bfdd0
DD		 832 uasm_i_ori(p, ptr, ptr, 0x540);
833 uasm_i_drotr(p, ptr, ptr, 11);
82622284 834#elif defined(CONFIG_SMP)
f4ae17aa
J		 835 UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
836 uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
837 UASM_i_LA_mostly(p, tmp, pgdc);
838 uasm_i_daddu(p, ptr, ptr, tmp);
839 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
840 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
1da177e4 841#else
f4ae17aa
J		 842 UASM_i_LA_mostly(p, ptr, pgdc);
843 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
1da177e4 844#endif
f4ae17aa 845 }
1da177e4 846
e30ec452 847 uasm_l_vmalloc_done(l, *p);
242954b5 848
3be6022c
DD		 849 /* get pgd offset in bytes */
850 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
e30ec452
TS		 851
852 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
853 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
325f8a0a 854#ifndef __PAGETABLE_PMD_FOLDED
e30ec452
TS		 855 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
856 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
3be6022c 857 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
e30ec452
TS		 858 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
859 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
325f8a0a 860#endif
1da177e4
LT		 861}
862
863/*
864 * BVADDR is the faulting address, PTR is scratch.
865 * PTR will hold the pgd for vmalloc.
866 */
078a55fc 867static void
e30ec452 868build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1ec56329
DD		 869 unsigned int bvaddr, unsigned int ptr,
870 enum vmalloc64_mode mode)
1da177e4
LT		 871{
872 long swpd = (long)swapper_pg_dir;
1ec56329
DD		 873 int single_insn_swpd;
874 int did_vmalloc_branch = 0;
875
876 single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
1da177e4 877
e30ec452 878 uasm_l_vmalloc(l, *p);
1da177e4 879
2c8c53e2 880 if (mode != not_refill && check_for_high_segbits) {
1ec56329
DD		 881 if (single_insn_swpd) {
882 uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
883 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
884 did_vmalloc_branch = 1;
885 /* fall through */
886 } else {
887 uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
888 }
889 }
890 if (!did_vmalloc_branch) {
2f8f8c04 891 if (single_insn_swpd) {
1ec56329
DD		 892 uasm_il_b(p, r, label_vmalloc_done);
893 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
894 } else {
895 UASM_i_LA_mostly(p, ptr, swpd);
896 uasm_il_b(p, r, label_vmalloc_done);
897 if (uasm_in_compat_space_p(swpd))
898 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
899 else
900 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
901 }
902 }
2c8c53e2 903 if (mode != not_refill && check_for_high_segbits) {
1ec56329
DD		 904 uasm_l_large_segbits_fault(l, *p);
905 /*
906 * We get here if we are an xsseg address, or if we are
907 * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
908 *
909 * Ignoring xsseg (assume disabled so would generate
910 * (address errors?), the only remaining possibility
911 * is the upper xuseg addresses. On processors with
912 * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
913 * addresses would have taken an address error. We try
914 * to mimic that here by taking a load/istream page
915 * fault.
916 */
917 UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
918 uasm_i_jr(p, ptr);
2c8c53e2
DD		 919
920 if (mode == refill_scratch) {
0e6ecc1a 921 if (scratch_reg >= 0)
7777b939 922 UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
2c8c53e2
DD		 923 else
924 UASM_i_LW(p, 1, scratchpad_offset(0), 0);
925 } else {
926 uasm_i_nop(p);
927 }
1da177e4
LT		 928 }
929}
930
875d43e7 931#else /* !CONFIG_64BIT */
1da177e4
LT		 932
933/*
934 * TMP and PTR are scratch.
935 * TMP will be clobbered, PTR will hold the pgd entry.
936 */
078a55fc 937static void __maybe_unused
1da177e4
LT		 938build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
939{
f4ae17aa
J		 940 if (pgd_reg != -1) {
941 /* pgd is in pgd_reg */
942 uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
943 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
944 } else {
945 long pgdc = (long)pgd_current;
1da177e4 946
f4ae17aa 947 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
1da177e4 948#ifdef CONFIG_SMP
f4ae17aa
J		 949 uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
950 UASM_i_LA_mostly(p, tmp, pgdc);
951 uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
952 uasm_i_addu(p, ptr, tmp, ptr);
1da177e4 953#else
f4ae17aa 954 UASM_i_LA_mostly(p, ptr, pgdc);
1da177e4 955#endif
f4ae17aa
J		 956 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
957 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
958 }
e30ec452
TS		 959 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
960 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
961 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
1da177e4
LT		 962}
963
875d43e7 964#endif /* !CONFIG_64BIT */
1da177e4 965
078a55fc 966static void build_adjust_context(u32 **p, unsigned int ctx)
1da177e4 967{
242954b5 968 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
1da177e4
LT		 969 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
970
10cc3529 971 switch (current_cpu_type()) {
1da177e4
LT		 972 case CPU_VR41XX:
973 case CPU_VR4111:
974 case CPU_VR4121:
975 case CPU_VR4122:
976 case CPU_VR4131:
977 case CPU_VR4181:
978 case CPU_VR4181A:
979 case CPU_VR4133:
980 shift += 2;
981 break;
982
983 default:
984 break;
985 }
986
987 if (shift)
e30ec452
TS		 988 UASM_i_SRL(p, ctx, ctx, shift);
989 uasm_i_andi(p, ctx, ctx, mask);
1da177e4
LT		 990}
991
078a55fc 992static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1da177e4
LT		 993{
994 /*
995 * Bug workaround for the Nevada. It seems as if under certain
996 * circumstances the move from cp0_context might produce a
997 * bogus result when the mfc0 instruction and its consumer are
998 * in a different cacheline or a load instruction, probably any
999 * memory reference, is between them.
1000 */
10cc3529 1001 switch (current_cpu_type()) {
1da177e4 1002 case CPU_NEVADA:
e30ec452 1003 UASM_i_LW(p, ptr, 0, ptr);
1da177e4
LT		 1004 GET_CONTEXT(p, tmp); /* get context reg */
1005 break;
1006
1007 default:
1008 GET_CONTEXT(p, tmp); /* get context reg */
e30ec452 1009 UASM_i_LW(p, ptr, 0, ptr);
1da177e4
LT		 1010 break;
1011 }
1012
1013 build_adjust_context(p, tmp);
e30ec452 1014 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1da177e4
LT		 1015}
1016
078a55fc 1017static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
1da177e4 1018{
2caa89b4
PB		 1019 int pte_off_even = 0;
1020 int pte_off_odd = sizeof(pte_t);
7b2cb64f 1021
2caa89b4
PB		 1022#if defined(CONFIG_CPU_MIPS32) && defined(CONFIG_PHYS_ADDR_T_64BIT)
1023 /* The low 32 bits of EntryLo is stored in pte_high */
1024 pte_off_even += offsetof(pte_t, pte_high);
1025 pte_off_odd += offsetof(pte_t, pte_high);
1026#endif
1027
97f2645f 1028 if (IS_ENABLED(CONFIG_XPA)) {
c5b36783 1029 uasm_i_lw(p, tmp, pte_off_even, ptep); /* even pte */
c5b36783 1030 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
c5b36783 1031 UASM_i_MTC0(p, tmp, C0_ENTRYLO0);
7b2cb64f 1032
4b6f99d3
JH		 1033 if (cpu_has_xpa && !mips_xpa_disabled) {
1034 uasm_i_lw(p, tmp, 0, ptep);
1035 uasm_i_ext(p, tmp, tmp, 0, 24);
1036 uasm_i_mthc0(p, tmp, C0_ENTRYLO0);
1037 }
f3832196
JH		 1038
1039 uasm_i_lw(p, tmp, pte_off_odd, ptep); /* odd pte */
1040 UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
1041 UASM_i_MTC0(p, tmp, C0_ENTRYLO1);
1042
4b6f99d3
JH		 1043 if (cpu_has_xpa && !mips_xpa_disabled) {
1044 uasm_i_lw(p, tmp, sizeof(pte_t), ptep);
1045 uasm_i_ext(p, tmp, tmp, 0, 24);
1046 uasm_i_mthc0(p, tmp, C0_ENTRYLO1);
1047 }
7b2cb64f
PB		 1048 return;
1049 }
1050
2caa89b4
PB		 1051 UASM_i_LW(p, tmp, pte_off_even, ptep); /* get even pte */
1052 UASM_i_LW(p, ptep, pte_off_odd, ptep); /* get odd pte */
1da177e4
LT		 1053 if (r45k_bvahwbug())
1054 build_tlb_probe_entry(p);
974a0b6a
PB		 1055 build_convert_pte_to_entrylo(p, tmp);
1056 if (r4k_250MHZhwbug())
1057 UASM_i_MTC0(p, 0, C0_ENTRYLO0);
1058 UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
1059 build_convert_pte_to_entrylo(p, ptep);
1060 if (r45k_bvahwbug())
1061 uasm_i_mfc0(p, tmp, C0_INDEX);
1da177e4 1062 if (r4k_250MHZhwbug())
9b8c3891
DD		 1063 UASM_i_MTC0(p, 0, C0_ENTRYLO1);
1064 UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
1da177e4
LT		 1065}
1066
2c8c53e2
DD		 1067struct mips_huge_tlb_info {
1068 int huge_pte;
1069 int restore_scratch;
9e0f162a 1070 bool need_reload_pte;
2c8c53e2
DD		 1071};
1072
078a55fc 1073static struct mips_huge_tlb_info
2c8c53e2
DD		 1074build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
1075 struct uasm_reloc **r, unsigned int tmp,
7777b939 1076 unsigned int ptr, int c0_scratch_reg)
2c8c53e2
DD		 1077{
1078 struct mips_huge_tlb_info rv;
1079 unsigned int even, odd;
1080 int vmalloc_branch_delay_filled = 0;
1081 const int scratch = 1; /* Our extra working register */
1082
1083 rv.huge_pte = scratch;
1084 rv.restore_scratch = 0;
9e0f162a 1085 rv.need_reload_pte = false;
2c8c53e2
DD		 1086
1087 if (check_for_high_segbits) {
1088 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1089
1090 if (pgd_reg != -1)
7777b939 1091 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
2c8c53e2
DD		 1092 else
1093 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1094
7777b939
J		 1095 if (c0_scratch_reg >= 0)
1096 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
2c8c53e2
DD		 1097 else
1098 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1099
1100 uasm_i_dsrl_safe(p, scratch, tmp,
1101 PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1102 uasm_il_bnez(p, r, scratch, label_vmalloc);
1103
1104 if (pgd_reg == -1) {
1105 vmalloc_branch_delay_filled = 1;
1106 /* Clear lower 23 bits of context. */
1107 uasm_i_dins(p, ptr, 0, 0, 23);
1108 }
1109 } else {
1110 if (pgd_reg != -1)
7777b939 1111 UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
2c8c53e2
DD		 1112 else
1113 UASM_i_MFC0(p, ptr, C0_CONTEXT);
1114
1115 UASM_i_MFC0(p, tmp, C0_BADVADDR);
1116
7777b939
J		 1117 if (c0_scratch_reg >= 0)
1118 UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
2c8c53e2
DD		 1119 else
1120 UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
1121
1122 if (pgd_reg == -1)
1123 /* Clear lower 23 bits of context. */
1124 uasm_i_dins(p, ptr, 0, 0, 23);
1125
1126 uasm_il_bltz(p, r, tmp, label_vmalloc);
1127 }
1128
1129 if (pgd_reg == -1) {
1130 vmalloc_branch_delay_filled = 1;
70342287 1131 /* 1 0 1 0 1 << 6 xkphys cached */
2c8c53e2
DD		 1132 uasm_i_ori(p, ptr, ptr, 0x540);
1133 uasm_i_drotr(p, ptr, ptr, 11);
1134 }
1135
1136#ifdef __PAGETABLE_PMD_FOLDED
1137#define LOC_PTEP scratch
1138#else
1139#define LOC_PTEP ptr
1140#endif
1141
1142 if (!vmalloc_branch_delay_filled)
1143 /* get pgd offset in bytes */
1144 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1145
1146 uasm_l_vmalloc_done(l, *p);
1147
1148 /*
70342287
RB		 1149 * tmp ptr
1150 * fall-through case = badvaddr *pgd_current
1151 * vmalloc case = badvaddr swapper_pg_dir
2c8c53e2
DD		 1152 */
1153
1154 if (vmalloc_branch_delay_filled)
1155 /* get pgd offset in bytes */
1156 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);
1157
1158#ifdef __PAGETABLE_PMD_FOLDED
1159 GET_CONTEXT(p, tmp); /* get context reg */
1160#endif
1161 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);
1162
1163 if (use_lwx_insns()) {
1164 UASM_i_LWX(p, LOC_PTEP, scratch, ptr);
1165 } else {
1166 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */
1167 uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */
1168 }
1169
1170#ifndef __PAGETABLE_PMD_FOLDED
1171 /* get pmd offset in bytes */
1172 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);
1173 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);
1174 GET_CONTEXT(p, tmp); /* get context reg */
1175
1176 if (use_lwx_insns()) {
1177 UASM_i_LWX(p, scratch, scratch, ptr);
1178 } else {
1179 uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */
1180 UASM_i_LW(p, scratch, 0, ptr);
1181 }
1182#endif
1183 /* Adjust the context during the load latency. */
1184 build_adjust_context(p, tmp);
1185
aa1762f4 1186#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2c8c53e2
DD		 1187 uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
1188 /*
1189 * The in the LWX case we don't want to do the load in the
70342287 1190 * delay slot. It cannot issue in the same cycle and may be
2c8c53e2
DD		 1191 * speculative and unneeded.
1192 */
1193 if (use_lwx_insns())
1194 uasm_i_nop(p);
aa1762f4 1195#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
2c8c53e2
DD		 1196
1197
1198 /* build_update_entries */
1199 if (use_lwx_insns()) {
1200 even = ptr;
1201 odd = tmp;
1202 UASM_i_LWX(p, even, scratch, tmp);
1203 UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t));
1204 UASM_i_LWX(p, odd, scratch, tmp);
1205 } else {
1206 UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */
1207 even = tmp;
1208 odd = ptr;
1209 UASM_i_LW(p, even, 0, ptr); /* get even pte */
1210 UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
1211 }
05857c64 1212 if (cpu_has_rixi) {
748e787e 1213 uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
2c8c53e2 1214 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
748e787e 1215 uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
2c8c53e2
DD		 1216 } else {
1217 uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
1218 UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
1219 uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL));
1220 }
1221 UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
1222
7777b939
J		 1223 if (c0_scratch_reg >= 0) {
1224 UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
2c8c53e2
DD		 1225 build_tlb_write_entry(p, l, r, tlb_random);
1226 uasm_l_leave(l, *p);
1227 rv.restore_scratch = 1;
1228 } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) {
1229 build_tlb_write_entry(p, l, r, tlb_random);
1230 uasm_l_leave(l, *p);
1231 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1232 } else {
1233 UASM_i_LW(p, scratch, scratchpad_offset(0), 0);
1234 build_tlb_write_entry(p, l, r, tlb_random);
1235 uasm_l_leave(l, *p);
1236 rv.restore_scratch = 1;
1237 }
1238
1239 uasm_i_eret(p); /* return from trap */
1240
1241 return rv;
1242}
1243
e6f72d3a
DD		 1244/*
1245 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1246 * because EXL == 0. If we wrap, we can also use the 32 instruction
1247 * slots before the XTLB refill exception handler which belong to the
1248 * unused TLB refill exception.
1249 */
1250#define MIPS64_REFILL_INSNS 32
1251
078a55fc 1252static void build_r4000_tlb_refill_handler(void)
1da177e4
LT		 1253{
1254 u32 *p = tlb_handler;
e30ec452
TS		 1255 struct uasm_label *l = labels;
1256 struct uasm_reloc *r = relocs;
1da177e4
LT		 1257 u32 *f;
1258 unsigned int final_len;
4a9040f4
RB		 1259 struct mips_huge_tlb_info htlb_info __maybe_unused;
1260 enum vmalloc64_mode vmalloc_mode __maybe_unused;
18280eda 1261
1da177e4
LT		 1262 memset(tlb_handler, 0, sizeof(tlb_handler));
1263 memset(labels, 0, sizeof(labels));
1264 memset(relocs, 0, sizeof(relocs));
1265 memset(final_handler, 0, sizeof(final_handler));
1266
18280eda 1267 if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
2c8c53e2
DD		 1268 htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
1269 scratch_reg);
1270 vmalloc_mode = refill_scratch;
1271 } else {
1272 htlb_info.huge_pte = K0;
1273 htlb_info.restore_scratch = 0;
9e0f162a 1274 htlb_info.need_reload_pte = true;
2c8c53e2
DD		 1275 vmalloc_mode = refill_noscratch;
1276 /*
1277 * create the plain linear handler
1278 */
1279 if (bcm1250_m3_war()) {
1280 unsigned int segbits = 44;
1281
1282 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1283 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
1284 uasm_i_xor(&p, K0, K0, K1);
1285 uasm_i_dsrl_safe(&p, K1, K0, 62);
1286 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
1287 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
1288 uasm_i_or(&p, K0, K0, K1);
1289 uasm_il_bnez(&p, &r, K0, label_leave);
1290 /* No need for uasm_i_nop */
1291 }
1da177e4 1292
875d43e7 1293#ifdef CONFIG_64BIT
2c8c53e2 1294 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1da177e4 1295#else
2c8c53e2 1296 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1da177e4
LT		 1297#endif
1298
aa1762f4 1299#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
2c8c53e2 1300 build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
fd062c84
DD		 1301#endif
1302
2c8c53e2
DD		 1303 build_get_ptep(&p, K0, K1);
1304 build_update_entries(&p, K0, K1);
1305 build_tlb_write_entry(&p, &l, &r, tlb_random);
1306 uasm_l_leave(&l, p);
1307 uasm_i_eret(&p); /* return from trap */
1308 }
aa1762f4 1309#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84 1310 uasm_l_tlb_huge_update(&l, p);
9e0f162a
DD		 1311 if (htlb_info.need_reload_pte)
1312 UASM_i_LW(&p, htlb_info.huge_pte, 0, K1);
2c8c53e2
DD		 1313 build_huge_update_entries(&p, htlb_info.huge_pte, K1);
1314 build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
1315 htlb_info.restore_scratch);
fd062c84
DD		 1316#endif
1317
875d43e7 1318#ifdef CONFIG_64BIT
2c8c53e2 1319 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode);
1da177e4
LT		 1320#endif
1321
1322 /*
1323 * Overflow check: For the 64bit handler, we need at least one
1324 * free instruction slot for the wrap-around branch. In worst
1325 * case, if the intended insertion point is a delay slot, we
4b3f686d 1326 * need three, with the second nop'ed and the third being
1da177e4
LT		 1327 * unused.
1328 */
14bd8c08
RB		 1329 switch (boot_cpu_type()) {
1330 default:
1331 if (sizeof(long) == 4) {
1332 case CPU_LOONGSON2:
1333 /* Loongson2 ebase is different than r4k, we have more space */
1334 if ((p - tlb_handler) > 64)
1335 panic("TLB refill handler space exceeded");
95affdda 1336 /*
14bd8c08 1337 * Now fold the handler in the TLB refill handler space.
95affdda 1338 */
14bd8c08
RB		 1339 f = final_handler;
1340 /* Simplest case, just copy the handler. */
1341 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1342 final_len = p - tlb_handler;
1343 break;
1344 } else {
1345 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
1346 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
1347 && uasm_insn_has_bdelay(relocs,
1348 tlb_handler + MIPS64_REFILL_INSNS - 3)))
1349 panic("TLB refill handler space exceeded");
95affdda 1350 /*
14bd8c08 1351 * Now fold the handler in the TLB refill handler space.
95affdda 1352 */
14bd8c08
RB		 1353 f = final_handler + MIPS64_REFILL_INSNS;
1354 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
1355 /* Just copy the handler. */
1356 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1357 final_len = p - tlb_handler;
1358 } else {
1359#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1360 const enum label_id ls = label_tlb_huge_update;
1361#else
1362 const enum label_id ls = label_vmalloc;
1363#endif
1364 u32 *split;
1365 int ov = 0;
1366 int i;
1367
1368 for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
1369 ;
1370 BUG_ON(i == ARRAY_SIZE(labels));
1371 split = labels[i].addr;
1372
1373 /*
1374 * See if we have overflown one way or the other.
1375 */
1376 if (split > tlb_handler + MIPS64_REFILL_INSNS ||
1377 split < p - MIPS64_REFILL_INSNS)
1378 ov = 1;
1379
1380 if (ov) {
1381 /*
1382 * Split two instructions before the end. One
1383 * for the branch and one for the instruction
1384 * in the delay slot.
1385 */
1386 split = tlb_handler + MIPS64_REFILL_INSNS - 2;
1387
1388 /*
1389 * If the branch would fall in a delay slot,
1390 * we must back up an additional instruction
1391 * so that it is no longer in a delay slot.
1392 */
1393 if (uasm_insn_has_bdelay(relocs, split - 1))
1394 split--;
1395 }
1396 /* Copy first part of the handler. */
1397 uasm_copy_handler(relocs, labels, tlb_handler, split, f);
1398 f += split - tlb_handler;
1399
1400 if (ov) {
1401 /* Insert branch. */
1402 uasm_l_split(&l, final_handler);
1403 uasm_il_b(&f, &r, label_split);
1404 if (uasm_insn_has_bdelay(relocs, split))
1405 uasm_i_nop(&f);
1406 else {
1407 uasm_copy_handler(relocs, labels,
1408 split, split + 1, f);
1409 uasm_move_labels(labels, f, f + 1, -1);
1410 f++;
1411 split++;
1412 }
1413 }
1414
1415 /* Copy the rest of the handler. */
1416 uasm_copy_handler(relocs, labels, split, p, final_handler);
1417 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
1418 (p - split);
95affdda 1419 }
1da177e4 1420 }
14bd8c08 1421 break;
1da177e4 1422 }
1da177e4 1423
e30ec452
TS		 1424 uasm_resolve_relocs(relocs, labels);
1425 pr_debug("Wrote TLB refill handler (%u instructions).\n",
1426 final_len);
1da177e4 1427
91b05e67 1428 memcpy((void *)ebase, final_handler, 0x100);
1062080a 1429 local_flush_icache_range(ebase, ebase + 0x100);
92b1e6a6 1430
a2c763e0 1431 dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
1da177e4
LT		 1432}
1433
380cd582
HC		 1434static void setup_pw(void)
1435{
1436 unsigned long pgd_i, pgd_w;
1437#ifndef __PAGETABLE_PMD_FOLDED
1438 unsigned long pmd_i, pmd_w;
1439#endif
1440 unsigned long pt_i, pt_w;
1441 unsigned long pte_i, pte_w;
1442#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1443 unsigned long psn;
1444
1445 psn = ilog2(_PAGE_HUGE); /* bit used to indicate huge page */
1446#endif
1447 pgd_i = PGDIR_SHIFT; /* 1st level PGD */
1448#ifndef __PAGETABLE_PMD_FOLDED
1449 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_ORDER;
1450
1451 pmd_i = PMD_SHIFT; /* 2nd level PMD */
1452 pmd_w = PMD_SHIFT - PAGE_SHIFT;
1453#else
1454 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_ORDER;
1455#endif
1456
1457 pt_i = PAGE_SHIFT; /* 3rd level PTE */
1458 pt_w = PAGE_SHIFT - 3;
1459
1460 pte_i = ilog2(_PAGE_GLOBAL);
1461 pte_w = 0;
1462
1463#ifndef __PAGETABLE_PMD_FOLDED
1464 write_c0_pwfield(pgd_i << 24 | pmd_i << 12 | pt_i << 6 | pte_i);
1465 write_c0_pwsize(1 << 30 | pgd_w << 24 | pmd_w << 12 | pt_w << 6 | pte_w);
1466#else
1467 write_c0_pwfield(pgd_i << 24 | pt_i << 6 | pte_i);
1468 write_c0_pwsize(1 << 30 | pgd_w << 24 | pt_w << 6 | pte_w);
1469#endif
1470
1471#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
1472 write_c0_pwctl(1 << 6 | psn);
1473#endif
1474 write_c0_kpgd(swapper_pg_dir);
1475 kscratch_used_mask |= (1 << 7); /* KScratch6 is used for KPGD */
1476}
1477
1478static void build_loongson3_tlb_refill_handler(void)
1479{
1480 u32 *p = tlb_handler;
1481 struct uasm_label *l = labels;
1482 struct uasm_reloc *r = relocs;
1483
1484 memset(labels, 0, sizeof(labels));
1485 memset(relocs, 0, sizeof(relocs));
1486 memset(tlb_handler, 0, sizeof(tlb_handler));
1487
1488 if (check_for_high_segbits) {
1489 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
1490 uasm_i_dsrl_safe(&p, K1, K0, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
1491 uasm_il_beqz(&p, &r, K1, label_vmalloc);
1492 uasm_i_nop(&p);
1493
1494 uasm_il_bgez(&p, &r, K0, label_large_segbits_fault);
1495 uasm_i_nop(&p);
1496 uasm_l_vmalloc(&l, p);
1497 }
1498
1499 uasm_i_dmfc0(&p, K1, C0_PGD);
1500
1501 uasm_i_lddir(&p, K0, K1, 3); /* global page dir */
1502#ifndef __PAGETABLE_PMD_FOLDED
1503 uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */
1504#endif
1505 uasm_i_ldpte(&p, K1, 0); /* even */
1506 uasm_i_ldpte(&p, K1, 1); /* odd */
1507 uasm_i_tlbwr(&p);
1508
1509 /* restore page mask */
1510 if (PM_DEFAULT_MASK >> 16) {
1511 uasm_i_lui(&p, K0, PM_DEFAULT_MASK >> 16);
1512 uasm_i_ori(&p, K0, K0, PM_DEFAULT_MASK & 0xffff);
1513 uasm_i_mtc0(&p, K0, C0_PAGEMASK);
1514 } else if (PM_DEFAULT_MASK) {
1515 uasm_i_ori(&p, K0, 0, PM_DEFAULT_MASK);
1516 uasm_i_mtc0(&p, K0, C0_PAGEMASK);
1517 } else {
1518 uasm_i_mtc0(&p, 0, C0_PAGEMASK);
1519 }
1520
1521 uasm_i_eret(&p);
1522
1523 if (check_for_high_segbits) {
1524 uasm_l_large_segbits_fault(&l, p);
1525 UASM_i_LA(&p, K1, (unsigned long)tlb_do_page_fault_0);
1526 uasm_i_jr(&p, K1);
1527 uasm_i_nop(&p);
1528 }
1529
1530 uasm_resolve_relocs(relocs, labels);
1531 memcpy((void *)(ebase + 0x80), tlb_handler, 0x80);
1532 local_flush_icache_range(ebase + 0x80, ebase + 0x100);
1533 dump_handler("loongson3_tlb_refill", (u32 *)(ebase + 0x80), 32);
1534}
1535
6ba045f9
J		 1536extern u32 handle_tlbl[], handle_tlbl_end[];
1537extern u32 handle_tlbs[], handle_tlbs_end[];
1538extern u32 handle_tlbm[], handle_tlbm_end[];
7bb39409
SH		 1539extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[];
1540extern u32 tlbmiss_handler_setup_pgd_end[];
3d8bfdd0 1541
f4ae17aa 1542static void build_setup_pgd(void)
3d8bfdd0
DD		 1543{
1544 const int a0 = 4;
f4ae17aa
J		 1545 const int __maybe_unused a1 = 5;
1546 const int __maybe_unused a2 = 6;
7bb39409 1547 u32 *p = tlbmiss_handler_setup_pgd_start;
6ba045f9 1548 const int tlbmiss_handler_setup_pgd_size =
7bb39409 1549 tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
f4ae17aa
J		 1550#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
1551 long pgdc = (long)pgd_current;
1552#endif
3d8bfdd0 1553
6ba045f9
J		 1554 memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
1555 sizeof(tlbmiss_handler_setup_pgd[0]));
3d8bfdd0
DD		 1556 memset(labels, 0, sizeof(labels));
1557 memset(relocs, 0, sizeof(relocs));
3d8bfdd0 1558 pgd_reg = allocate_kscratch();
f4ae17aa 1559#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
3d8bfdd0 1560 if (pgd_reg == -1) {
f4ae17aa
J		 1561 struct uasm_label *l = labels;
1562 struct uasm_reloc *r = relocs;
1563
3d8bfdd0
DD		 1564 /* PGD << 11 in c0_Context */
1565 /*
1566 * If it is a ckseg0 address, convert to a physical
1567 * address. Shifting right by 29 and adding 4 will
1568 * result in zero for these addresses.
1569 *
1570 */
1571 UASM_i_SRA(&p, a1, a0, 29);
1572 UASM_i_ADDIU(&p, a1, a1, 4);
1573 uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1);
1574 uasm_i_nop(&p);
1575 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);
1576 uasm_l_tlbl_goaround1(&l, p);
1577 UASM_i_SLL(&p, a0, a0, 11);
1578 uasm_i_jr(&p, 31);
1579 UASM_i_MTC0(&p, a0, C0_CONTEXT);
1580 } else {
1581 /* PGD in c0_KScratch */
1582 uasm_i_jr(&p, 31);
380cd582
HC		 1583 if (cpu_has_ldpte)
1584 UASM_i_MTC0(&p, a0, C0_PWBASE);
1585 else
1586 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
3d8bfdd0 1587 }
f4ae17aa
J		 1588#else
1589#ifdef CONFIG_SMP
1590 /* Save PGD to pgd_current[smp_processor_id()] */
1591 UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
1592 UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
1593 UASM_i_LA_mostly(&p, a2, pgdc);
1594 UASM_i_ADDU(&p, a2, a2, a1);
1595 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1596#else
1597 UASM_i_LA_mostly(&p, a2, pgdc);
1598 UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
1599#endif /* SMP */
1600 uasm_i_jr(&p, 31);
1601
1602 /* if pgd_reg is allocated, save PGD also to scratch register */
1603 if (pgd_reg != -1)
1604 UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
1605 else
1606 uasm_i_nop(&p);
1607#endif
6ba045f9
J		 1608 if (p >= tlbmiss_handler_setup_pgd_end)
1609 panic("tlbmiss_handler_setup_pgd space exceeded");
1610
3d8bfdd0 1611 uasm_resolve_relocs(relocs, labels);
6ba045f9
J		 1612 pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
1613 (unsigned int)(p - tlbmiss_handler_setup_pgd));
3d8bfdd0 1614
6ba045f9
J		 1615 dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
1616 tlbmiss_handler_setup_pgd_size);
3d8bfdd0 1617}
1da177e4 1618
078a55fc 1619static void
bd1437e4 1620iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
1da177e4
LT		 1621{
1622#ifdef CONFIG_SMP
34adb28d 1623# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1624 if (cpu_has_64bits)
e30ec452 1625 uasm_i_lld(p, pte, 0, ptr);
1da177e4
LT		 1626 else
1627# endif
e30ec452 1628 UASM_i_LL(p, pte, 0, ptr);
1da177e4 1629#else
34adb28d 1630# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1631 if (cpu_has_64bits)
e30ec452 1632 uasm_i_ld(p, pte, 0, ptr);
1da177e4
LT		 1633 else
1634# endif
e30ec452 1635 UASM_i_LW(p, pte, 0, ptr);
1da177e4
LT		 1636#endif
1637}
1638
078a55fc 1639static void
e30ec452 1640iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
bbeeffec 1641 unsigned int mode, unsigned int scratch)
1da177e4 1642{
63b2d2f4 1643 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
b4ebbb87 1644 unsigned int swmode = mode & ~hwmode;
63b2d2f4 1645
97f2645f 1646 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_64bits) {
b4ebbb87 1647 uasm_i_lui(p, scratch, swmode >> 16);
c5b36783 1648 uasm_i_or(p, pte, pte, scratch);
b4ebbb87
PB		 1649 BUG_ON(swmode & 0xffff);
1650 } else {
1651 uasm_i_ori(p, pte, pte, mode);
1652 }
1653
1da177e4 1654#ifdef CONFIG_SMP
34adb28d 1655# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1656 if (cpu_has_64bits)
e30ec452 1657 uasm_i_scd(p, pte, 0, ptr);
1da177e4
LT		 1658 else
1659# endif
e30ec452 1660 UASM_i_SC(p, pte, 0, ptr);
1da177e4
LT		 1661
1662 if (r10000_llsc_war())
e30ec452 1663 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
1da177e4 1664 else
e30ec452 1665 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1da177e4 1666
34adb28d 1667# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1668 if (!cpu_has_64bits) {
e30ec452
TS		 1669 /* no uasm_i_nop needed */
1670 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1671 uasm_i_ori(p, pte, pte, hwmode);
b4ebbb87 1672 BUG_ON(hwmode & ~0xffff);
e30ec452
TS		 1673 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1674 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1675 /* no uasm_i_nop needed */
1676 uasm_i_lw(p, pte, 0, ptr);
1da177e4 1677 } else
e30ec452 1678 uasm_i_nop(p);
1da177e4 1679# else
e30ec452 1680 uasm_i_nop(p);
1da177e4
LT		 1681# endif
1682#else
34adb28d 1683# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1684 if (cpu_has_64bits)
e30ec452 1685 uasm_i_sd(p, pte, 0, ptr);
1da177e4
LT		 1686 else
1687# endif
e30ec452 1688 UASM_i_SW(p, pte, 0, ptr);
1da177e4 1689
34adb28d 1690# ifdef CONFIG_PHYS_ADDR_T_64BIT
1da177e4 1691 if (!cpu_has_64bits) {
e30ec452
TS		 1692 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1693 uasm_i_ori(p, pte, pte, hwmode);
b4ebbb87 1694 BUG_ON(hwmode & ~0xffff);
e30ec452
TS		 1695 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1696 uasm_i_lw(p, pte, 0, ptr);
1da177e4
LT		 1697 }
1698# endif
1699#endif
1700}
1701
1702/*
1703 * Check if PTE is present, if not then jump to LABEL. PTR points to
1704 * the page table where this PTE is located, PTE will be re-loaded
1705 * with it's original value.
1706 */
078a55fc 1707static void
bd1437e4 1708build_pte_present(u32 **p, struct uasm_reloc **r,
bf28607f 1709 int pte, int ptr, int scratch, enum label_id lid)
1da177e4 1710{
bf28607f 1711 int t = scratch >= 0 ? scratch : pte;
8fe4908b 1712 int cur = pte;
bf28607f 1713
05857c64 1714 if (cpu_has_rixi) {
cc33ae43
DD		 1715 if (use_bbit_insns()) {
1716 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1717 uasm_i_nop(p);
1718 } else {
8fe4908b
JH		 1719 if (_PAGE_PRESENT_SHIFT) {
1720 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1721 cur = t;
1722 }
1723 uasm_i_andi(p, t, cur, 1);
bf28607f
DD		 1724 uasm_il_beqz(p, r, t, lid);
1725 if (pte == t)
1726 /* You lose the SMP race :-(*/
1727 iPTE_LW(p, pte, ptr);
cc33ae43 1728 }
6dd9344c 1729 } else {
8fe4908b
JH		 1730 if (_PAGE_PRESENT_SHIFT) {
1731 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1732 cur = t;
1733 }
1734 uasm_i_andi(p, t, cur,
780602d7
PB		 1735 (_PAGE_PRESENT | _PAGE_NO_READ) >> _PAGE_PRESENT_SHIFT);
1736 uasm_i_xori(p, t, t, _PAGE_PRESENT >> _PAGE_PRESENT_SHIFT);
bf28607f
DD		 1737 uasm_il_bnez(p, r, t, lid);
1738 if (pte == t)
1739 /* You lose the SMP race :-(*/
1740 iPTE_LW(p, pte, ptr);
6dd9344c 1741 }
1da177e4
LT		 1742}
1743
1744/* Make PTE valid, store result in PTR. */
078a55fc 1745static void
e30ec452 1746build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
bbeeffec 1747 unsigned int ptr, unsigned int scratch)
1da177e4 1748{
63b2d2f4
TS		 1749 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1750
bbeeffec 1751 iPTE_SW(p, r, pte, ptr, mode, scratch);
1da177e4
LT		 1752}
1753
1754/*
1755 * Check if PTE can be written to, if not branch to LABEL. Regardless
1756 * restore PTE with value from PTR when done.
1757 */
078a55fc 1758static void
bd1437e4 1759build_pte_writable(u32 **p, struct uasm_reloc **r,
bf28607f
DD		 1760 unsigned int pte, unsigned int ptr, int scratch,
1761 enum label_id lid)
1da177e4 1762{
bf28607f 1763 int t = scratch >= 0 ? scratch : pte;
8fe4908b 1764 int cur = pte;
bf28607f 1765
8fe4908b
JH		 1766 if (_PAGE_PRESENT_SHIFT) {
1767 uasm_i_srl(p, t, cur, _PAGE_PRESENT_SHIFT);
1768 cur = t;
1769 }
1770 uasm_i_andi(p, t, cur,
a3ae565a
JH		 1771 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
1772 uasm_i_xori(p, t, t,
1773 (_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT);
bf28607f
DD		 1774 uasm_il_bnez(p, r, t, lid);
1775 if (pte == t)
1776 /* You lose the SMP race :-(*/
cc33ae43 1777 iPTE_LW(p, pte, ptr);
bf28607f
DD		 1778 else
1779 uasm_i_nop(p);
1da177e4
LT		 1780}
1781
1782/* Make PTE writable, update software status bits as well, then store
1783 * at PTR.
1784 */
078a55fc 1785static void
e30ec452 1786build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
bbeeffec 1787 unsigned int ptr, unsigned int scratch)
1da177e4 1788{
63b2d2f4
TS		 1789 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1790 | _PAGE_DIRTY);
1791
bbeeffec 1792 iPTE_SW(p, r, pte, ptr, mode, scratch);
1da177e4
LT		 1793}
1794
1795/*
1796 * Check if PTE can be modified, if not branch to LABEL. Regardless
1797 * restore PTE with value from PTR when done.
1798 */
078a55fc 1799static void
bd1437e4 1800build_pte_modifiable(u32 **p, struct uasm_reloc **r,
bf28607f
DD		 1801 unsigned int pte, unsigned int ptr, int scratch,
1802 enum label_id lid)
1da177e4 1803{
cc33ae43
DD		 1804 if (use_bbit_insns()) {
1805 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1806 uasm_i_nop(p);
1807 } else {
bf28607f 1808 int t = scratch >= 0 ? scratch : pte;
c5b36783
SH		 1809 uasm_i_srl(p, t, pte, _PAGE_WRITE_SHIFT);
1810 uasm_i_andi(p, t, t, 1);
bf28607f
DD		 1811 uasm_il_beqz(p, r, t, lid);
1812 if (pte == t)
1813 /* You lose the SMP race :-(*/
1814 iPTE_LW(p, pte, ptr);
cc33ae43 1815 }
1da177e4
LT		 1816}
1817
82622284 1818#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
3d8bfdd0
DD		 1819
1820
1da177e4
LT		 1821/*
1822 * R3000 style TLB load/store/modify handlers.
1823 */
1824
fded2e50
MR		 1825/*
1826 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1827 * Then it returns.
1828 */
078a55fc 1829static void
fded2e50 1830build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1da177e4 1831{
e30ec452
TS		 1832 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1833 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1834 uasm_i_tlbwi(p);
1835 uasm_i_jr(p, tmp);
1836 uasm_i_rfe(p); /* branch delay */
1da177e4
LT		 1837}
1838
1839/*
fded2e50
MR		 1840 * This places the pte into ENTRYLO0 and writes it with tlbwi
1841 * or tlbwr as appropriate. This is because the index register
1842 * may have the probe fail bit set as a result of a trap on a
1843 * kseg2 access, i.e. without refill. Then it returns.
1da177e4 1844 */
078a55fc 1845static void
e30ec452
TS		 1846build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
1847 struct uasm_reloc **r, unsigned int pte,
1848 unsigned int tmp)
1849{
1850 uasm_i_mfc0(p, tmp, C0_INDEX);
1851 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1852 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1853 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
1854 uasm_i_tlbwi(p); /* cp0 delay */
1855 uasm_i_jr(p, tmp);
1856 uasm_i_rfe(p); /* branch delay */
1857 uasm_l_r3000_write_probe_fail(l, *p);
1858 uasm_i_tlbwr(p); /* cp0 delay */
1859 uasm_i_jr(p, tmp);
1860 uasm_i_rfe(p); /* branch delay */
1da177e4
LT		 1861}
1862
078a55fc 1863static void
1da177e4
LT		 1864build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1865 unsigned int ptr)
1866{
1867 long pgdc = (long)pgd_current;
1868
e30ec452
TS		 1869 uasm_i_mfc0(p, pte, C0_BADVADDR);
1870 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
1871 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
1872 uasm_i_srl(p, pte, pte, 22); /* load delay */
1873 uasm_i_sll(p, pte, pte, 2);
1874 uasm_i_addu(p, ptr, ptr, pte);
1875 uasm_i_mfc0(p, pte, C0_CONTEXT);
1876 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
1877 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
1878 uasm_i_addu(p, ptr, ptr, pte);
1879 uasm_i_lw(p, pte, 0, ptr);
1880 uasm_i_tlbp(p); /* load delay */
1da177e4
LT		 1881}
1882
078a55fc 1883static void build_r3000_tlb_load_handler(void)
1da177e4
LT		 1884{
1885 u32 *p = handle_tlbl;
6ba045f9 1886 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
e30ec452
TS		 1887 struct uasm_label *l = labels;
1888 struct uasm_reloc *r = relocs;
1da177e4 1889
6ba045f9 1890 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1da177e4
LT		 1891 memset(labels, 0, sizeof(labels));
1892 memset(relocs, 0, sizeof(relocs));
1893
1894 build_r3000_tlbchange_handler_head(&p, K0, K1);
bf28607f 1895 build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
e30ec452 1896 uasm_i_nop(&p); /* load delay */
bbeeffec 1897 build_make_valid(&p, &r, K0, K1, -1);
fded2e50 1898 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1da177e4 1899
e30ec452
TS		 1900 uasm_l_nopage_tlbl(&l, p);
1901 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1902 uasm_i_nop(&p);
1da177e4 1903
6ba045f9 1904 if (p >= handle_tlbl_end)
1da177e4
LT		 1905 panic("TLB load handler fastpath space exceeded");
1906
e30ec452
TS		 1907 uasm_resolve_relocs(relocs, labels);
1908 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1909 (unsigned int)(p - handle_tlbl));
1da177e4 1910
6ba045f9 1911 dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
1da177e4
LT		 1912}
1913
078a55fc 1914static void build_r3000_tlb_store_handler(void)
1da177e4
LT		 1915{
1916 u32 *p = handle_tlbs;
6ba045f9 1917 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
e30ec452
TS		 1918 struct uasm_label *l = labels;
1919 struct uasm_reloc *r = relocs;
1da177e4 1920
6ba045f9 1921 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
1da177e4
LT		 1922 memset(labels, 0, sizeof(labels));
1923 memset(relocs, 0, sizeof(relocs));
1924
1925 build_r3000_tlbchange_handler_head(&p, K0, K1);
bf28607f 1926 build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
e30ec452 1927 uasm_i_nop(&p); /* load delay */
bbeeffec 1928 build_make_write(&p, &r, K0, K1, -1);
fded2e50 1929 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1da177e4 1930
e30ec452
TS		 1931 uasm_l_nopage_tlbs(&l, p);
1932 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1933 uasm_i_nop(&p);
1da177e4 1934
afc813ae 1935 if (p >= handle_tlbs_end)
1da177e4
LT		 1936 panic("TLB store handler fastpath space exceeded");
1937
e30ec452
TS		 1938 uasm_resolve_relocs(relocs, labels);
1939 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1940 (unsigned int)(p - handle_tlbs));
1da177e4 1941
6ba045f9 1942 dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
1da177e4
LT		 1943}
1944
078a55fc 1945static void build_r3000_tlb_modify_handler(void)
1da177e4
LT		 1946{
1947 u32 *p = handle_tlbm;
6ba045f9 1948 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
e30ec452
TS		 1949 struct uasm_label *l = labels;
1950 struct uasm_reloc *r = relocs;
1da177e4 1951
6ba045f9 1952 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
1da177e4
LT		 1953 memset(labels, 0, sizeof(labels));
1954 memset(relocs, 0, sizeof(relocs));
1955
1956 build_r3000_tlbchange_handler_head(&p, K0, K1);
d954ffe3 1957 build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
e30ec452 1958 uasm_i_nop(&p); /* load delay */
bbeeffec 1959 build_make_write(&p, &r, K0, K1, -1);
fded2e50 1960 build_r3000_pte_reload_tlbwi(&p, K0, K1);
1da177e4 1961
e30ec452
TS		 1962 uasm_l_nopage_tlbm(&l, p);
1963 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1964 uasm_i_nop(&p);
1da177e4 1965
6ba045f9 1966 if (p >= handle_tlbm_end)
1da177e4
LT		 1967 panic("TLB modify handler fastpath space exceeded");
1968
e30ec452
TS		 1969 uasm_resolve_relocs(relocs, labels);
1970 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1971 (unsigned int)(p - handle_tlbm));
1da177e4 1972
6ba045f9 1973 dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
1da177e4 1974}
82622284 1975#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
1da177e4
LT		 1976
1977/*
1978 * R4000 style TLB load/store/modify handlers.
1979 */
078a55fc 1980static struct work_registers
e30ec452 1981build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
bf28607f 1982 struct uasm_reloc **r)
1da177e4 1983{
bf28607f
DD		 1984 struct work_registers wr = build_get_work_registers(p);
1985
875d43e7 1986#ifdef CONFIG_64BIT
bf28607f 1987 build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
1da177e4 1988#else
bf28607f 1989 build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
1da177e4
LT		 1990#endif
1991
aa1762f4 1992#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 1993 /*
1994 * For huge tlb entries, pmd doesn't contain an address but
1995 * instead contains the tlb pte. Check the PAGE_HUGE bit and
1996 * see if we need to jump to huge tlb processing.
1997 */
bf28607f 1998 build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
fd062c84
DD		 1999#endif
2000
bf28607f
DD		 2001 UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
2002 UASM_i_LW(p, wr.r2, 0, wr.r2);
2003 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
2004 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
2005 UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
1da177e4
LT		 2006
2007#ifdef CONFIG_SMP
e30ec452
TS		 2008 uasm_l_smp_pgtable_change(l, *p);
2009#endif
bf28607f 2010 iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
070e76cb 2011 if (!m4kc_tlbp_war()) {
8df5beac 2012 build_tlb_probe_entry(p);
070e76cb
LY		 2013 if (cpu_has_htw) {
2014 /* race condition happens, leaving */
2015 uasm_i_ehb(p);
2016 uasm_i_mfc0(p, wr.r3, C0_INDEX);
2017 uasm_il_bltz(p, r, wr.r3, label_leave);
2018 uasm_i_nop(p);
2019 }
2020 }
bf28607f 2021 return wr;
1da177e4
LT		 2022}
2023
078a55fc 2024static void
e30ec452
TS		 2025build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
2026 struct uasm_reloc **r, unsigned int tmp,
1da177e4
LT		 2027 unsigned int ptr)
2028{
e30ec452
TS		 2029 uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
2030 uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
1da177e4
LT		 2031 build_update_entries(p, tmp, ptr);
2032 build_tlb_write_entry(p, l, r, tlb_indexed);
e30ec452 2033 uasm_l_leave(l, *p);
bf28607f 2034 build_restore_work_registers(p);
e30ec452 2035 uasm_i_eret(p); /* return from trap */
1da177e4 2036
875d43e7 2037#ifdef CONFIG_64BIT
1ec56329 2038 build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
1da177e4
LT		 2039#endif
2040}
2041
078a55fc 2042static void build_r4000_tlb_load_handler(void)
1da177e4
LT		 2043{
2044 u32 *p = handle_tlbl;
6ba045f9 2045 const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
e30ec452
TS		 2046 struct uasm_label *l = labels;
2047 struct uasm_reloc *r = relocs;
bf28607f 2048 struct work_registers wr;
1da177e4 2049
6ba045f9 2050 memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
1da177e4
LT		 2051 memset(labels, 0, sizeof(labels));
2052 memset(relocs, 0, sizeof(relocs));
2053
2054 if (bcm1250_m3_war()) {
3d45285d
RB		 2055 unsigned int segbits = 44;
2056
2057 uasm_i_dmfc0(&p, K0, C0_BADVADDR);
2058 uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
e30ec452 2059 uasm_i_xor(&p, K0, K0, K1);
3be6022c
DD		 2060 uasm_i_dsrl_safe(&p, K1, K0, 62);
2061 uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
2062 uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
3d45285d 2063 uasm_i_or(&p, K0, K0, K1);
e30ec452
TS		 2064 uasm_il_bnez(&p, &r, K0, label_leave);
2065 /* No need for uasm_i_nop */
1da177e4
LT		 2066 }
2067
bf28607f
DD		 2068 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2069 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
8df5beac
MR		 2070 if (m4kc_tlbp_war())
2071 build_tlb_probe_entry(&p);
6dd9344c 2072
5890f70f 2073 if (cpu_has_rixi && !cpu_has_rixiex) {
6dd9344c
DD		 2074 /*
2075 * If the page is not _PAGE_VALID, RI or XI could not
2076 * have triggered it. Skip the expensive test..
2077 */
cc33ae43 2078 if (use_bbit_insns()) {
bf28607f 2079 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
cc33ae43
DD		 2080 label_tlbl_goaround1);
2081 } else {
bf28607f
DD		 2082 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
2083 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
cc33ae43 2084 }
6dd9344c
DD		 2085 uasm_i_nop(&p);
2086
2087 uasm_i_tlbr(&p);
73acc7df
RB		 2088
2089 switch (current_cpu_type()) {
2090 default:
77f3ee59 2091 if (cpu_has_mips_r2_exec_hazard) {
73acc7df
RB		 2092 uasm_i_ehb(&p);
2093
2094 case CPU_CAVIUM_OCTEON:
2095 case CPU_CAVIUM_OCTEON_PLUS:
2096 case CPU_CAVIUM_OCTEON2:
2097 break;
2098 }
2099 }
2100
6dd9344c 2101 /* Examine entrylo 0 or 1 based on ptr. */
cc33ae43 2102 if (use_bbit_insns()) {
bf28607f 2103 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
cc33ae43 2104 } else {
bf28607f
DD		 2105 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
2106 uasm_i_beqz(&p, wr.r3, 8);
cc33ae43 2107 }
bf28607f
DD		 2108 /* load it in the delay slot*/
2109 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2110 /* load it if ptr is odd */
2111 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
6dd9344c 2112 /*
bf28607f 2113 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
6dd9344c
DD		 2114 * XI must have triggered it.
2115 */
cc33ae43 2116 if (use_bbit_insns()) {
bf28607f
DD		 2117 uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
2118 uasm_i_nop(&p);
cc33ae43
DD		 2119 uasm_l_tlbl_goaround1(&l, p);
2120 } else {
bf28607f
DD		 2121 uasm_i_andi(&p, wr.r3, wr.r3, 2);
2122 uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
2123 uasm_i_nop(&p);
cc33ae43 2124 }
bf28607f 2125 uasm_l_tlbl_goaround1(&l, p);
6dd9344c 2126 }
bbeeffec 2127 build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3);
bf28607f 2128 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
1da177e4 2129
aa1762f4 2130#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 2131 /*
2132 * This is the entry point when build_r4000_tlbchange_handler_head
2133 * spots a huge page.
2134 */
2135 uasm_l_tlb_huge_update(&l, p);
bf28607f
DD		 2136 iPTE_LW(&p, wr.r1, wr.r2);
2137 build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
fd062c84 2138 build_tlb_probe_entry(&p);
6dd9344c 2139
5890f70f 2140 if (cpu_has_rixi && !cpu_has_rixiex) {
6dd9344c
DD		 2141 /*
2142 * If the page is not _PAGE_VALID, RI or XI could not
2143 * have triggered it. Skip the expensive test..
2144 */
cc33ae43 2145 if (use_bbit_insns()) {
bf28607f 2146 uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
cc33ae43
DD		 2147 label_tlbl_goaround2);
2148 } else {
bf28607f
DD		 2149 uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
2150 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
cc33ae43 2151 }
6dd9344c
DD		 2152 uasm_i_nop(&p);
2153
2154 uasm_i_tlbr(&p);
73acc7df
RB		 2155
2156 switch (current_cpu_type()) {
2157 default:
77f3ee59 2158 if (cpu_has_mips_r2_exec_hazard) {
73acc7df
RB		 2159 uasm_i_ehb(&p);
2160
2161 case CPU_CAVIUM_OCTEON:
2162 case CPU_CAVIUM_OCTEON_PLUS:
2163 case CPU_CAVIUM_OCTEON2:
2164 break;
2165 }
2166 }
2167
6dd9344c 2168 /* Examine entrylo 0 or 1 based on ptr. */
cc33ae43 2169 if (use_bbit_insns()) {
bf28607f 2170 uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
cc33ae43 2171 } else {
bf28607f
DD		 2172 uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
2173 uasm_i_beqz(&p, wr.r3, 8);
cc33ae43 2174 }
bf28607f
DD		 2175 /* load it in the delay slot*/
2176 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
2177 /* load it if ptr is odd */
2178 UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
6dd9344c 2179 /*
bf28607f 2180 * If the entryLo (now in wr.r3) is valid (bit 1), RI or
6dd9344c
DD		 2181 * XI must have triggered it.
2182 */
cc33ae43 2183 if (use_bbit_insns()) {
bf28607f 2184 uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
cc33ae43 2185 } else {
bf28607f
DD		 2186 uasm_i_andi(&p, wr.r3, wr.r3, 2);
2187 uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
cc33ae43 2188 }
0f4ccbc8
DD		 2189 if (PM_DEFAULT_MASK == 0)
2190 uasm_i_nop(&p);
6dd9344c
DD		 2191 /*
2192 * We clobbered C0_PAGEMASK, restore it. On the other branch
2193 * it is restored in build_huge_tlb_write_entry.
2194 */
bf28607f 2195 build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
6dd9344c
DD		 2196
2197 uasm_l_tlbl_goaround2(&l, p);
2198 }
bf28607f
DD		 2199 uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
2200 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
fd062c84
DD		 2201#endif
2202
e30ec452 2203 uasm_l_nopage_tlbl(&l, p);
bf28607f 2204 build_restore_work_registers(&p);
2a0b24f5
SH		 2205#ifdef CONFIG_CPU_MICROMIPS
2206 if ((unsigned long)tlb_do_page_fault_0 & 1) {
2207 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
2208 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
2209 uasm_i_jr(&p, K0);
2210 } else
2211#endif
e30ec452
TS		 2212 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
2213 uasm_i_nop(&p);
1da177e4 2214
6ba045f9 2215 if (p >= handle_tlbl_end)
1da177e4
LT		 2216 panic("TLB load handler fastpath space exceeded");
2217
e30ec452
TS		 2218 uasm_resolve_relocs(relocs, labels);
2219 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
2220 (unsigned int)(p - handle_tlbl));
1da177e4 2221
6ba045f9 2222 dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
1da177e4
LT		 2223}
2224
078a55fc 2225static void build_r4000_tlb_store_handler(void)
1da177e4
LT		 2226{
2227 u32 *p = handle_tlbs;
6ba045f9 2228 const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
e30ec452
TS		 2229 struct uasm_label *l = labels;
2230 struct uasm_reloc *r = relocs;
bf28607f 2231 struct work_registers wr;
1da177e4 2232
6ba045f9 2233 memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
1da177e4
LT		 2234 memset(labels, 0, sizeof(labels));
2235 memset(relocs, 0, sizeof(relocs));
2236
bf28607f
DD		 2237 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2238 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
8df5beac
MR		 2239 if (m4kc_tlbp_war())
2240 build_tlb_probe_entry(&p);
bbeeffec 2241 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
bf28607f 2242 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
1da177e4 2243
aa1762f4 2244#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 2245 /*
2246 * This is the entry point when
2247 * build_r4000_tlbchange_handler_head spots a huge page.
2248 */
2249 uasm_l_tlb_huge_update(&l, p);
bf28607f
DD		 2250 iPTE_LW(&p, wr.r1, wr.r2);
2251 build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
fd062c84 2252 build_tlb_probe_entry(&p);
bf28607f 2253 uasm_i_ori(&p, wr.r1, wr.r1,
fd062c84 2254 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
bf28607f 2255 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
fd062c84
DD		 2256#endif
2257
e30ec452 2258 uasm_l_nopage_tlbs(&l, p);
bf28607f 2259 build_restore_work_registers(&p);
2a0b24f5
SH		 2260#ifdef CONFIG_CPU_MICROMIPS
2261 if ((unsigned long)tlb_do_page_fault_1 & 1) {
2262 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2263 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2264 uasm_i_jr(&p, K0);
2265 } else
2266#endif
e30ec452
TS		 2267 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2268 uasm_i_nop(&p);
1da177e4 2269
6ba045f9 2270 if (p >= handle_tlbs_end)
1da177e4
LT		 2271 panic("TLB store handler fastpath space exceeded");
2272
e30ec452
TS		 2273 uasm_resolve_relocs(relocs, labels);
2274 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
2275 (unsigned int)(p - handle_tlbs));
1da177e4 2276
6ba045f9 2277 dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
1da177e4
LT		 2278}
2279
078a55fc 2280static void build_r4000_tlb_modify_handler(void)
1da177e4
LT		 2281{
2282 u32 *p = handle_tlbm;
6ba045f9 2283 const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
e30ec452
TS		 2284 struct uasm_label *l = labels;
2285 struct uasm_reloc *r = relocs;
bf28607f 2286 struct work_registers wr;
1da177e4 2287
6ba045f9 2288 memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
1da177e4
LT		 2289 memset(labels, 0, sizeof(labels));
2290 memset(relocs, 0, sizeof(relocs));
2291
bf28607f
DD		 2292 wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
2293 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
8df5beac
MR		 2294 if (m4kc_tlbp_war())
2295 build_tlb_probe_entry(&p);
1da177e4 2296 /* Present and writable bits set, set accessed and dirty bits. */
bbeeffec 2297 build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);
bf28607f 2298 build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
1da177e4 2299
aa1762f4 2300#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
fd062c84
DD		 2301 /*
2302 * This is the entry point when
2303 * build_r4000_tlbchange_handler_head spots a huge page.
2304 */
2305 uasm_l_tlb_huge_update(&l, p);
bf28607f
DD		 2306 iPTE_LW(&p, wr.r1, wr.r2);
2307 build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
fd062c84 2308 build_tlb_probe_entry(&p);
bf28607f 2309 uasm_i_ori(&p, wr.r1, wr.r1,
fd062c84 2310 _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
bf28607f 2311 build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
fd062c84
DD		 2312#endif
2313
e30ec452 2314 uasm_l_nopage_tlbm(&l, p);
bf28607f 2315 build_restore_work_registers(&p);
2a0b24f5
SH		 2316#ifdef CONFIG_CPU_MICROMIPS
2317 if ((unsigned long)tlb_do_page_fault_1 & 1) {
2318 uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
2319 uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
2320 uasm_i_jr(&p, K0);
2321 } else
2322#endif
e30ec452
TS		 2323 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
2324 uasm_i_nop(&p);
1da177e4 2325
6ba045f9 2326 if (p >= handle_tlbm_end)
1da177e4
LT		 2327 panic("TLB modify handler fastpath space exceeded");
2328
e30ec452
TS		 2329 uasm_resolve_relocs(relocs, labels);
2330 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
2331 (unsigned int)(p - handle_tlbm));
115f2a44 2332
6ba045f9 2333 dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
1da177e4
LT		 2334}
2335
078a55fc 2336static void flush_tlb_handlers(void)
a3d9086b
JG		 2337{
2338 local_flush_icache_range((unsigned long)handle_tlbl,
6ac5310e 2339 (unsigned long)handle_tlbl_end);
a3d9086b 2340 local_flush_icache_range((unsigned long)handle_tlbs,
6ac5310e 2341 (unsigned long)handle_tlbs_end);
a3d9086b 2342 local_flush_icache_range((unsigned long)handle_tlbm,
6ac5310e 2343 (unsigned long)handle_tlbm_end);
6ac5310e
RB		 2344 local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
2345 (unsigned long)tlbmiss_handler_setup_pgd_end);
a3d9086b
JG		 2346}
2347
f1014d1b
MC		 2348static void print_htw_config(void)
2349{
2350 unsigned long config;
2351 unsigned int pwctl;
2352 const int field = 2 * sizeof(unsigned long);
2353
2354 config = read_c0_pwfield();
2355 pr_debug("PWField (0x%0*lx): GDI: 0x%02lx UDI: 0x%02lx MDI: 0x%02lx PTI: 0x%02lx PTEI: 0x%02lx\n",
2356 field, config,
2357 (config & MIPS_PWFIELD_GDI_MASK) >> MIPS_PWFIELD_GDI_SHIFT,
2358 (config & MIPS_PWFIELD_UDI_MASK) >> MIPS_PWFIELD_UDI_SHIFT,
2359 (config & MIPS_PWFIELD_MDI_MASK) >> MIPS_PWFIELD_MDI_SHIFT,
2360 (config & MIPS_PWFIELD_PTI_MASK) >> MIPS_PWFIELD_PTI_SHIFT,
2361 (config & MIPS_PWFIELD_PTEI_MASK) >> MIPS_PWFIELD_PTEI_SHIFT);
2362
2363 config = read_c0_pwsize();
6446e6cf 2364 pr_debug("PWSize (0x%0*lx): PS: 0x%lx GDW: 0x%02lx UDW: 0x%02lx MDW: 0x%02lx PTW: 0x%02lx PTEW: 0x%02lx\n",
f1014d1b 2365 field, config,
6446e6cf 2366 (config & MIPS_PWSIZE_PS_MASK) >> MIPS_PWSIZE_PS_SHIFT,
f1014d1b
MC		 2367 (config & MIPS_PWSIZE_GDW_MASK) >> MIPS_PWSIZE_GDW_SHIFT,
2368 (config & MIPS_PWSIZE_UDW_MASK) >> MIPS_PWSIZE_UDW_SHIFT,
2369 (config & MIPS_PWSIZE_MDW_MASK) >> MIPS_PWSIZE_MDW_SHIFT,
2370 (config & MIPS_PWSIZE_PTW_MASK) >> MIPS_PWSIZE_PTW_SHIFT,
2371 (config & MIPS_PWSIZE_PTEW_MASK) >> MIPS_PWSIZE_PTEW_SHIFT);
2372
2373 pwctl = read_c0_pwctl();
6446e6cf 2374 pr_debug("PWCtl (0x%x): PWEn: 0x%x XK: 0x%x XS: 0x%x XU: 0x%x DPH: 0x%x HugePg: 0x%x Psn: 0x%x\n",
f1014d1b
MC		 2375 pwctl,
2376 (pwctl & MIPS_PWCTL_PWEN_MASK) >> MIPS_PWCTL_PWEN_SHIFT,
6446e6cf
JH		 2377 (pwctl & MIPS_PWCTL_XK_MASK) >> MIPS_PWCTL_XK_SHIFT,
2378 (pwctl & MIPS_PWCTL_XS_MASK) >> MIPS_PWCTL_XS_SHIFT,
2379 (pwctl & MIPS_PWCTL_XU_MASK) >> MIPS_PWCTL_XU_SHIFT,
f1014d1b
MC		 2380 (pwctl & MIPS_PWCTL_DPH_MASK) >> MIPS_PWCTL_DPH_SHIFT,
2381 (pwctl & MIPS_PWCTL_HUGEPG_MASK) >> MIPS_PWCTL_HUGEPG_SHIFT,
2382 (pwctl & MIPS_PWCTL_PSN_MASK) >> MIPS_PWCTL_PSN_SHIFT);
2383}
2384
2385static void config_htw_params(void)
2386{
2387 unsigned long pwfield, pwsize, ptei;
2388 unsigned int config;
2389
2390 /*
2391 * We are using 2-level page tables, so we only need to
2392 * setup GDW and PTW appropriately. UDW and MDW will remain 0.
2393 * The default value of GDI/UDI/MDI/PTI is 0xc. It is illegal to
2394 * write values less than 0xc in these fields because the entire
2395 * write will be dropped. As a result of which, we must preserve
2396 * the original reset values and overwrite only what we really want.
2397 */
2398
2399 pwfield = read_c0_pwfield();
2400 /* re-initialize the GDI field */
2401 pwfield &= ~MIPS_PWFIELD_GDI_MASK;
2402 pwfield |= PGDIR_SHIFT << MIPS_PWFIELD_GDI_SHIFT;
2403 /* re-initialize the PTI field including the even/odd bit */
2404 pwfield &= ~MIPS_PWFIELD_PTI_MASK;
2405 pwfield |= PAGE_SHIFT << MIPS_PWFIELD_PTI_SHIFT;
cab25bc7
PB		 2406 if (CONFIG_PGTABLE_LEVELS >= 3) {
2407 pwfield &= ~MIPS_PWFIELD_MDI_MASK;
2408 pwfield |= PMD_SHIFT << MIPS_PWFIELD_MDI_SHIFT;
2409 }
f1014d1b
MC		 2410 /* Set the PTEI right shift */
2411 ptei = _PAGE_GLOBAL_SHIFT << MIPS_PWFIELD_PTEI_SHIFT;
2412 pwfield |= ptei;
2413 write_c0_pwfield(pwfield);
2414 /* Check whether the PTEI value is supported */
2415 back_to_back_c0_hazard();
2416 pwfield = read_c0_pwfield();
2417 if (((pwfield & MIPS_PWFIELD_PTEI_MASK) << MIPS_PWFIELD_PTEI_SHIFT)
2418 != ptei) {
2419 pr_warn("Unsupported PTEI field value: 0x%lx. HTW will not be enabled",
2420 ptei);
2421 /*
2422 * Drop option to avoid HTW being enabled via another path
2423 * (eg htw_reset())
2424 */
2425 current_cpu_data.options &= ~MIPS_CPU_HTW;
2426 return;
2427 }
2428
2429 pwsize = ilog2(PTRS_PER_PGD) << MIPS_PWSIZE_GDW_SHIFT;
2430 pwsize |= ilog2(PTRS_PER_PTE) << MIPS_PWSIZE_PTW_SHIFT;
cab25bc7
PB		 2431 if (CONFIG_PGTABLE_LEVELS >= 3)
2432 pwsize |= ilog2(PTRS_PER_PMD) << MIPS_PWSIZE_MDW_SHIFT;
c5b36783 2433
aa76042a 2434 /* Set pointer size to size of directory pointers */
97f2645f 2435 if (IS_ENABLED(CONFIG_64BIT))
aa76042a
JH		 2436 pwsize |= MIPS_PWSIZE_PS_MASK;
2437 /* PTEs may be multiple pointers long (e.g. with XPA) */
2438 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT)
2439 & MIPS_PWSIZE_PTEW_MASK;
c5b36783 2440
f1014d1b
MC		 2441 write_c0_pwsize(pwsize);
2442
2443 /* Make sure everything is set before we enable the HTW */
2444 back_to_back_c0_hazard();
2445
aa76042a
JH		 2446 /*
2447 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of
2448 * the pwctl fields.
2449 */
f1014d1b 2450 config = 1 << MIPS_PWCTL_PWEN_SHIFT;
97f2645f 2451 if (IS_ENABLED(CONFIG_64BIT))
aa76042a 2452 config |= MIPS_PWCTL_XU_MASK;
f1014d1b
MC		 2453 write_c0_pwctl(config);
2454 pr_info("Hardware Page Table Walker enabled\n");
2455
2456 print_htw_config();
2457}
2458
c5b36783
SH		 2459static void config_xpa_params(void)
2460{
2461#ifdef CONFIG_XPA
2462 unsigned int pagegrain;
2463
2464 if (mips_xpa_disabled) {
2465 pr_info("Extended Physical Addressing (XPA) disabled\n");
2466 return;
2467 }
2468
2469 pagegrain = read_c0_pagegrain();
2470 write_c0_pagegrain(pagegrain | PG_ELPA);
2471 back_to_back_c0_hazard();
2472 pagegrain = read_c0_pagegrain();
2473
2474 if (pagegrain & PG_ELPA)
2475 pr_info("Extended Physical Addressing (XPA) enabled\n");
2476 else
2477 panic("Extended Physical Addressing (XPA) disabled");
2478#endif
2479}
2480
00bf1c69
PB		 2481static void check_pabits(void)
2482{
2483 unsigned long entry;
2484 unsigned pabits, fillbits;
2485
2486 if (!cpu_has_rixi || !_PAGE_NO_EXEC) {
2487 /*
2488 * We'll only be making use of the fact that we can rotate bits
2489 * into the fill if the CPU supports RIXI, so don't bother
2490 * probing this for CPUs which don't.
2491 */
2492 return;
2493 }
2494
2495 write_c0_entrylo0(~0ul);
2496 back_to_back_c0_hazard();
2497 entry = read_c0_entrylo0();
2498
2499 /* clear all non-PFN bits */
2500 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1);
2501 entry &= ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI);
2502
2503 /* find a lower bound on PABITS, and upper bound on fill bits */
2504 pabits = fls_long(entry) + 6;
2505 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0);
2506
2507 /* minus the RI & XI bits */
2508 fillbits -= min_t(unsigned, fillbits, 2);
2509
2510 if (fillbits >= ilog2(_PAGE_NO_EXEC))
2511 fill_includes_sw_bits = true;
2512
2513 pr_debug("Entry* registers contain %u fill bits\n", fillbits);
2514}
2515
078a55fc 2516void build_tlb_refill_handler(void)
1da177e4
LT		 2517{
2518 /*
2519 * The refill handler is generated per-CPU, multi-node systems
2520 * may have local storage for it. The other handlers are only
2521 * needed once.
2522 */
2523 static int run_once = 0;
2524
97f2645f 2525 if (IS_ENABLED(CONFIG_XPA) && !cpu_has_rixi)
e56c7e18
PB		 2526 panic("Kernels supporting XPA currently require CPUs with RIXI");
2527
a2c763e0 2528 output_pgtable_bits_defines();
00bf1c69 2529 check_pabits();
a2c763e0 2530
1ec56329
DD		 2531#ifdef CONFIG_64BIT
2532 check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
2533#endif
2534
10cc3529 2535 switch (current_cpu_type()) {
1da177e4
LT		 2536 case CPU_R2000:
2537 case CPU_R3000:
2538 case CPU_R3000A:
2539 case CPU_R3081E:
2540 case CPU_TX3912:
2541 case CPU_TX3922:
2542 case CPU_TX3927:
82622284 2543#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
8759934e
HC		 2544 if (cpu_has_local_ebase)
2545 build_r3000_tlb_refill_handler();
1da177e4 2546 if (!run_once) {
8759934e
HC		 2547 if (!cpu_has_local_ebase)
2548 build_r3000_tlb_refill_handler();
f4ae17aa 2549 build_setup_pgd();
1da177e4
LT		 2550 build_r3000_tlb_load_handler();
2551 build_r3000_tlb_store_handler();
2552 build_r3000_tlb_modify_handler();
a3d9086b 2553 flush_tlb_handlers();
1da177e4
LT		 2554 run_once++;
2555 }
82622284
DD		 2556#else
2557 panic("No R3000 TLB refill handler");
2558#endif
1da177e4
LT		 2559 break;
2560
2561 case CPU_R6000:
2562 case CPU_R6000A:
2563 panic("No R6000 TLB refill handler yet");
2564 break;
2565
2566 case CPU_R8000:
2567 panic("No R8000 TLB refill handler yet");
2568 break;
2569
2570 default:
380cd582
HC		 2571 if (cpu_has_ldpte)
2572 setup_pw();
2573
1da177e4 2574 if (!run_once) {
bf28607f 2575 scratch_reg = allocate_kscratch();
f4ae17aa 2576 build_setup_pgd();
1da177e4
LT		 2577 build_r4000_tlb_load_handler();
2578 build_r4000_tlb_store_handler();
2579 build_r4000_tlb_modify_handler();
380cd582
HC		 2580 if (cpu_has_ldpte)
2581 build_loongson3_tlb_refill_handler();
2582 else if (!cpu_has_local_ebase)
8759934e 2583 build_r4000_tlb_refill_handler();
a3d9086b 2584 flush_tlb_handlers();
1da177e4
LT		 2585 run_once++;
2586 }
8759934e
HC		 2587 if (cpu_has_local_ebase)
2588 build_r4000_tlb_refill_handler();
c5b36783
SH		 2589 if (cpu_has_xpa)
2590 config_xpa_params();
f1014d1b
MC		 2591 if (cpu_has_htw)
2592 config_htw_params();
1da177e4
LT		 2593 }
2594}
Linux kernel - Deliverables
This page took 1.429676 seconds and 5 git commands to generate.