| 1 | /* sun4v_tlb_miss.S: Sun4v TLB miss handlers. |
| 2 | * |
| 3 | * Copyright (C) 2006 <davem@davemloft.net> |
| 4 | */ |
| 5 | |
| 6 | .text |
| 7 | .align 32 |
| 8 | |
| 9 | sun4v_itlb_miss: |
| 10 | /* Load MMU Miss base into %g2. */ |
| 11 | ldxa [%g0] ASI_SCRATCHPAD, %g3 |
| 12 | |
| 13 | /* Load UTSB reg into %g1. */ |
| 14 | mov SCRATCHPAD_UTSBREG1, %g1 |
| 15 | ldxa [%g1] ASI_SCRATCHPAD, %g1 |
| 16 | |
| 17 | /* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6. |
| 18 | * Branch if kernel TLB miss. The kernel TSB and user TSB miss |
| 19 | * code wants the missing virtual address in %g4, so that value |
| 20 | * cannot be modified through the entirety of this handler. |
| 21 | */ |
| 22 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 |
| 23 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 |
| 24 | srlx %g4, 22, %g3 |
| 25 | sllx %g5, 48, %g6 |
| 26 | or %g6, %g3, %g6 |
| 27 | brz,pn %g5, kvmap_itlb_4v |
| 28 | nop |
| 29 | |
| 30 | /* Create TSB pointer. This is something like: |
| 31 | * |
| 32 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; |
| 33 | * tsb_base = tsb_reg & ~0x7UL; |
| 34 | */ |
| 35 | and %g1, 0x7, %g3 |
| 36 | andn %g1, 0x7, %g1 |
| 37 | mov 512, %g7 |
| 38 | sllx %g7, %g3, %g7 |
| 39 | sub %g7, 1, %g7 |
| 40 | |
| 41 | /* TSB index mask is in %g7, tsb base is in %g1. Compute |
| 42 | * the TSB entry pointer into %g1: |
| 43 | * |
| 44 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); |
| 45 | * tsb_ptr = tsb_base + (tsb_index * 16); |
| 46 | */ |
| 47 | srlx %g4, PAGE_SHIFT, %g3 |
| 48 | and %g3, %g7, %g3 |
| 49 | sllx %g3, 4, %g3 |
| 50 | add %g1, %g3, %g1 |
| 51 | |
| 52 | /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ |
| 53 | ldda [%g1] ASI_QUAD_LDD_PHYS, %g2 |
| 54 | cmp %g2, %g6 |
| 55 | sethi %hi(_PAGE_EXEC), %g7 |
| 56 | bne,a,pn %xcc, tsb_miss_page_table_walk |
| 57 | mov FAULT_CODE_ITLB, %g3 |
| 58 | andcc %g3, %g7, %g0 |
| 59 | be,a,pn %xcc, tsb_do_fault |
| 60 | mov FAULT_CODE_ITLB, %g3 |
| 61 | |
| 62 | /* We have a valid entry, make hypervisor call to load |
| 63 | * I-TLB and return from trap. |
| 64 | * |
| 65 | * %g3: PTE |
| 66 | * %g4: vaddr |
| 67 | * %g6: TAG TARGET (only "CTX << 48" part matters) |
| 68 | */ |
| 69 | sun4v_itlb_load: |
| 70 | mov %o0, %g1 ! save %o0 |
| 71 | mov %o1, %g2 ! save %o1 |
| 72 | mov %o2, %g5 ! save %o2 |
| 73 | mov %o3, %g7 ! save %o3 |
| 74 | mov %g4, %o0 ! vaddr |
| 75 | srlx %g6, 48, %o1 ! ctx |
| 76 | mov %g3, %o2 ! PTE |
| 77 | mov HV_MMU_IMMU, %o3 ! flags |
| 78 | ta HV_MMU_MAP_ADDR_TRAP |
| 79 | mov %g1, %o0 ! restore %o0 |
| 80 | mov %g2, %o1 ! restore %o1 |
| 81 | mov %g5, %o2 ! restore %o2 |
| 82 | mov %g7, %o3 ! restore %o3 |
| 83 | |
| 84 | retry |
| 85 | |
| 86 | sun4v_dtlb_miss: |
| 87 | /* Load MMU Miss base into %g2. */ |
| 88 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 89 | |
| 90 | /* Load UTSB reg into %g1. */ |
| 91 | mov SCRATCHPAD_UTSBREG1, %g1 |
| 92 | ldxa [%g1 + %g1] ASI_SCRATCHPAD, %g1 |
| 93 | |
| 94 | /* Create a TAG TARGET, "(vaddr>>22) | (ctx << 48)", in %g6. |
| 95 | * Branch if kernel TLB miss. The kernel TSB and user TSB miss |
| 96 | * code wants the missing virtual address in %g4, so that value |
| 97 | * cannot be modified through the entirety of this handler. |
| 98 | */ |
| 99 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 100 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 101 | srlx %g4, 22, %g3 |
| 102 | sllx %g5, 48, %g6 |
| 103 | or %g6, %g3, %g6 |
| 104 | brz,pn %g5, kvmap_dtlb_4v |
| 105 | nop |
| 106 | |
| 107 | /* Create TSB pointer. This is something like: |
| 108 | * |
| 109 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; |
| 110 | * tsb_base = tsb_reg & ~0x7UL; |
| 111 | */ |
| 112 | and %g1, 0x7, %g3 |
| 113 | andn %g1, 0x7, %g1 |
| 114 | mov 512, %g7 |
| 115 | sllx %g7, %g3, %g7 |
| 116 | sub %g7, 1, %g7 |
| 117 | |
| 118 | /* TSB index mask is in %g7, tsb base is in %g1. Compute |
| 119 | * the TSB entry pointer into %g1: |
| 120 | * |
| 121 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); |
| 122 | * tsb_ptr = tsb_base + (tsb_index * 16); |
| 123 | */ |
| 124 | srlx %g4, PAGE_SHIFT, %g3 |
| 125 | and %g3, %g7, %g3 |
| 126 | sllx %g3, 4, %g3 |
| 127 | add %g1, %g3, %g1 |
| 128 | |
| 129 | /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ |
| 130 | ldda [%g1] ASI_QUAD_LDD_PHYS, %g2 |
| 131 | cmp %g2, %g6 |
| 132 | bne,a,pn %xcc, tsb_miss_page_table_walk |
| 133 | mov FAULT_CODE_ITLB, %g3 |
| 134 | |
| 135 | /* We have a valid entry, make hypervisor call to load |
| 136 | * D-TLB and return from trap. |
| 137 | * |
| 138 | * %g3: PTE |
| 139 | * %g4: vaddr |
| 140 | * %g6: TAG TARGET (only "CTX << 48" part matters) |
| 141 | */ |
| 142 | sun4v_dtlb_load: |
| 143 | mov %o0, %g1 ! save %o0 |
| 144 | mov %o1, %g2 ! save %o1 |
| 145 | mov %o2, %g5 ! save %o2 |
| 146 | mov %o3, %g7 ! save %o3 |
| 147 | mov %g4, %o0 ! vaddr |
| 148 | srlx %g6, 48, %o1 ! ctx |
| 149 | mov %g3, %o2 ! PTE |
| 150 | mov HV_MMU_DMMU, %o3 ! flags |
| 151 | ta HV_MMU_MAP_ADDR_TRAP |
| 152 | mov %g1, %o0 ! restore %o0 |
| 153 | mov %g2, %o1 ! restore %o1 |
| 154 | mov %g5, %o2 ! restore %o2 |
| 155 | mov %g7, %o3 ! restore %o3 |
| 156 | |
| 157 | retry |
| 158 | |
| 159 | sun4v_dtlb_prot: |
| 160 | /* Load MMU Miss base into %g2. */ |
| 161 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 162 | |
| 163 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 |
| 164 | rdpr %tl, %g1 |
| 165 | cmp %g1, 1 |
| 166 | bgu,pn %xcc, winfix_trampoline |
| 167 | nop |
| 168 | ba,pt %xcc, sparc64_realfault_common |
| 169 | mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 |
| 170 | |
| 171 | /* Called from trap table with TAG TARGET placed into |
| 172 | * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1. |
| 173 | */ |
| 174 | sun4v_itsb_miss: |
| 175 | ba,pt %xcc, sun4v_tsb_miss_common |
| 176 | mov FAULT_CODE_ITLB, %g3 |
| 177 | |
| 178 | /* Called from trap table with TAG TARGET placed into |
| 179 | * %g6 and SCRATCHPAD_UTSBREG1 contents in %g1. |
| 180 | */ |
| 181 | sun4v_dtsb_miss: |
| 182 | mov FAULT_CODE_DTLB, %g3 |
| 183 | |
| 184 | /* Create TSB pointer into %g1. This is something like: |
| 185 | * |
| 186 | * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; |
| 187 | * tsb_base = tsb_reg & ~0x7UL; |
| 188 | * tsb_index = ((vaddr >> PAGE_SHIFT) & tsb_mask); |
| 189 | * tsb_ptr = tsb_base + (tsb_index * 16); |
| 190 | */ |
| 191 | sun4v_tsb_miss_common: |
| 192 | and %g1, 0x7, %g2 |
| 193 | andn %g1, 0x7, %g1 |
| 194 | mov 512, %g7 |
| 195 | sllx %g7, %g2, %g7 |
| 196 | sub %g7, 1, %g7 |
| 197 | srlx %g4, PAGE_SHIFT, %g2 |
| 198 | and %g2, %g7, %g2 |
| 199 | sllx %g2, 4, %g2 |
| 200 | ba,pt %xcc, tsb_miss_page_table_walk |
| 201 | add %g1, %g2, %g1 |
| 202 | |
| 203 | /* Instruction Access Exception, tl0. */ |
| 204 | sun4v_iacc: |
| 205 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 206 | ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 |
| 207 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 |
| 208 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 |
| 209 | sllx %g3, 16, %g3 |
| 210 | or %g5, %g3, %g5 |
| 211 | ba,pt %xcc, etrap |
| 212 | rd %pc, %g7 |
| 213 | mov %l4, %o1 |
| 214 | mov %l5, %o2 |
| 215 | call sun4v_insn_access_exception |
| 216 | add %sp, PTREGS_OFF, %o0 |
| 217 | ba,a,pt %xcc, rtrap_clr_l6 |
| 218 | |
| 219 | /* Instruction Access Exception, tl1. */ |
| 220 | sun4v_iacc_tl1: |
| 221 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 222 | ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 |
| 223 | ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 |
| 224 | ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 |
| 225 | sllx %g3, 16, %g3 |
| 226 | or %g5, %g3, %g5 |
| 227 | ba,pt %xcc, etraptl1 |
| 228 | rd %pc, %g7 |
| 229 | mov %l4, %o1 |
| 230 | mov %l5, %o2 |
| 231 | call sun4v_insn_access_exception_tl1 |
| 232 | add %sp, PTREGS_OFF, %o0 |
| 233 | ba,a,pt %xcc, rtrap_clr_l6 |
| 234 | |
| 235 | /* Data Access Exception, tl0. */ |
| 236 | sun4v_dacc: |
| 237 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 238 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 |
| 239 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 240 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 241 | sllx %g3, 16, %g3 |
| 242 | or %g5, %g3, %g5 |
| 243 | ba,pt %xcc, etrap |
| 244 | rd %pc, %g7 |
| 245 | mov %l4, %o1 |
| 246 | mov %l5, %o2 |
| 247 | call sun4v_data_access_exception |
| 248 | add %sp, PTREGS_OFF, %o0 |
| 249 | ba,a,pt %xcc, rtrap_clr_l6 |
| 250 | |
| 251 | /* Data Access Exception, tl1. */ |
| 252 | sun4v_dacc_tl1: |
| 253 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 254 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 |
| 255 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 256 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 257 | sllx %g3, 16, %g3 |
| 258 | or %g5, %g3, %g5 |
| 259 | ba,pt %xcc, etraptl1 |
| 260 | rd %pc, %g7 |
| 261 | mov %l4, %o1 |
| 262 | mov %l5, %o2 |
| 263 | call sun4v_data_access_exception_tl1 |
| 264 | add %sp, PTREGS_OFF, %o0 |
| 265 | ba,a,pt %xcc, rtrap_clr_l6 |
| 266 | |
| 267 | /* Memory Address Unaligned. */ |
| 268 | sun4v_mna: |
| 269 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 270 | mov HV_FAULT_TYPE_UNALIGNED, %g3 |
| 271 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 272 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 273 | sllx %g3, 16, %g3 |
| 274 | or %g5, %g3, %g5 |
| 275 | |
| 276 | /* Window fixup? */ |
| 277 | rdpr %tl, %g2 |
| 278 | cmp %g2, 1 |
| 279 | bgu,pn %icc, winfix_mna |
| 280 | rdpr %tpc, %g3 |
| 281 | |
| 282 | ba,pt %xcc, etrap |
| 283 | rd %pc, %g7 |
| 284 | mov %l4, %o1 |
| 285 | mov %l5, %o2 |
| 286 | call sun4v_mna |
| 287 | add %sp, PTREGS_OFF, %o0 |
| 288 | ba,a,pt %xcc, rtrap_clr_l6 |
| 289 | |
| 290 | /* Privileged Action. */ |
| 291 | sun4v_privact: |
| 292 | ba,pt %xcc, etrap |
| 293 | rd %pc, %g7 |
| 294 | call do_privact |
| 295 | add %sp, PTREGS_OFF, %o0 |
| 296 | ba,a,pt %xcc, rtrap_clr_l6 |
| 297 | |
| 298 | /* Unaligned ldd float, tl0. */ |
| 299 | sun4v_lddfmna: |
| 300 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 301 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 |
| 302 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 303 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 304 | sllx %g3, 16, %g3 |
| 305 | or %g5, %g3, %g5 |
| 306 | ba,pt %xcc, etrap |
| 307 | rd %pc, %g7 |
| 308 | mov %l4, %o1 |
| 309 | mov %l5, %o2 |
| 310 | call handle_lddfmna |
| 311 | add %sp, PTREGS_OFF, %o0 |
| 312 | ba,a,pt %xcc, rtrap_clr_l6 |
| 313 | |
| 314 | /* Unaligned std float, tl0. */ |
| 315 | sun4v_stdfmna: |
| 316 | ldxa [%g0] ASI_SCRATCHPAD, %g2 |
| 317 | ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 |
| 318 | ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 |
| 319 | ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 |
| 320 | sllx %g3, 16, %g3 |
| 321 | or %g5, %g3, %g5 |
| 322 | ba,pt %xcc, etrap |
| 323 | rd %pc, %g7 |
| 324 | mov %l4, %o1 |
| 325 | mov %l5, %o2 |
| 326 | call handle_stdfmna |
| 327 | add %sp, PTREGS_OFF, %o0 |
| 328 | ba,a,pt %xcc, rtrap_clr_l6 |
| 329 | |
| 330 | #define BRANCH_ALWAYS 0x10680000 |
| 331 | #define NOP 0x01000000 |
| 332 | #define SUN4V_DO_PATCH(OLD, NEW) \ |
| 333 | sethi %hi(NEW), %g1; \ |
| 334 | or %g1, %lo(NEW), %g1; \ |
| 335 | sethi %hi(OLD), %g2; \ |
| 336 | or %g2, %lo(OLD), %g2; \ |
| 337 | sub %g1, %g2, %g1; \ |
| 338 | sethi %hi(BRANCH_ALWAYS), %g3; \ |
| 339 | srl %g1, 2, %g1; \ |
| 340 | or %g3, %lo(BRANCH_ALWAYS), %g3; \ |
| 341 | or %g3, %g1, %g3; \ |
| 342 | stw %g3, [%g2]; \ |
| 343 | sethi %hi(NOP), %g3; \ |
| 344 | or %g3, %lo(NOP), %g3; \ |
| 345 | stw %g3, [%g2 + 0x4]; \ |
| 346 | flush %g2; |
| 347 | |
| 348 | .globl sun4v_patch_tlb_handlers |
| 349 | .type sun4v_patch_tlb_handlers,#function |
| 350 | sun4v_patch_tlb_handlers: |
| 351 | SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) |
| 352 | SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) |
| 353 | SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) |
| 354 | SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) |
| 355 | SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) |
| 356 | SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) |
| 357 | SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) |
| 358 | SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) |
| 359 | SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) |
| 360 | SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) |
| 361 | SUN4V_DO_PATCH(tl0_mna, sun4v_mna) |
| 362 | SUN4V_DO_PATCH(tl1_mna, sun4v_mna) |
| 363 | SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) |
| 364 | SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) |
| 365 | SUN4V_DO_PATCH(tl0_privact, sun4v_privact) |
| 366 | retl |
| 367 | nop |
| 368 | .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers |