2 * PARISC TLB and cache flushing support
3 * Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 * Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 * Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2, or (at your option)
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 * should only use index and base registers that are not shadowed,
25 * so that the fast path emulation in the non access miss handler
43 #include <linux/config.h>
46 #include <asm/assembly.h>
47 #include <asm/pgtable.h>
48 #include <asm/cache.h>
53 .export flush_tlb_all_local,code
61 * The pitlbe and pdtlbe instructions should only be used to
62 * flush the entire tlb. Also, there needs to be no intervening
63 * tlb operations, e.g. tlb misses, so the operation needs
64 * to happen in real mode with all interruptions disabled.
67 /* pcxt_ssm_bug - relied upon translation! PA 2.0 Arch. F-4 and F-5 */
68 rsm PSW_SM_I, %r19 /* save I-bit state */
76 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
77 mtctl %r0, %cr17 /* Clear IIASQ tail */
78 mtctl %r0, %cr17 /* Clear IIASQ head */
79 mtctl %r1, %cr18 /* IIAOQ head */
81 mtctl %r1, %cr18 /* IIAOQ tail */
82 load32 REAL_MODE_PSW, %r1
87 1: load32 PA(cache_info), %r1
89 /* Flush Instruction Tlb */
91 LDREG ITLB_SID_BASE(%r1), %r20
92 LDREG ITLB_SID_STRIDE(%r1), %r21
93 LDREG ITLB_SID_COUNT(%r1), %r22
94 LDREG ITLB_OFF_BASE(%r1), %arg0
95 LDREG ITLB_OFF_STRIDE(%r1), %arg1
96 LDREG ITLB_OFF_COUNT(%r1), %arg2
97 LDREG ITLB_LOOP(%r1), %arg3
99 ADDIB= -1, %arg3, fitoneloop /* Preadjust and test */
100 movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
101 copy %arg0, %r28 /* Init base addr */
103 fitmanyloop: /* Loop if LOOP >= 2 */
105 add %r21, %r20, %r20 /* increment space */
106 copy %arg2, %r29 /* Init middle loop count */
108 fitmanymiddle: /* Loop if LOOP >= 2 */
109 ADDIB> -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
111 pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
112 ADDIB> -1, %r29, fitmanymiddle /* Middle loop decr */
113 copy %arg3, %r31 /* Re-init inner loop count */
115 movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
116 ADDIB<=,n -1, %r22, fitdone /* Outer loop count decr */
118 fitoneloop: /* Loop if LOOP = 1 */
120 copy %arg0, %r28 /* init base addr */
121 copy %arg2, %r29 /* init middle loop count */
123 fitonemiddle: /* Loop if LOOP = 1 */
124 ADDIB> -1, %r29, fitonemiddle /* Middle loop count decr */
125 pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
127 ADDIB> -1, %r22, fitoneloop /* Outer loop count decr */
128 add %r21, %r20, %r20 /* increment space */
134 LDREG DTLB_SID_BASE(%r1), %r20
135 LDREG DTLB_SID_STRIDE(%r1), %r21
136 LDREG DTLB_SID_COUNT(%r1), %r22
137 LDREG DTLB_OFF_BASE(%r1), %arg0
138 LDREG DTLB_OFF_STRIDE(%r1), %arg1
139 LDREG DTLB_OFF_COUNT(%r1), %arg2
140 LDREG DTLB_LOOP(%r1), %arg3
142 ADDIB= -1, %arg3, fdtoneloop /* Preadjust and test */
143 movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
144 copy %arg0, %r28 /* Init base addr */
146 fdtmanyloop: /* Loop if LOOP >= 2 */
148 add %r21, %r20, %r20 /* increment space */
149 copy %arg2, %r29 /* Init middle loop count */
151 fdtmanymiddle: /* Loop if LOOP >= 2 */
152 ADDIB> -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
154 pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
155 ADDIB> -1, %r29, fdtmanymiddle /* Middle loop decr */
156 copy %arg3, %r31 /* Re-init inner loop count */
158 movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
159 ADDIB<=,n -1, %r22,fdtdone /* Outer loop count decr */
161 fdtoneloop: /* Loop if LOOP = 1 */
163 copy %arg0, %r28 /* init base addr */
164 copy %arg2, %r29 /* init middle loop count */
166 fdtonemiddle: /* Loop if LOOP = 1 */
167 ADDIB> -1, %r29, fdtonemiddle /* Middle loop count decr */
168 pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
170 ADDIB> -1, %r22, fdtoneloop /* Outer loop count decr */
171 add %r21, %r20, %r20 /* increment space */
176 * Switch back to virtual mode
187 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
188 mtctl %r0, %cr17 /* Clear IIASQ tail */
189 mtctl %r0, %cr17 /* Clear IIASQ head */
190 mtctl %r1, %cr18 /* IIAOQ head */
192 mtctl %r1, %cr18 /* IIAOQ tail */
193 load32 KERNEL_PSW, %r1
194 or %r1, %r19, %r1 /* I-bit to state on entry */
195 mtctl %r1, %ipsw /* restore I-bit (entire PSW) */
205 .export flush_instruction_cache_local,code
206 .import cache_info,data
208 flush_instruction_cache_local:
214 load32 cache_info, %r1
216 /* Flush Instruction Cache */
218 LDREG ICACHE_BASE(%r1), %arg0
219 LDREG ICACHE_STRIDE(%r1), %arg1
220 LDREG ICACHE_COUNT(%r1), %arg2
221 LDREG ICACHE_LOOP(%r1), %arg3
222 rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
223 ADDIB= -1, %arg3, fioneloop /* Preadjust and test */
224 movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
226 fimanyloop: /* Loop if LOOP >= 2 */
227 ADDIB> -1, %r31, fimanyloop /* Adjusted inner loop decr */
228 fice %r0(%sr1, %arg0)
229 fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
230 movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
231 ADDIB<=,n -1, %arg2, fisync /* Outer loop decr */
233 fioneloop: /* Loop if LOOP = 1 */
234 ADDIB> -1, %arg2, fioneloop /* Outer loop count decr */
235 fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
239 mtsm %r22 /* restore I-bit */
246 .export flush_data_cache_local, code
247 .import cache_info, data
249 flush_data_cache_local:
255 load32 cache_info, %r1
257 /* Flush Data Cache */
259 LDREG DCACHE_BASE(%r1), %arg0
260 LDREG DCACHE_STRIDE(%r1), %arg1
261 LDREG DCACHE_COUNT(%r1), %arg2
262 LDREG DCACHE_LOOP(%r1), %arg3
264 ADDIB= -1, %arg3, fdoneloop /* Preadjust and test */
265 movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
267 fdmanyloop: /* Loop if LOOP >= 2 */
268 ADDIB> -1, %r31, fdmanyloop /* Adjusted inner loop decr */
269 fdce %r0(%sr1, %arg0)
270 fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
271 movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
272 ADDIB<=,n -1, %arg2, fdsync /* Outer loop decr */
274 fdoneloop: /* Loop if LOOP = 1 */
275 ADDIB> -1, %arg2, fdoneloop /* Outer loop count decr */
276 fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
281 mtsm %r22 /* restore I-bit */
288 .export copy_user_page_asm,code
297 /* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
298 * Unroll the loop by hand and arrange insn appropriately.
299 * GCC probably can do this just as well.
303 ldi ASM_PAGE_SIZE_DIV128, %r1
305 ldw 64(%r25), %r0 /* prefetch 1 cacheline ahead */
306 ldw 128(%r25), %r0 /* prefetch 2 */
309 ldw 192(%r25), %r0 /* prefetch 3 */
310 ldw 256(%r25), %r0 /* prefetch 4 */
352 /* conditional branches nullify on forward taken branch, and on
353 * non-taken backward branch. Note that .+4 is a backwards branch.
354 * The ldd should only get executed if the branch is taken.
356 ADDIB>,n -1, %r1, 1b /* bundle 10 */
357 ldd 0(%r25), %r19 /* start next loads */
362 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
363 * bundles (very restricted rules for bundling).
364 * Note that until (if) we start saving
365 * the full 64 bit register values on interrupt, we can't
366 * use ldd/std on a 32 bit kernel.
369 ldi ASM_PAGE_SIZE_DIV64, %r1
415 * NOTE: Code in clear_user_page has a hard coded dependency on the
416 * maximum alias boundary being 4 Mb. We've been assured by the
417 * parisc chip designers that there will not ever be a parisc
418 * chip with a larger alias boundary (Never say never :-) ).
420 * Subtle: the dtlb miss handlers support the temp alias region by
421 * "knowing" that if a dtlb miss happens within the temp alias
422 * region it must have occurred while in clear_user_page. Since
423 * this routine makes use of processor local translations, we
424 * don't want to insert them into the kernel page table. Instead,
425 * we load up some general registers (they need to be registers
426 * which aren't shadowed) with the physical page numbers (preshifted
427 * for tlb insertion) needed to insert the translations. When we
428 * miss on the translation, the dtlb miss handler inserts the
429 * translation into the tlb using these values:
431 * %r26 physical page (shifted for tlb insert) of "to" translation
432 * %r23 physical page (shifted for tlb insert) of "from" translation
438 * We can't do this since copy_user_page is used to bring in
439 * file data that might have instructions. Since the data would
440 * then need to be flushed out so the i-fetch can see it, it
441 * makes more sense to just copy through the kernel translation
444 * I'm still keeping this around because it may be possible to
445 * use it if more information is passed into copy_user_page().
446 * Have to do some measurements to see if it is worthwhile to
447 * lobby for such a change.
450 .export copy_user_page_asm,code
457 ldil L%(__PAGE_OFFSET), %r1
459 sub %r25, %r1, %r23 /* move physical addr into non shadowed reg */
461 ldil L%(TMPALIAS_MAP_START), %r28
462 /* FIXME for different page sizes != 4k */
464 extrd,u %r26,56,32, %r26 /* convert phys addr to tlb insert format */
465 extrd,u %r23,56,32, %r23 /* convert phys addr to tlb insert format */
466 depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
467 depdi 0, 63,12, %r28 /* Clear any offset bits */
469 depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
471 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
472 extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
473 depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
474 depwi 0, 31,12, %r28 /* Clear any offset bits */
476 depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
479 /* Purge any old translations */
487 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
488 * bundles (very restricted rules for bundling). It probably
489 * does OK on PCXU and better, but we could do better with
490 * ldd/std instructions. Note that until (if) we start saving
491 * the full 64 bit register values on interrupt, we can't
492 * use ldd/std on a 32 bit kernel.
540 .export __clear_user_page_asm,code
542 __clear_user_page_asm:
549 ldil L%(TMPALIAS_MAP_START), %r28
551 #if (TMPALIAS_MAP_START >= 0x80000000)
552 depdi 0, 31,32, %r28 /* clear any sign extension */
553 /* FIXME: page size dependend */
555 extrd,u %r26, 56,32, %r26 /* convert phys addr to tlb insert format */
556 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
557 depdi 0, 63,12, %r28 /* Clear any offset bits */
559 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
560 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
561 depwi 0, 31,12, %r28 /* Clear any offset bits */
564 /* Purge any old translation */
569 ldi ASM_PAGE_SIZE_DIV128, %r1
571 /* PREFETCH (Write) has not (yet) been proven to help here */
572 /* #define PREFETCHW_OP ldd 256(%0), %r0 */
593 #else /* ! CONFIG_64BIT */
594 ldi ASM_PAGE_SIZE_DIV64, %r1
615 #endif /* CONFIG_64BIT */
623 .export flush_kernel_dcache_page_asm
625 flush_kernel_dcache_page_asm:
630 ldil L%dcache_stride, %r1
631 ldw R%dcache_stride(%r1), %r23
634 depdi,z 1, 63-PAGE_SHIFT,1, %r25
636 depwi,z 1, 31-PAGE_SHIFT,1, %r25
667 .export flush_user_dcache_page
669 flush_user_dcache_page:
674 ldil L%dcache_stride, %r1
675 ldw R%dcache_stride(%r1), %r23
678 depdi,z 1,63-PAGE_SHIFT,1, %r25
680 depwi,z 1,31-PAGE_SHIFT,1, %r25
686 1: fdc,m %r23(%sr3, %r26)
687 fdc,m %r23(%sr3, %r26)
688 fdc,m %r23(%sr3, %r26)
689 fdc,m %r23(%sr3, %r26)
690 fdc,m %r23(%sr3, %r26)
691 fdc,m %r23(%sr3, %r26)
692 fdc,m %r23(%sr3, %r26)
693 fdc,m %r23(%sr3, %r26)
694 fdc,m %r23(%sr3, %r26)
695 fdc,m %r23(%sr3, %r26)
696 fdc,m %r23(%sr3, %r26)
697 fdc,m %r23(%sr3, %r26)
698 fdc,m %r23(%sr3, %r26)
699 fdc,m %r23(%sr3, %r26)
700 fdc,m %r23(%sr3, %r26)
702 fdc,m %r23(%sr3, %r26)
711 .export flush_user_icache_page
713 flush_user_icache_page:
718 ldil L%dcache_stride, %r1
719 ldw R%dcache_stride(%r1), %r23
722 depdi,z 1, 63-PAGE_SHIFT,1, %r25
724 depwi,z 1, 31-PAGE_SHIFT,1, %r25
730 1: fic,m %r23(%sr3, %r26)
731 fic,m %r23(%sr3, %r26)
732 fic,m %r23(%sr3, %r26)
733 fic,m %r23(%sr3, %r26)
734 fic,m %r23(%sr3, %r26)
735 fic,m %r23(%sr3, %r26)
736 fic,m %r23(%sr3, %r26)
737 fic,m %r23(%sr3, %r26)
738 fic,m %r23(%sr3, %r26)
739 fic,m %r23(%sr3, %r26)
740 fic,m %r23(%sr3, %r26)
741 fic,m %r23(%sr3, %r26)
742 fic,m %r23(%sr3, %r26)
743 fic,m %r23(%sr3, %r26)
744 fic,m %r23(%sr3, %r26)
746 fic,m %r23(%sr3, %r26)
756 .export purge_kernel_dcache_page
758 purge_kernel_dcache_page:
763 ldil L%dcache_stride, %r1
764 ldw R%dcache_stride(%r1), %r23
767 depdi,z 1, 63-PAGE_SHIFT,1, %r25
769 depwi,z 1, 31-PAGE_SHIFT,1, %r25
789 CMPB<< %r26, %r25, 1b
800 /* Currently not used, but it still is a possible alternate
804 .export flush_alias_page
813 ldil L%(TMPALIAS_MAP_START), %r28
815 extrd,u %r26, 56,32, %r26 /* convert phys addr to tlb insert format */
816 depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
817 depdi 0, 63,12, %r28 /* Clear any offset bits */
819 extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
820 depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
821 depwi 0, 31,12, %r28 /* Clear any offset bits */
824 /* Purge any old translation */
828 ldil L%dcache_stride, %r1
829 ldw R%dcache_stride(%r1), %r23
832 depdi,z 1, 63-PAGE_SHIFT,1, %r29
834 depwi,z 1, 31-PAGE_SHIFT,1, %r29
854 CMPB<< %r28, %r29, 1b
865 .export flush_user_dcache_range_asm
867 flush_user_dcache_range_asm:
872 ldil L%dcache_stride, %r1
873 ldw R%dcache_stride(%r1), %r23
875 ANDCM %r26, %r21, %r26
877 1: CMPB<<,n %r26, %r25, 1b
878 fdc,m %r23(%sr3, %r26)
887 .export flush_kernel_dcache_range_asm
889 flush_kernel_dcache_range_asm:
894 ldil L%dcache_stride, %r1
895 ldw R%dcache_stride(%r1), %r23
897 ANDCM %r26, %r21, %r26
899 1: CMPB<<,n %r26, %r25,1b
910 .export flush_user_icache_range_asm
912 flush_user_icache_range_asm:
917 ldil L%icache_stride, %r1
918 ldw R%icache_stride(%r1), %r23
920 ANDCM %r26, %r21, %r26
922 1: CMPB<<,n %r26, %r25,1b
923 fic,m %r23(%sr3, %r26)
932 .export flush_kernel_icache_page
934 flush_kernel_icache_page:
939 ldil L%icache_stride, %r1
940 ldw R%icache_stride(%r1), %r23
943 depdi,z 1, 63-PAGE_SHIFT,1, %r25
945 depwi,z 1, 31-PAGE_SHIFT,1, %r25
951 1: fic,m %r23(%sr4, %r26)
952 fic,m %r23(%sr4, %r26)
953 fic,m %r23(%sr4, %r26)
954 fic,m %r23(%sr4, %r26)
955 fic,m %r23(%sr4, %r26)
956 fic,m %r23(%sr4, %r26)
957 fic,m %r23(%sr4, %r26)
958 fic,m %r23(%sr4, %r26)
959 fic,m %r23(%sr4, %r26)
960 fic,m %r23(%sr4, %r26)
961 fic,m %r23(%sr4, %r26)
962 fic,m %r23(%sr4, %r26)
963 fic,m %r23(%sr4, %r26)
964 fic,m %r23(%sr4, %r26)
965 fic,m %r23(%sr4, %r26)
966 CMPB<< %r26, %r25, 1b
967 fic,m %r23(%sr4, %r26)
976 .export flush_kernel_icache_range_asm
978 flush_kernel_icache_range_asm:
983 ldil L%icache_stride, %r1
984 ldw R%icache_stride(%r1), %r23
986 ANDCM %r26, %r21, %r26
988 1: CMPB<<,n %r26, %r25, 1b
989 fic,m %r23(%sr4, %r26)
997 /* align should cover use of rfi in disable_sr_hashing_asm and
1001 .export disable_sr_hashing_asm,code
1003 disable_sr_hashing_asm:
1009 * Switch to real mode
1020 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1021 mtctl %r0, %cr17 /* Clear IIASQ tail */
1022 mtctl %r0, %cr17 /* Clear IIASQ head */
1023 mtctl %r1, %cr18 /* IIAOQ head */
1025 mtctl %r1, %cr18 /* IIAOQ tail */
1026 load32 REAL_MODE_PSW, %r1
1031 1: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
1032 cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
1033 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
1038 /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1040 .word 0x141c1a00 /* mfdiag %dr0, %r28 */
1041 .word 0x141c1a00 /* must issue twice */
1042 depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
1043 depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
1044 .word 0x141c1600 /* mtdiag %r28, %dr0 */
1045 .word 0x141c1600 /* must issue twice */
1050 /* Disable Space Register Hashing for PCXL */
1052 .word 0x141c0600 /* mfdiag %dr0, %r28 */
1053 depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
1054 .word 0x141c0240 /* mtdiag %r28, %dr0 */
1059 /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1061 .word 0x144008bc /* mfdiag %dr2, %r28 */
1062 depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
1063 .word 0x145c1840 /* mtdiag %r28, %dr2 */
1067 /* Switch back to virtual mode */
1068 rsm PSW_SM_I, %r0 /* prep to load iia queue */
1076 rsm PSW_SM_Q, %r0 /* prep to load iia queue */
1077 mtctl %r0, %cr17 /* Clear IIASQ tail */
1078 mtctl %r0, %cr17 /* Clear IIASQ head */
1079 mtctl %r1, %cr18 /* IIAOQ head */
1081 mtctl %r1, %cr18 /* IIAOQ tail */
1082 load32 KERNEL_PSW, %r1