Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Here is where the ball gets rolling as far as the kernel is concerned. | |
3 | * When control is transferred to _start, the bootload has already | |
4 | * loaded us to the correct address. All that's left to do here is | |
5 | * to set up the kernel's global pointer and jump to the kernel | |
6 | * entry point. | |
7 | * | |
8 | * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co | |
9 | * David Mosberger-Tang <davidm@hpl.hp.com> | |
10 | * Stephane Eranian <eranian@hpl.hp.com> | |
11 | * Copyright (C) 1999 VA Linux Systems | |
12 | * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> | |
13 | * Copyright (C) 1999 Intel Corp. | |
14 | * Copyright (C) 1999 Asit Mallick <Asit.K.Mallick@intel.com> | |
15 | * Copyright (C) 1999 Don Dugger <Don.Dugger@intel.com> | |
16 | * Copyright (C) 2002 Fenghua Yu <fenghua.yu@intel.com> | |
17 | * -Optimize __ia64_save_fpu() and __ia64_load_fpu() for Itanium 2. | |
b8d8b883 AR |
18 | * Copyright (C) 2004 Ashok Raj <ashok.raj@intel.com> |
19 | * Support for CPU Hotplug | |
1da177e4 LT |
20 | */ |
21 | ||
1da177e4 LT |
22 | |
23 | #include <asm/asmmacro.h> | |
24 | #include <asm/fpu.h> | |
25 | #include <asm/kregs.h> | |
26 | #include <asm/mmu_context.h> | |
39e01cb8 | 27 | #include <asm/asm-offsets.h> |
1da177e4 | 28 | #include <asm/pal.h> |
3e0879de | 29 | #include <asm/paravirt.h> |
1da177e4 LT |
30 | #include <asm/pgtable.h> |
31 | #include <asm/processor.h> | |
32 | #include <asm/ptrace.h> | |
33 | #include <asm/system.h> | |
b8d8b883 | 34 | #include <asm/mca_asm.h> |
3e0879de IY |
35 | #include <linux/init.h> |
36 | #include <linux/linkage.h> | |
b8d8b883 AR |
37 | |
38 | #ifdef CONFIG_HOTPLUG_CPU | |
39 | #define SAL_PSR_BITS_TO_SET \ | |
40 | (IA64_PSR_AC | IA64_PSR_BN | IA64_PSR_MFH | IA64_PSR_MFL) | |
41 | ||
42 | #define SAVE_FROM_REG(src, ptr, dest) \ | |
43 | mov dest=src;; \ | |
44 | st8 [ptr]=dest,0x08 | |
45 | ||
46 | #define RESTORE_REG(reg, ptr, _tmp) \ | |
47 | ld8 _tmp=[ptr],0x08;; \ | |
48 | mov reg=_tmp | |
49 | ||
50 | #define SAVE_BREAK_REGS(ptr, _idx, _breg, _dest)\ | |
51 | mov ar.lc=IA64_NUM_DBG_REGS-1;; \ | |
52 | mov _idx=0;; \ | |
53 | 1: \ | |
54 | SAVE_FROM_REG(_breg[_idx], ptr, _dest);; \ | |
55 | add _idx=1,_idx;; \ | |
56 | br.cloop.sptk.many 1b | |
57 | ||
58 | #define RESTORE_BREAK_REGS(ptr, _idx, _breg, _tmp, _lbl)\ | |
59 | mov ar.lc=IA64_NUM_DBG_REGS-1;; \ | |
60 | mov _idx=0;; \ | |
61 | _lbl: RESTORE_REG(_breg[_idx], ptr, _tmp);; \ | |
62 | add _idx=1, _idx;; \ | |
63 | br.cloop.sptk.many _lbl | |
64 | ||
65 | #define SAVE_ONE_RR(num, _reg, _tmp) \ | |
66 | movl _tmp=(num<<61);; \ | |
67 | mov _reg=rr[_tmp] | |
68 | ||
69 | #define SAVE_REGION_REGS(_tmp, _r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7) \ | |
70 | SAVE_ONE_RR(0,_r0, _tmp);; \ | |
71 | SAVE_ONE_RR(1,_r1, _tmp);; \ | |
72 | SAVE_ONE_RR(2,_r2, _tmp);; \ | |
73 | SAVE_ONE_RR(3,_r3, _tmp);; \ | |
74 | SAVE_ONE_RR(4,_r4, _tmp);; \ | |
75 | SAVE_ONE_RR(5,_r5, _tmp);; \ | |
76 | SAVE_ONE_RR(6,_r6, _tmp);; \ | |
77 | SAVE_ONE_RR(7,_r7, _tmp);; | |
78 | ||
79 | #define STORE_REGION_REGS(ptr, _r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7) \ | |
80 | st8 [ptr]=_r0, 8;; \ | |
81 | st8 [ptr]=_r1, 8;; \ | |
82 | st8 [ptr]=_r2, 8;; \ | |
83 | st8 [ptr]=_r3, 8;; \ | |
84 | st8 [ptr]=_r4, 8;; \ | |
85 | st8 [ptr]=_r5, 8;; \ | |
86 | st8 [ptr]=_r6, 8;; \ | |
87 | st8 [ptr]=_r7, 8;; | |
88 | ||
89 | #define RESTORE_REGION_REGS(ptr, _idx1, _idx2, _tmp) \ | |
90 | mov ar.lc=0x08-1;; \ | |
91 | movl _idx1=0x00;; \ | |
92 | RestRR: \ | |
93 | dep.z _idx2=_idx1,61,3;; \ | |
94 | ld8 _tmp=[ptr],8;; \ | |
95 | mov rr[_idx2]=_tmp;; \ | |
96 | srlz.d;; \ | |
97 | add _idx1=1,_idx1;; \ | |
98 | br.cloop.sptk.few RestRR | |
99 | ||
df6c6804 AR |
100 | #define SET_AREA_FOR_BOOTING_CPU(reg1, reg2) \ |
101 | movl reg1=sal_state_for_booting_cpu;; \ | |
102 | ld8 reg2=[reg1];; | |
103 | ||
b8d8b883 AR |
104 | /* |
105 | * Adjust region registers saved before starting to save | |
106 | * break regs and rest of the states that need to be preserved. | |
107 | */ | |
108 | #define SAL_TO_OS_BOOT_HANDOFF_STATE_SAVE(_reg1,_reg2,_pred) \ | |
109 | SAVE_FROM_REG(b0,_reg1,_reg2);; \ | |
110 | SAVE_FROM_REG(b1,_reg1,_reg2);; \ | |
111 | SAVE_FROM_REG(b2,_reg1,_reg2);; \ | |
112 | SAVE_FROM_REG(b3,_reg1,_reg2);; \ | |
113 | SAVE_FROM_REG(b4,_reg1,_reg2);; \ | |
114 | SAVE_FROM_REG(b5,_reg1,_reg2);; \ | |
115 | st8 [_reg1]=r1,0x08;; \ | |
116 | st8 [_reg1]=r12,0x08;; \ | |
117 | st8 [_reg1]=r13,0x08;; \ | |
118 | SAVE_FROM_REG(ar.fpsr,_reg1,_reg2);; \ | |
119 | SAVE_FROM_REG(ar.pfs,_reg1,_reg2);; \ | |
120 | SAVE_FROM_REG(ar.rnat,_reg1,_reg2);; \ | |
121 | SAVE_FROM_REG(ar.unat,_reg1,_reg2);; \ | |
122 | SAVE_FROM_REG(ar.bspstore,_reg1,_reg2);; \ | |
123 | SAVE_FROM_REG(cr.dcr,_reg1,_reg2);; \ | |
124 | SAVE_FROM_REG(cr.iva,_reg1,_reg2);; \ | |
125 | SAVE_FROM_REG(cr.pta,_reg1,_reg2);; \ | |
126 | SAVE_FROM_REG(cr.itv,_reg1,_reg2);; \ | |
127 | SAVE_FROM_REG(cr.pmv,_reg1,_reg2);; \ | |
128 | SAVE_FROM_REG(cr.cmcv,_reg1,_reg2);; \ | |
129 | SAVE_FROM_REG(cr.lrr0,_reg1,_reg2);; \ | |
130 | SAVE_FROM_REG(cr.lrr1,_reg1,_reg2);; \ | |
131 | st8 [_reg1]=r4,0x08;; \ | |
132 | st8 [_reg1]=r5,0x08;; \ | |
133 | st8 [_reg1]=r6,0x08;; \ | |
134 | st8 [_reg1]=r7,0x08;; \ | |
135 | st8 [_reg1]=_pred,0x08;; \ | |
136 | SAVE_FROM_REG(ar.lc, _reg1, _reg2);; \ | |
137 | stf.spill.nta [_reg1]=f2,16;; \ | |
138 | stf.spill.nta [_reg1]=f3,16;; \ | |
139 | stf.spill.nta [_reg1]=f4,16;; \ | |
140 | stf.spill.nta [_reg1]=f5,16;; \ | |
141 | stf.spill.nta [_reg1]=f16,16;; \ | |
142 | stf.spill.nta [_reg1]=f17,16;; \ | |
143 | stf.spill.nta [_reg1]=f18,16;; \ | |
144 | stf.spill.nta [_reg1]=f19,16;; \ | |
145 | stf.spill.nta [_reg1]=f20,16;; \ | |
146 | stf.spill.nta [_reg1]=f21,16;; \ | |
147 | stf.spill.nta [_reg1]=f22,16;; \ | |
148 | stf.spill.nta [_reg1]=f23,16;; \ | |
149 | stf.spill.nta [_reg1]=f24,16;; \ | |
150 | stf.spill.nta [_reg1]=f25,16;; \ | |
151 | stf.spill.nta [_reg1]=f26,16;; \ | |
152 | stf.spill.nta [_reg1]=f27,16;; \ | |
153 | stf.spill.nta [_reg1]=f28,16;; \ | |
154 | stf.spill.nta [_reg1]=f29,16;; \ | |
155 | stf.spill.nta [_reg1]=f30,16;; \ | |
156 | stf.spill.nta [_reg1]=f31,16;; | |
157 | ||
158 | #else | |
df6c6804 AR |
159 | #define SET_AREA_FOR_BOOTING_CPU(a1, a2) |
160 | #define SAL_TO_OS_BOOT_HANDOFF_STATE_SAVE(a1,a2, a3) | |
b8d8b883 AR |
161 | #define SAVE_REGION_REGS(_tmp, _r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7) |
162 | #define STORE_REGION_REGS(ptr, _r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7) | |
163 | #endif | |
164 | ||
165 | #define SET_ONE_RR(num, pgsize, _tmp1, _tmp2, vhpt) \ | |
166 | movl _tmp1=(num << 61);; \ | |
167 | mov _tmp2=((ia64_rid(IA64_REGION_ID_KERNEL, (num<<61)) << 8) | (pgsize << 2) | vhpt);; \ | |
168 | mov rr[_tmp1]=_tmp2 | |
1da177e4 LT |
169 | |
170 | .section __special_page_section,"ax" | |
171 | ||
172 | .global empty_zero_page | |
173 | empty_zero_page: | |
174 | .skip PAGE_SIZE | |
175 | ||
176 | .global swapper_pg_dir | |
177 | swapper_pg_dir: | |
178 | .skip PAGE_SIZE | |
179 | ||
180 | .rodata | |
181 | halt_msg: | |
182 | stringz "Halting kernel\n" | |
183 | ||
9d6f40b8 | 184 | .section .text.head,"ax" |
1da177e4 LT |
185 | |
186 | .global start_ap | |
187 | ||
188 | /* | |
189 | * Start the kernel. When the bootloader passes control to _start(), r28 | |
190 | * points to the address of the boot parameter area. Execution reaches | |
191 | * here in physical mode. | |
192 | */ | |
193 | GLOBAL_ENTRY(_start) | |
194 | start_ap: | |
195 | .prologue | |
196 | .save rp, r0 // terminate unwind chain with a NULL rp | |
197 | .body | |
198 | ||
199 | rsm psr.i | psr.ic | |
200 | ;; | |
201 | srlz.i | |
202 | ;; | |
1c7d6707 JS |
203 | { |
204 | flushrs // must be first insn in group | |
205 | srlz.i | |
206 | } | |
207 | ;; | |
b8d8b883 AR |
208 | /* |
209 | * Save the region registers, predicate before they get clobbered | |
210 | */ | |
211 | SAVE_REGION_REGS(r2, r8,r9,r10,r11,r12,r13,r14,r15); | |
212 | mov r25=pr;; | |
213 | ||
1da177e4 LT |
214 | /* |
215 | * Initialize kernel region registers: | |
216 | * rr[0]: VHPT enabled, page size = PAGE_SHIFT | |
217 | * rr[1]: VHPT enabled, page size = PAGE_SHIFT | |
218 | * rr[2]: VHPT enabled, page size = PAGE_SHIFT | |
219 | * rr[3]: VHPT enabled, page size = PAGE_SHIFT | |
220 | * rr[4]: VHPT enabled, page size = PAGE_SHIFT | |
221 | * rr[5]: VHPT enabled, page size = PAGE_SHIFT | |
222 | * rr[6]: VHPT disabled, page size = IA64_GRANULE_SHIFT | |
223 | * rr[7]: VHPT disabled, page size = IA64_GRANULE_SHIFT | |
224 | * We initialize all of them to prevent inadvertently assuming | |
225 | * something about the state of address translation early in boot. | |
226 | */ | |
b8d8b883 AR |
227 | SET_ONE_RR(0, PAGE_SHIFT, r2, r16, 1);; |
228 | SET_ONE_RR(1, PAGE_SHIFT, r2, r16, 1);; | |
229 | SET_ONE_RR(2, PAGE_SHIFT, r2, r16, 1);; | |
230 | SET_ONE_RR(3, PAGE_SHIFT, r2, r16, 1);; | |
231 | SET_ONE_RR(4, PAGE_SHIFT, r2, r16, 1);; | |
232 | SET_ONE_RR(5, PAGE_SHIFT, r2, r16, 1);; | |
233 | SET_ONE_RR(6, IA64_GRANULE_SHIFT, r2, r16, 0);; | |
234 | SET_ONE_RR(7, IA64_GRANULE_SHIFT, r2, r16, 0);; | |
1da177e4 LT |
235 | /* |
236 | * Now pin mappings into the TLB for kernel text and data | |
237 | */ | |
238 | mov r18=KERNEL_TR_PAGE_SHIFT<<2 | |
239 | movl r17=KERNEL_START | |
240 | ;; | |
241 | mov cr.itir=r18 | |
242 | mov cr.ifa=r17 | |
243 | mov r16=IA64_TR_KERNEL | |
244 | mov r3=ip | |
245 | movl r18=PAGE_KERNEL | |
246 | ;; | |
247 | dep r2=0,r3,0,KERNEL_TR_PAGE_SHIFT | |
248 | ;; | |
249 | or r18=r2,r18 | |
250 | ;; | |
251 | srlz.i | |
252 | ;; | |
253 | itr.i itr[r16]=r18 | |
254 | ;; | |
255 | itr.d dtr[r16]=r18 | |
256 | ;; | |
257 | srlz.i | |
258 | ||
259 | /* | |
260 | * Switch into virtual mode: | |
261 | */ | |
262 | movl r16=(IA64_PSR_IT|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFH|IA64_PSR_BN \ | |
b704882e | 263 | |IA64_PSR_DI|IA64_PSR_AC) |
1da177e4 LT |
264 | ;; |
265 | mov cr.ipsr=r16 | |
266 | movl r17=1f | |
267 | ;; | |
268 | mov cr.iip=r17 | |
269 | mov cr.ifs=r0 | |
270 | ;; | |
271 | rfi | |
272 | ;; | |
273 | 1: // now we are in virtual mode | |
274 | ||
df6c6804 | 275 | SET_AREA_FOR_BOOTING_CPU(r2, r16); |
b8d8b883 AR |
276 | |
277 | STORE_REGION_REGS(r16, r8,r9,r10,r11,r12,r13,r14,r15); | |
278 | SAL_TO_OS_BOOT_HANDOFF_STATE_SAVE(r16,r17,r25) | |
279 | ;; | |
280 | ||
1da177e4 LT |
281 | // set IVT entry point---can't access I/O ports without it |
282 | movl r3=ia64_ivt | |
283 | ;; | |
284 | mov cr.iva=r3 | |
285 | movl r2=FPSR_DEFAULT | |
286 | ;; | |
287 | srlz.i | |
288 | movl gp=__gp | |
289 | ||
290 | mov ar.fpsr=r2 | |
291 | ;; | |
292 | ||
293 | #define isAP p2 // are we an Application Processor? | |
294 | #define isBP p3 // are we the Bootstrap Processor? | |
295 | ||
296 | #ifdef CONFIG_SMP | |
297 | /* | |
298 | * Find the init_task for the currently booting CPU. At poweron, and in | |
299 | * UP mode, task_for_booting_cpu is NULL. | |
300 | */ | |
301 | movl r3=task_for_booting_cpu | |
302 | ;; | |
303 | ld8 r3=[r3] | |
304 | movl r2=init_task | |
305 | ;; | |
306 | cmp.eq isBP,isAP=r3,r0 | |
307 | ;; | |
308 | (isAP) mov r2=r3 | |
309 | #else | |
310 | movl r2=init_task | |
311 | cmp.eq isBP,isAP=r0,r0 | |
312 | #endif | |
313 | ;; | |
314 | tpa r3=r2 // r3 == phys addr of task struct | |
315 | mov r16=-1 | |
316 | (isBP) br.cond.dpnt .load_current // BP stack is on region 5 --- no need to map it | |
317 | ||
318 | // load mapping for stack (virtaddr in r2, physaddr in r3) | |
319 | rsm psr.ic | |
320 | movl r17=PAGE_KERNEL | |
321 | ;; | |
322 | srlz.d | |
323 | dep r18=0,r3,0,12 | |
324 | ;; | |
325 | or r18=r17,r18 | |
326 | dep r2=-1,r3,61,3 // IMVA of task | |
327 | ;; | |
328 | mov r17=rr[r2] | |
329 | shr.u r16=r3,IA64_GRANULE_SHIFT | |
330 | ;; | |
331 | dep r17=0,r17,8,24 | |
332 | ;; | |
333 | mov cr.itir=r17 | |
334 | mov cr.ifa=r2 | |
335 | ||
336 | mov r19=IA64_TR_CURRENT_STACK | |
337 | ;; | |
338 | itr.d dtr[r19]=r18 | |
339 | ;; | |
340 | ssm psr.ic | |
341 | srlz.d | |
342 | ;; | |
343 | ||
344 | .load_current: | |
345 | // load the "current" pointer (r13) and ar.k6 with the current task | |
346 | mov IA64_KR(CURRENT)=r2 // virtual address | |
347 | mov IA64_KR(CURRENT_STACK)=r16 | |
348 | mov r13=r2 | |
349 | /* | |
b8d8b883 AR |
350 | * Reserve space at the top of the stack for "struct pt_regs". Kernel |
351 | * threads don't store interesting values in that structure, but the space | |
352 | * still needs to be there because time-critical stuff such as the context | |
353 | * switching can be implemented more efficiently (for example, __switch_to() | |
1da177e4 LT |
354 | * always sets the psr.dfh bit of the task it is switching to). |
355 | */ | |
b8d8b883 | 356 | |
1da177e4 LT |
357 | addl r12=IA64_STK_OFFSET-IA64_PT_REGS_SIZE-16,r2 |
358 | addl r2=IA64_RBS_OFFSET,r2 // initialize the RSE | |
359 | mov ar.rsc=0 // place RSE in enforced lazy mode | |
360 | ;; | |
361 | loadrs // clear the dirty partition | |
10617bbe TL |
362 | movl r19=__phys_per_cpu_start |
363 | mov r18=PERCPU_PAGE_SIZE | |
364 | ;; | |
365 | #ifndef CONFIG_SMP | |
366 | add r19=r19,r18 | |
367 | ;; | |
368 | #else | |
369 | (isAP) br.few 2f | |
c459ce8b | 370 | movl r20=__cpu0_per_cpu |
10617bbe TL |
371 | ;; |
372 | shr.u r18=r18,3 | |
373 | 1: | |
c459ce8b TL |
374 | ld8 r21=[r19],8;; |
375 | st8[r20]=r21,8 | |
10617bbe TL |
376 | adds r18=-1,r18;; |
377 | cmp4.lt p7,p6=0,r18 | |
378 | (p7) br.cond.dptk.few 1b | |
c459ce8b TL |
379 | mov r19=r20 |
380 | ;; | |
10617bbe TL |
381 | 2: |
382 | #endif | |
383 | tpa r19=r19 | |
384 | ;; | |
385 | .pred.rel.mutex isBP,isAP | |
386 | (isBP) mov IA64_KR(PER_CPU_DATA)=r19 // per-CPU base for cpu0 | |
387 | (isAP) mov IA64_KR(PER_CPU_DATA)=r0 // clear physical per-CPU base | |
1da177e4 LT |
388 | ;; |
389 | mov ar.bspstore=r2 // establish the new RSE stack | |
390 | ;; | |
391 | mov ar.rsc=0x3 // place RSE in eager mode | |
392 | ||
393 | (isBP) dep r28=-1,r28,61,3 // make address virtual | |
394 | (isBP) movl r2=ia64_boot_param | |
395 | ;; | |
396 | (isBP) st8 [r2]=r28 // save the address of the boot param area passed by the bootloader | |
397 | ||
3e0879de IY |
398 | #ifdef CONFIG_PARAVIRT |
399 | ||
400 | movl r14=hypervisor_setup_hooks | |
401 | movl r15=hypervisor_type | |
402 | mov r16=num_hypervisor_hooks | |
403 | ;; | |
404 | ld8 r2=[r15] | |
405 | ;; | |
406 | cmp.ltu p7,p0=r2,r16 // array size check | |
407 | shladd r8=r2,3,r14 | |
408 | ;; | |
409 | (p7) ld8 r9=[r8] | |
410 | ;; | |
411 | (p7) mov b1=r9 | |
412 | (p7) cmp.ne.unc p7,p0=r9,r0 // no actual branch to NULL | |
413 | ;; | |
414 | (p7) br.call.sptk.many rp=b1 | |
415 | ||
416 | __INITDATA | |
417 | ||
418 | default_setup_hook = 0 // Currently nothing needs to be done. | |
419 | ||
420 | .weak xen_setup_hook | |
421 | ||
422 | .global hypervisor_type | |
423 | hypervisor_type: | |
424 | data8 PARAVIRT_HYPERVISOR_TYPE_DEFAULT | |
425 | ||
426 | // must have the same order with PARAVIRT_HYPERVISOR_TYPE_xxx | |
427 | ||
428 | hypervisor_setup_hooks: | |
429 | data8 default_setup_hook | |
430 | data8 xen_setup_hook | |
431 | num_hypervisor_hooks = (. - hypervisor_setup_hooks) / 8 | |
432 | .previous | |
433 | ||
434 | #endif | |
435 | ||
1da177e4 LT |
436 | #ifdef CONFIG_SMP |
437 | (isAP) br.call.sptk.many rp=start_secondary | |
438 | .ret0: | |
439 | (isAP) br.cond.sptk self | |
440 | #endif | |
441 | ||
442 | // This is executed by the bootstrap processor (bsp) only: | |
443 | ||
444 | #ifdef CONFIG_IA64_FW_EMU | |
445 | // initialize PAL & SAL emulator: | |
446 | br.call.sptk.many rp=sys_fw_init | |
447 | .ret1: | |
448 | #endif | |
449 | br.call.sptk.many rp=start_kernel | |
450 | .ret2: addl r3=@ltoff(halt_msg),gp | |
451 | ;; | |
452 | alloc r2=ar.pfs,8,0,2,0 | |
453 | ;; | |
454 | ld8 out0=[r3] | |
455 | br.call.sptk.many b0=console_print | |
456 | ||
457 | self: hint @pause | |
458 | br.sptk.many self // endless loop | |
459 | END(_start) | |
460 | ||
9d6f40b8 TL |
461 | .text |
462 | ||
1da177e4 LT |
463 | GLOBAL_ENTRY(ia64_save_debug_regs) |
464 | alloc r16=ar.pfs,1,0,0,0 | |
465 | mov r20=ar.lc // preserve ar.lc | |
466 | mov ar.lc=IA64_NUM_DBG_REGS-1 | |
467 | mov r18=0 | |
468 | add r19=IA64_NUM_DBG_REGS*8,in0 | |
469 | ;; | |
470 | 1: mov r16=dbr[r18] | |
471 | #ifdef CONFIG_ITANIUM | |
472 | ;; | |
473 | srlz.d | |
474 | #endif | |
475 | mov r17=ibr[r18] | |
476 | add r18=1,r18 | |
477 | ;; | |
478 | st8.nta [in0]=r16,8 | |
479 | st8.nta [r19]=r17,8 | |
480 | br.cloop.sptk.many 1b | |
481 | ;; | |
482 | mov ar.lc=r20 // restore ar.lc | |
483 | br.ret.sptk.many rp | |
484 | END(ia64_save_debug_regs) | |
485 | ||
486 | GLOBAL_ENTRY(ia64_load_debug_regs) | |
487 | alloc r16=ar.pfs,1,0,0,0 | |
488 | lfetch.nta [in0] | |
489 | mov r20=ar.lc // preserve ar.lc | |
490 | add r19=IA64_NUM_DBG_REGS*8,in0 | |
491 | mov ar.lc=IA64_NUM_DBG_REGS-1 | |
492 | mov r18=-1 | |
493 | ;; | |
494 | 1: ld8.nta r16=[in0],8 | |
495 | ld8.nta r17=[r19],8 | |
496 | add r18=1,r18 | |
497 | ;; | |
498 | mov dbr[r18]=r16 | |
499 | #ifdef CONFIG_ITANIUM | |
500 | ;; | |
501 | srlz.d // Errata 132 (NoFix status) | |
502 | #endif | |
503 | mov ibr[r18]=r17 | |
504 | br.cloop.sptk.many 1b | |
505 | ;; | |
506 | mov ar.lc=r20 // restore ar.lc | |
507 | br.ret.sptk.many rp | |
508 | END(ia64_load_debug_regs) | |
509 | ||
510 | GLOBAL_ENTRY(__ia64_save_fpu) | |
511 | alloc r2=ar.pfs,1,4,0,0 | |
512 | adds loc0=96*16-16,in0 | |
513 | adds loc1=96*16-16-128,in0 | |
514 | ;; | |
515 | stf.spill.nta [loc0]=f127,-256 | |
516 | stf.spill.nta [loc1]=f119,-256 | |
517 | ;; | |
518 | stf.spill.nta [loc0]=f111,-256 | |
519 | stf.spill.nta [loc1]=f103,-256 | |
520 | ;; | |
521 | stf.spill.nta [loc0]=f95,-256 | |
522 | stf.spill.nta [loc1]=f87,-256 | |
523 | ;; | |
524 | stf.spill.nta [loc0]=f79,-256 | |
525 | stf.spill.nta [loc1]=f71,-256 | |
526 | ;; | |
527 | stf.spill.nta [loc0]=f63,-256 | |
528 | stf.spill.nta [loc1]=f55,-256 | |
529 | adds loc2=96*16-32,in0 | |
530 | ;; | |
531 | stf.spill.nta [loc0]=f47,-256 | |
532 | stf.spill.nta [loc1]=f39,-256 | |
533 | adds loc3=96*16-32-128,in0 | |
534 | ;; | |
535 | stf.spill.nta [loc2]=f126,-256 | |
536 | stf.spill.nta [loc3]=f118,-256 | |
537 | ;; | |
538 | stf.spill.nta [loc2]=f110,-256 | |
539 | stf.spill.nta [loc3]=f102,-256 | |
540 | ;; | |
541 | stf.spill.nta [loc2]=f94,-256 | |
542 | stf.spill.nta [loc3]=f86,-256 | |
543 | ;; | |
544 | stf.spill.nta [loc2]=f78,-256 | |
545 | stf.spill.nta [loc3]=f70,-256 | |
546 | ;; | |
547 | stf.spill.nta [loc2]=f62,-256 | |
548 | stf.spill.nta [loc3]=f54,-256 | |
549 | adds loc0=96*16-48,in0 | |
550 | ;; | |
551 | stf.spill.nta [loc2]=f46,-256 | |
552 | stf.spill.nta [loc3]=f38,-256 | |
553 | adds loc1=96*16-48-128,in0 | |
554 | ;; | |
555 | stf.spill.nta [loc0]=f125,-256 | |
556 | stf.spill.nta [loc1]=f117,-256 | |
557 | ;; | |
558 | stf.spill.nta [loc0]=f109,-256 | |
559 | stf.spill.nta [loc1]=f101,-256 | |
560 | ;; | |
561 | stf.spill.nta [loc0]=f93,-256 | |
562 | stf.spill.nta [loc1]=f85,-256 | |
563 | ;; | |
564 | stf.spill.nta [loc0]=f77,-256 | |
565 | stf.spill.nta [loc1]=f69,-256 | |
566 | ;; | |
567 | stf.spill.nta [loc0]=f61,-256 | |
568 | stf.spill.nta [loc1]=f53,-256 | |
569 | adds loc2=96*16-64,in0 | |
570 | ;; | |
571 | stf.spill.nta [loc0]=f45,-256 | |
572 | stf.spill.nta [loc1]=f37,-256 | |
573 | adds loc3=96*16-64-128,in0 | |
574 | ;; | |
575 | stf.spill.nta [loc2]=f124,-256 | |
576 | stf.spill.nta [loc3]=f116,-256 | |
577 | ;; | |
578 | stf.spill.nta [loc2]=f108,-256 | |
579 | stf.spill.nta [loc3]=f100,-256 | |
580 | ;; | |
581 | stf.spill.nta [loc2]=f92,-256 | |
582 | stf.spill.nta [loc3]=f84,-256 | |
583 | ;; | |
584 | stf.spill.nta [loc2]=f76,-256 | |
585 | stf.spill.nta [loc3]=f68,-256 | |
586 | ;; | |
587 | stf.spill.nta [loc2]=f60,-256 | |
588 | stf.spill.nta [loc3]=f52,-256 | |
589 | adds loc0=96*16-80,in0 | |
590 | ;; | |
591 | stf.spill.nta [loc2]=f44,-256 | |
592 | stf.spill.nta [loc3]=f36,-256 | |
593 | adds loc1=96*16-80-128,in0 | |
594 | ;; | |
595 | stf.spill.nta [loc0]=f123,-256 | |
596 | stf.spill.nta [loc1]=f115,-256 | |
597 | ;; | |
598 | stf.spill.nta [loc0]=f107,-256 | |
599 | stf.spill.nta [loc1]=f99,-256 | |
600 | ;; | |
601 | stf.spill.nta [loc0]=f91,-256 | |
602 | stf.spill.nta [loc1]=f83,-256 | |
603 | ;; | |
604 | stf.spill.nta [loc0]=f75,-256 | |
605 | stf.spill.nta [loc1]=f67,-256 | |
606 | ;; | |
607 | stf.spill.nta [loc0]=f59,-256 | |
608 | stf.spill.nta [loc1]=f51,-256 | |
609 | adds loc2=96*16-96,in0 | |
610 | ;; | |
611 | stf.spill.nta [loc0]=f43,-256 | |
612 | stf.spill.nta [loc1]=f35,-256 | |
613 | adds loc3=96*16-96-128,in0 | |
614 | ;; | |
615 | stf.spill.nta [loc2]=f122,-256 | |
616 | stf.spill.nta [loc3]=f114,-256 | |
617 | ;; | |
618 | stf.spill.nta [loc2]=f106,-256 | |
619 | stf.spill.nta [loc3]=f98,-256 | |
620 | ;; | |
621 | stf.spill.nta [loc2]=f90,-256 | |
622 | stf.spill.nta [loc3]=f82,-256 | |
623 | ;; | |
624 | stf.spill.nta [loc2]=f74,-256 | |
625 | stf.spill.nta [loc3]=f66,-256 | |
626 | ;; | |
627 | stf.spill.nta [loc2]=f58,-256 | |
628 | stf.spill.nta [loc3]=f50,-256 | |
629 | adds loc0=96*16-112,in0 | |
630 | ;; | |
631 | stf.spill.nta [loc2]=f42,-256 | |
632 | stf.spill.nta [loc3]=f34,-256 | |
633 | adds loc1=96*16-112-128,in0 | |
634 | ;; | |
635 | stf.spill.nta [loc0]=f121,-256 | |
636 | stf.spill.nta [loc1]=f113,-256 | |
637 | ;; | |
638 | stf.spill.nta [loc0]=f105,-256 | |
639 | stf.spill.nta [loc1]=f97,-256 | |
640 | ;; | |
641 | stf.spill.nta [loc0]=f89,-256 | |
642 | stf.spill.nta [loc1]=f81,-256 | |
643 | ;; | |
644 | stf.spill.nta [loc0]=f73,-256 | |
645 | stf.spill.nta [loc1]=f65,-256 | |
646 | ;; | |
647 | stf.spill.nta [loc0]=f57,-256 | |
648 | stf.spill.nta [loc1]=f49,-256 | |
649 | adds loc2=96*16-128,in0 | |
650 | ;; | |
651 | stf.spill.nta [loc0]=f41,-256 | |
652 | stf.spill.nta [loc1]=f33,-256 | |
653 | adds loc3=96*16-128-128,in0 | |
654 | ;; | |
655 | stf.spill.nta [loc2]=f120,-256 | |
656 | stf.spill.nta [loc3]=f112,-256 | |
657 | ;; | |
658 | stf.spill.nta [loc2]=f104,-256 | |
659 | stf.spill.nta [loc3]=f96,-256 | |
660 | ;; | |
661 | stf.spill.nta [loc2]=f88,-256 | |
662 | stf.spill.nta [loc3]=f80,-256 | |
663 | ;; | |
664 | stf.spill.nta [loc2]=f72,-256 | |
665 | stf.spill.nta [loc3]=f64,-256 | |
666 | ;; | |
667 | stf.spill.nta [loc2]=f56,-256 | |
668 | stf.spill.nta [loc3]=f48,-256 | |
669 | ;; | |
670 | stf.spill.nta [loc2]=f40 | |
671 | stf.spill.nta [loc3]=f32 | |
672 | br.ret.sptk.many rp | |
673 | END(__ia64_save_fpu) | |
674 | ||
675 | GLOBAL_ENTRY(__ia64_load_fpu) | |
676 | alloc r2=ar.pfs,1,2,0,0 | |
677 | adds r3=128,in0 | |
678 | adds r14=256,in0 | |
679 | adds r15=384,in0 | |
680 | mov loc0=512 | |
681 | mov loc1=-1024+16 | |
682 | ;; | |
683 | ldf.fill.nta f32=[in0],loc0 | |
684 | ldf.fill.nta f40=[ r3],loc0 | |
685 | ldf.fill.nta f48=[r14],loc0 | |
686 | ldf.fill.nta f56=[r15],loc0 | |
687 | ;; | |
688 | ldf.fill.nta f64=[in0],loc0 | |
689 | ldf.fill.nta f72=[ r3],loc0 | |
690 | ldf.fill.nta f80=[r14],loc0 | |
691 | ldf.fill.nta f88=[r15],loc0 | |
692 | ;; | |
693 | ldf.fill.nta f96=[in0],loc1 | |
694 | ldf.fill.nta f104=[ r3],loc1 | |
695 | ldf.fill.nta f112=[r14],loc1 | |
696 | ldf.fill.nta f120=[r15],loc1 | |
697 | ;; | |
698 | ldf.fill.nta f33=[in0],loc0 | |
699 | ldf.fill.nta f41=[ r3],loc0 | |
700 | ldf.fill.nta f49=[r14],loc0 | |
701 | ldf.fill.nta f57=[r15],loc0 | |
702 | ;; | |
703 | ldf.fill.nta f65=[in0],loc0 | |
704 | ldf.fill.nta f73=[ r3],loc0 | |
705 | ldf.fill.nta f81=[r14],loc0 | |
706 | ldf.fill.nta f89=[r15],loc0 | |
707 | ;; | |
708 | ldf.fill.nta f97=[in0],loc1 | |
709 | ldf.fill.nta f105=[ r3],loc1 | |
710 | ldf.fill.nta f113=[r14],loc1 | |
711 | ldf.fill.nta f121=[r15],loc1 | |
712 | ;; | |
713 | ldf.fill.nta f34=[in0],loc0 | |
714 | ldf.fill.nta f42=[ r3],loc0 | |
715 | ldf.fill.nta f50=[r14],loc0 | |
716 | ldf.fill.nta f58=[r15],loc0 | |
717 | ;; | |
718 | ldf.fill.nta f66=[in0],loc0 | |
719 | ldf.fill.nta f74=[ r3],loc0 | |
720 | ldf.fill.nta f82=[r14],loc0 | |
721 | ldf.fill.nta f90=[r15],loc0 | |
722 | ;; | |
723 | ldf.fill.nta f98=[in0],loc1 | |
724 | ldf.fill.nta f106=[ r3],loc1 | |
725 | ldf.fill.nta f114=[r14],loc1 | |
726 | ldf.fill.nta f122=[r15],loc1 | |
727 | ;; | |
728 | ldf.fill.nta f35=[in0],loc0 | |
729 | ldf.fill.nta f43=[ r3],loc0 | |
730 | ldf.fill.nta f51=[r14],loc0 | |
731 | ldf.fill.nta f59=[r15],loc0 | |
732 | ;; | |
733 | ldf.fill.nta f67=[in0],loc0 | |
734 | ldf.fill.nta f75=[ r3],loc0 | |
735 | ldf.fill.nta f83=[r14],loc0 | |
736 | ldf.fill.nta f91=[r15],loc0 | |
737 | ;; | |
738 | ldf.fill.nta f99=[in0],loc1 | |
739 | ldf.fill.nta f107=[ r3],loc1 | |
740 | ldf.fill.nta f115=[r14],loc1 | |
741 | ldf.fill.nta f123=[r15],loc1 | |
742 | ;; | |
743 | ldf.fill.nta f36=[in0],loc0 | |
744 | ldf.fill.nta f44=[ r3],loc0 | |
745 | ldf.fill.nta f52=[r14],loc0 | |
746 | ldf.fill.nta f60=[r15],loc0 | |
747 | ;; | |
748 | ldf.fill.nta f68=[in0],loc0 | |
749 | ldf.fill.nta f76=[ r3],loc0 | |
750 | ldf.fill.nta f84=[r14],loc0 | |
751 | ldf.fill.nta f92=[r15],loc0 | |
752 | ;; | |
753 | ldf.fill.nta f100=[in0],loc1 | |
754 | ldf.fill.nta f108=[ r3],loc1 | |
755 | ldf.fill.nta f116=[r14],loc1 | |
756 | ldf.fill.nta f124=[r15],loc1 | |
757 | ;; | |
758 | ldf.fill.nta f37=[in0],loc0 | |
759 | ldf.fill.nta f45=[ r3],loc0 | |
760 | ldf.fill.nta f53=[r14],loc0 | |
761 | ldf.fill.nta f61=[r15],loc0 | |
762 | ;; | |
763 | ldf.fill.nta f69=[in0],loc0 | |
764 | ldf.fill.nta f77=[ r3],loc0 | |
765 | ldf.fill.nta f85=[r14],loc0 | |
766 | ldf.fill.nta f93=[r15],loc0 | |
767 | ;; | |
768 | ldf.fill.nta f101=[in0],loc1 | |
769 | ldf.fill.nta f109=[ r3],loc1 | |
770 | ldf.fill.nta f117=[r14],loc1 | |
771 | ldf.fill.nta f125=[r15],loc1 | |
772 | ;; | |
773 | ldf.fill.nta f38 =[in0],loc0 | |
774 | ldf.fill.nta f46 =[ r3],loc0 | |
775 | ldf.fill.nta f54 =[r14],loc0 | |
776 | ldf.fill.nta f62 =[r15],loc0 | |
777 | ;; | |
778 | ldf.fill.nta f70 =[in0],loc0 | |
779 | ldf.fill.nta f78 =[ r3],loc0 | |
780 | ldf.fill.nta f86 =[r14],loc0 | |
781 | ldf.fill.nta f94 =[r15],loc0 | |
782 | ;; | |
783 | ldf.fill.nta f102=[in0],loc1 | |
784 | ldf.fill.nta f110=[ r3],loc1 | |
785 | ldf.fill.nta f118=[r14],loc1 | |
786 | ldf.fill.nta f126=[r15],loc1 | |
787 | ;; | |
788 | ldf.fill.nta f39 =[in0],loc0 | |
789 | ldf.fill.nta f47 =[ r3],loc0 | |
790 | ldf.fill.nta f55 =[r14],loc0 | |
791 | ldf.fill.nta f63 =[r15],loc0 | |
792 | ;; | |
793 | ldf.fill.nta f71 =[in0],loc0 | |
794 | ldf.fill.nta f79 =[ r3],loc0 | |
795 | ldf.fill.nta f87 =[r14],loc0 | |
796 | ldf.fill.nta f95 =[r15],loc0 | |
797 | ;; | |
798 | ldf.fill.nta f103=[in0] | |
799 | ldf.fill.nta f111=[ r3] | |
800 | ldf.fill.nta f119=[r14] | |
801 | ldf.fill.nta f127=[r15] | |
802 | br.ret.sptk.many rp | |
803 | END(__ia64_load_fpu) | |
804 | ||
805 | GLOBAL_ENTRY(__ia64_init_fpu) | |
806 | stf.spill [sp]=f0 // M3 | |
807 | mov f32=f0 // F | |
808 | nop.b 0 | |
809 | ||
810 | ldfps f33,f34=[sp] // M0 | |
811 | ldfps f35,f36=[sp] // M1 | |
812 | mov f37=f0 // F | |
813 | ;; | |
814 | ||
815 | setf.s f38=r0 // M2 | |
816 | setf.s f39=r0 // M3 | |
817 | mov f40=f0 // F | |
818 | ||
819 | ldfps f41,f42=[sp] // M0 | |
820 | ldfps f43,f44=[sp] // M1 | |
821 | mov f45=f0 // F | |
822 | ||
823 | setf.s f46=r0 // M2 | |
824 | setf.s f47=r0 // M3 | |
825 | mov f48=f0 // F | |
826 | ||
827 | ldfps f49,f50=[sp] // M0 | |
828 | ldfps f51,f52=[sp] // M1 | |
829 | mov f53=f0 // F | |
830 | ||
831 | setf.s f54=r0 // M2 | |
832 | setf.s f55=r0 // M3 | |
833 | mov f56=f0 // F | |
834 | ||
835 | ldfps f57,f58=[sp] // M0 | |
836 | ldfps f59,f60=[sp] // M1 | |
837 | mov f61=f0 // F | |
838 | ||
839 | setf.s f62=r0 // M2 | |
840 | setf.s f63=r0 // M3 | |
841 | mov f64=f0 // F | |
842 | ||
843 | ldfps f65,f66=[sp] // M0 | |
844 | ldfps f67,f68=[sp] // M1 | |
845 | mov f69=f0 // F | |
846 | ||
847 | setf.s f70=r0 // M2 | |
848 | setf.s f71=r0 // M3 | |
849 | mov f72=f0 // F | |
850 | ||
851 | ldfps f73,f74=[sp] // M0 | |
852 | ldfps f75,f76=[sp] // M1 | |
853 | mov f77=f0 // F | |
854 | ||
855 | setf.s f78=r0 // M2 | |
856 | setf.s f79=r0 // M3 | |
857 | mov f80=f0 // F | |
858 | ||
859 | ldfps f81,f82=[sp] // M0 | |
860 | ldfps f83,f84=[sp] // M1 | |
861 | mov f85=f0 // F | |
862 | ||
863 | setf.s f86=r0 // M2 | |
864 | setf.s f87=r0 // M3 | |
865 | mov f88=f0 // F | |
866 | ||
867 | /* | |
868 | * When the instructions are cached, it would be faster to initialize | |
869 | * the remaining registers with simply mov instructions (F-unit). | |
870 | * This gets the time down to ~29 cycles. However, this would use up | |
871 | * 33 bundles, whereas continuing with the above pattern yields | |
872 | * 10 bundles and ~30 cycles. | |
873 | */ | |
874 | ||
875 | ldfps f89,f90=[sp] // M0 | |
876 | ldfps f91,f92=[sp] // M1 | |
877 | mov f93=f0 // F | |
878 | ||
879 | setf.s f94=r0 // M2 | |
880 | setf.s f95=r0 // M3 | |
881 | mov f96=f0 // F | |
882 | ||
883 | ldfps f97,f98=[sp] // M0 | |
884 | ldfps f99,f100=[sp] // M1 | |
885 | mov f101=f0 // F | |
886 | ||
887 | setf.s f102=r0 // M2 | |
888 | setf.s f103=r0 // M3 | |
889 | mov f104=f0 // F | |
890 | ||
891 | ldfps f105,f106=[sp] // M0 | |
892 | ldfps f107,f108=[sp] // M1 | |
893 | mov f109=f0 // F | |
894 | ||
895 | setf.s f110=r0 // M2 | |
896 | setf.s f111=r0 // M3 | |
897 | mov f112=f0 // F | |
898 | ||
899 | ldfps f113,f114=[sp] // M0 | |
900 | ldfps f115,f116=[sp] // M1 | |
901 | mov f117=f0 // F | |
902 | ||
903 | setf.s f118=r0 // M2 | |
904 | setf.s f119=r0 // M3 | |
905 | mov f120=f0 // F | |
906 | ||
907 | ldfps f121,f122=[sp] // M0 | |
908 | ldfps f123,f124=[sp] // M1 | |
909 | mov f125=f0 // F | |
910 | ||
911 | setf.s f126=r0 // M2 | |
912 | setf.s f127=r0 // M3 | |
913 | br.ret.sptk.many rp // F | |
914 | END(__ia64_init_fpu) | |
915 | ||
916 | /* | |
917 | * Switch execution mode from virtual to physical | |
918 | * | |
919 | * Inputs: | |
920 | * r16 = new psr to establish | |
921 | * Output: | |
922 | * r19 = old virtual address of ar.bsp | |
923 | * r20 = old virtual address of sp | |
924 | * | |
925 | * Note: RSE must already be in enforced lazy mode | |
926 | */ | |
927 | GLOBAL_ENTRY(ia64_switch_mode_phys) | |
928 | { | |
1da177e4 LT |
929 | rsm psr.i | psr.ic // disable interrupts and interrupt collection |
930 | mov r15=ip | |
931 | } | |
932 | ;; | |
933 | { | |
934 | flushrs // must be first insn in group | |
935 | srlz.i | |
936 | } | |
937 | ;; | |
938 | mov cr.ipsr=r16 // set new PSR | |
939 | add r3=1f-ia64_switch_mode_phys,r15 | |
940 | ||
941 | mov r19=ar.bsp | |
942 | mov r20=sp | |
943 | mov r14=rp // get return address into a general register | |
944 | ;; | |
945 | ||
946 | // going to physical mode, use tpa to translate virt->phys | |
947 | tpa r17=r19 | |
948 | tpa r3=r3 | |
949 | tpa sp=sp | |
950 | tpa r14=r14 | |
951 | ;; | |
952 | ||
953 | mov r18=ar.rnat // save ar.rnat | |
954 | mov ar.bspstore=r17 // this steps on ar.rnat | |
955 | mov cr.iip=r3 | |
956 | mov cr.ifs=r0 | |
957 | ;; | |
958 | mov ar.rnat=r18 // restore ar.rnat | |
959 | rfi // must be last insn in group | |
960 | ;; | |
961 | 1: mov rp=r14 | |
962 | br.ret.sptk.many rp | |
963 | END(ia64_switch_mode_phys) | |
964 | ||
965 | /* | |
966 | * Switch execution mode from physical to virtual | |
967 | * | |
968 | * Inputs: | |
969 | * r16 = new psr to establish | |
970 | * r19 = new bspstore to establish | |
971 | * r20 = new sp to establish | |
972 | * | |
973 | * Note: RSE must already be in enforced lazy mode | |
974 | */ | |
975 | GLOBAL_ENTRY(ia64_switch_mode_virt) | |
976 | { | |
1da177e4 LT |
977 | rsm psr.i | psr.ic // disable interrupts and interrupt collection |
978 | mov r15=ip | |
979 | } | |
980 | ;; | |
981 | { | |
982 | flushrs // must be first insn in group | |
983 | srlz.i | |
984 | } | |
985 | ;; | |
986 | mov cr.ipsr=r16 // set new PSR | |
987 | add r3=1f-ia64_switch_mode_virt,r15 | |
988 | ||
989 | mov r14=rp // get return address into a general register | |
990 | ;; | |
991 | ||
992 | // going to virtual | |
993 | // - for code addresses, set upper bits of addr to KERNEL_START | |
994 | // - for stack addresses, copy from input argument | |
995 | movl r18=KERNEL_START | |
996 | dep r3=0,r3,KERNEL_TR_PAGE_SHIFT,64-KERNEL_TR_PAGE_SHIFT | |
997 | dep r14=0,r14,KERNEL_TR_PAGE_SHIFT,64-KERNEL_TR_PAGE_SHIFT | |
998 | mov sp=r20 | |
999 | ;; | |
1000 | or r3=r3,r18 | |
1001 | or r14=r14,r18 | |
1002 | ;; | |
1003 | ||
1004 | mov r18=ar.rnat // save ar.rnat | |
1005 | mov ar.bspstore=r19 // this steps on ar.rnat | |
1006 | mov cr.iip=r3 | |
1007 | mov cr.ifs=r0 | |
1008 | ;; | |
1009 | mov ar.rnat=r18 // restore ar.rnat | |
1010 | rfi // must be last insn in group | |
1011 | ;; | |
1012 | 1: mov rp=r14 | |
1013 | br.ret.sptk.many rp | |
1014 | END(ia64_switch_mode_virt) | |
1015 | ||
1016 | GLOBAL_ENTRY(ia64_delay_loop) | |
1017 | .prologue | |
1018 | { nop 0 // work around GAS unwind info generation bug... | |
1019 | .save ar.lc,r2 | |
1020 | mov r2=ar.lc | |
1021 | .body | |
1022 | ;; | |
1023 | mov ar.lc=r32 | |
1024 | } | |
1025 | ;; | |
1026 | // force loop to be 32-byte aligned (GAS bug means we cannot use .align | |
1027 | // inside function body without corrupting unwind info). | |
1028 | { nop 0 } | |
1029 | 1: br.cloop.sptk.few 1b | |
1030 | ;; | |
1031 | mov ar.lc=r2 | |
1032 | br.ret.sptk.many rp | |
1033 | END(ia64_delay_loop) | |
1034 | ||
1035 | /* | |
1036 | * Return a CPU-local timestamp in nano-seconds. This timestamp is | |
1037 | * NOT synchronized across CPUs its return value must never be | |
1038 | * compared against the values returned on another CPU. The usage in | |
1039 | * kernel/sched.c ensures that. | |
1040 | * | |
1041 | * The return-value of sched_clock() is NOT supposed to wrap-around. | |
1042 | * If it did, it would cause some scheduling hiccups (at the worst). | |
1043 | * Fortunately, with a 64-bit cycle-counter ticking at 100GHz, even | |
1044 | * that would happen only once every 5+ years. | |
1045 | * | |
1046 | * The code below basically calculates: | |
1047 | * | |
1048 | * (ia64_get_itc() * local_cpu_data->nsec_per_cyc) >> IA64_NSEC_PER_CYC_SHIFT | |
1049 | * | |
1050 | * except that the multiplication and the shift are done with 128-bit | |
1051 | * intermediate precision so that we can produce a full 64-bit result. | |
1052 | */ | |
f927da17 | 1053 | GLOBAL_ENTRY(ia64_native_sched_clock) |
1da177e4 LT |
1054 | addl r8=THIS_CPU(cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 |
1055 | mov.m r9=ar.itc // fetch cycle-counter (35 cyc) | |
1056 | ;; | |
1057 | ldf8 f8=[r8] | |
1058 | ;; | |
1059 | setf.sig f9=r9 // certain to stall, so issue it _after_ ldf8... | |
1060 | ;; | |
1061 | xmpy.lu f10=f9,f8 // calculate low 64 bits of 128-bit product (4 cyc) | |
1062 | xmpy.hu f11=f9,f8 // calculate high 64 bits of 128-bit product | |
1063 | ;; | |
1064 | getf.sig r8=f10 // (5 cyc) | |
1065 | getf.sig r9=f11 | |
1066 | ;; | |
1067 | shrp r8=r9,r8,IA64_NSEC_PER_CYC_SHIFT | |
1068 | br.ret.sptk.many rp | |
f927da17 IY |
1069 | END(ia64_native_sched_clock) |
1070 | #ifndef CONFIG_PARAVIRT | |
1071 | //unsigned long long | |
1072 | //sched_clock(void) __attribute__((alias("ia64_native_sched_clock"))); | |
1073 | .global sched_clock | |
1074 | sched_clock = ia64_native_sched_clock | |
1075 | #endif | |
1da177e4 | 1076 | |
b64f34cd HS |
1077 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
1078 | GLOBAL_ENTRY(cycle_to_cputime) | |
1079 | alloc r16=ar.pfs,1,0,0,0 | |
1080 | addl r8=THIS_CPU(cpu_info) + IA64_CPUINFO_NSEC_PER_CYC_OFFSET,r0 | |
1081 | ;; | |
1082 | ldf8 f8=[r8] | |
1083 | ;; | |
1084 | setf.sig f9=r32 | |
1085 | ;; | |
1086 | xmpy.lu f10=f9,f8 // calculate low 64 bits of 128-bit product (4 cyc) | |
1087 | xmpy.hu f11=f9,f8 // calculate high 64 bits of 128-bit product | |
1088 | ;; | |
1089 | getf.sig r8=f10 // (5 cyc) | |
1090 | getf.sig r9=f11 | |
1091 | ;; | |
1092 | shrp r8=r9,r8,IA64_NSEC_PER_CYC_SHIFT | |
1093 | br.ret.sptk.many rp | |
1094 | END(cycle_to_cputime) | |
1095 | #endif /* CONFIG_VIRT_CPU_ACCOUNTING */ | |
1096 | ||
1da177e4 LT |
1097 | GLOBAL_ENTRY(start_kernel_thread) |
1098 | .prologue | |
1099 | .save rp, r0 // this is the end of the call-chain | |
1100 | .body | |
1101 | alloc r2 = ar.pfs, 0, 0, 2, 0 | |
1102 | mov out0 = r9 | |
1103 | mov out1 = r11;; | |
1104 | br.call.sptk.many rp = kernel_thread_helper;; | |
1105 | mov out0 = r8 | |
1106 | br.call.sptk.many rp = sys_exit;; | |
1107 | 1: br.sptk.few 1b // not reached | |
1108 | END(start_kernel_thread) | |
1109 | ||
1110 | #ifdef CONFIG_IA64_BRL_EMU | |
1111 | ||
1112 | /* | |
1113 | * Assembly routines used by brl_emu.c to set preserved register state. | |
1114 | */ | |
1115 | ||
1116 | #define SET_REG(reg) \ | |
1117 | GLOBAL_ENTRY(ia64_set_##reg); \ | |
1118 | alloc r16=ar.pfs,1,0,0,0; \ | |
1119 | mov reg=r32; \ | |
1120 | ;; \ | |
1121 | br.ret.sptk.many rp; \ | |
1122 | END(ia64_set_##reg) | |
1123 | ||
1124 | SET_REG(b1); | |
1125 | SET_REG(b2); | |
1126 | SET_REG(b3); | |
1127 | SET_REG(b4); | |
1128 | SET_REG(b5); | |
1129 | ||
1130 | #endif /* CONFIG_IA64_BRL_EMU */ | |
1131 | ||
1132 | #ifdef CONFIG_SMP | |
1133 | /* | |
1134 | * This routine handles spinlock contention. It uses a non-standard calling | |
1135 | * convention to avoid converting leaf routines into interior routines. Because | |
1136 | * of this special convention, there are several restrictions: | |
1137 | * | |
1138 | * - do not use gp relative variables, this code is called from the kernel | |
1139 | * and from modules, r1 is undefined. | |
1140 | * - do not use stacked registers, the caller owns them. | |
1141 | * - do not use the scratch stack space, the caller owns it. | |
1142 | * - do not use any registers other than the ones listed below | |
1143 | * | |
1144 | * Inputs: | |
1145 | * ar.pfs - saved CFM of caller | |
1146 | * ar.ccv - 0 (and available for use) | |
1147 | * r27 - flags from spin_lock_irqsave or 0. Must be preserved. | |
1148 | * r28 - available for use. | |
1149 | * r29 - available for use. | |
1150 | * r30 - available for use. | |
1151 | * r31 - address of lock, available for use. | |
1152 | * b6 - return address | |
1153 | * p14 - available for use. | |
1154 | * p15 - used to track flag status. | |
1155 | * | |
1156 | * If you patch this code to use more registers, do not forget to update | |
7f30491c | 1157 | * the clobber lists for spin_lock() in arch/ia64/include/asm/spinlock.h. |
1da177e4 LT |
1158 | */ |
1159 | ||
a1365647 | 1160 | #if (__GNUC__ == 3 && __GNUC_MINOR__ < 3) |
1da177e4 LT |
1161 | |
1162 | GLOBAL_ENTRY(ia64_spinlock_contention_pre3_4) | |
1163 | .prologue | |
1164 | .save ar.pfs, r0 // this code effectively has a zero frame size | |
1165 | .save rp, r28 | |
1166 | .body | |
1167 | nop 0 | |
1168 | tbit.nz p15,p0=r27,IA64_PSR_I_BIT | |
1169 | .restore sp // pop existing prologue after next insn | |
1170 | mov b6 = r28 | |
1171 | .prologue | |
1172 | .save ar.pfs, r0 | |
1173 | .altrp b6 | |
1174 | .body | |
1175 | ;; | |
1176 | (p15) ssm psr.i // reenable interrupts if they were on | |
1177 | // DavidM says that srlz.d is slow and is not required in this case | |
1178 | .wait: | |
1179 | // exponential backoff, kdb, lockmeter etc. go in here | |
1180 | hint @pause | |
1181 | ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word | |
1182 | nop 0 | |
1183 | ;; | |
1184 | cmp4.ne p14,p0=r30,r0 | |
1185 | (p14) br.cond.sptk.few .wait | |
1186 | (p15) rsm psr.i // disable interrupts if we reenabled them | |
1187 | br.cond.sptk.few b6 // lock is now free, try to acquire | |
1188 | .global ia64_spinlock_contention_pre3_4_end // for kernprof | |
1189 | ia64_spinlock_contention_pre3_4_end: | |
1190 | END(ia64_spinlock_contention_pre3_4) | |
1191 | ||
1192 | #else | |
1193 | ||
1194 | GLOBAL_ENTRY(ia64_spinlock_contention) | |
1195 | .prologue | |
1196 | .altrp b6 | |
1197 | .body | |
1198 | tbit.nz p15,p0=r27,IA64_PSR_I_BIT | |
1199 | ;; | |
1200 | .wait: | |
1201 | (p15) ssm psr.i // reenable interrupts if they were on | |
1202 | // DavidM says that srlz.d is slow and is not required in this case | |
1203 | .wait2: | |
1204 | // exponential backoff, kdb, lockmeter etc. go in here | |
1205 | hint @pause | |
1206 | ld4 r30=[r31] // don't use ld4.bias; if it's contended, we won't write the word | |
1207 | ;; | |
1208 | cmp4.ne p14,p0=r30,r0 | |
1209 | mov r30 = 1 | |
1210 | (p14) br.cond.sptk.few .wait2 | |
1211 | (p15) rsm psr.i // disable interrupts if we reenabled them | |
1212 | ;; | |
1213 | cmpxchg4.acq r30=[r31], r30, ar.ccv | |
1214 | ;; | |
1215 | cmp4.ne p14,p0=r0,r30 | |
1216 | (p14) br.cond.sptk.few .wait | |
1217 | ||
1218 | br.ret.sptk.many b6 // lock is now taken | |
1219 | END(ia64_spinlock_contention) | |
1220 | ||
1221 | #endif | |
1222 | ||
b8d8b883 AR |
1223 | #ifdef CONFIG_HOTPLUG_CPU |
1224 | GLOBAL_ENTRY(ia64_jump_to_sal) | |
1225 | alloc r16=ar.pfs,1,0,0,0;; | |
1226 | rsm psr.i | psr.ic | |
1227 | { | |
1228 | flushrs | |
1229 | srlz.i | |
1230 | } | |
1231 | tpa r25=in0 | |
1232 | movl r18=tlb_purge_done;; | |
1233 | DATA_VA_TO_PA(r18);; | |
1234 | mov b1=r18 // Return location | |
1235 | movl r18=ia64_do_tlb_purge;; | |
1236 | DATA_VA_TO_PA(r18);; | |
1237 | mov b2=r18 // doing tlb_flush work | |
1238 | mov ar.rsc=0 // Put RSE in enforced lazy, LE mode | |
1239 | movl r17=1f;; | |
1240 | DATA_VA_TO_PA(r17);; | |
1241 | mov cr.iip=r17 | |
1242 | movl r16=SAL_PSR_BITS_TO_SET;; | |
1243 | mov cr.ipsr=r16 | |
1244 | mov cr.ifs=r0;; | |
1245 | rfi;; | |
1246 | 1: | |
1247 | /* | |
1248 | * Invalidate all TLB data/inst | |
1249 | */ | |
1250 | br.sptk.many b2;; // jump to tlb purge code | |
1251 | ||
1252 | tlb_purge_done: | |
1253 | RESTORE_REGION_REGS(r25, r17,r18,r19);; | |
1254 | RESTORE_REG(b0, r25, r17);; | |
1255 | RESTORE_REG(b1, r25, r17);; | |
1256 | RESTORE_REG(b2, r25, r17);; | |
1257 | RESTORE_REG(b3, r25, r17);; | |
1258 | RESTORE_REG(b4, r25, r17);; | |
1259 | RESTORE_REG(b5, r25, r17);; | |
1260 | ld8 r1=[r25],0x08;; | |
1261 | ld8 r12=[r25],0x08;; | |
1262 | ld8 r13=[r25],0x08;; | |
1263 | RESTORE_REG(ar.fpsr, r25, r17);; | |
1264 | RESTORE_REG(ar.pfs, r25, r17);; | |
1265 | RESTORE_REG(ar.rnat, r25, r17);; | |
1266 | RESTORE_REG(ar.unat, r25, r17);; | |
1267 | RESTORE_REG(ar.bspstore, r25, r17);; | |
1268 | RESTORE_REG(cr.dcr, r25, r17);; | |
1269 | RESTORE_REG(cr.iva, r25, r17);; | |
1270 | RESTORE_REG(cr.pta, r25, r17);; | |
09106228 | 1271 | srlz.d;; // required not to violate RAW dependency |
b8d8b883 AR |
1272 | RESTORE_REG(cr.itv, r25, r17);; |
1273 | RESTORE_REG(cr.pmv, r25, r17);; | |
1274 | RESTORE_REG(cr.cmcv, r25, r17);; | |
1275 | RESTORE_REG(cr.lrr0, r25, r17);; | |
1276 | RESTORE_REG(cr.lrr1, r25, r17);; | |
1277 | ld8 r4=[r25],0x08;; | |
1278 | ld8 r5=[r25],0x08;; | |
1279 | ld8 r6=[r25],0x08;; | |
1280 | ld8 r7=[r25],0x08;; | |
1281 | ld8 r17=[r25],0x08;; | |
1282 | mov pr=r17,-1;; | |
1283 | RESTORE_REG(ar.lc, r25, r17);; | |
1284 | /* | |
1285 | * Now Restore floating point regs | |
1286 | */ | |
1287 | ldf.fill.nta f2=[r25],16;; | |
1288 | ldf.fill.nta f3=[r25],16;; | |
1289 | ldf.fill.nta f4=[r25],16;; | |
1290 | ldf.fill.nta f5=[r25],16;; | |
1291 | ldf.fill.nta f16=[r25],16;; | |
1292 | ldf.fill.nta f17=[r25],16;; | |
1293 | ldf.fill.nta f18=[r25],16;; | |
1294 | ldf.fill.nta f19=[r25],16;; | |
1295 | ldf.fill.nta f20=[r25],16;; | |
1296 | ldf.fill.nta f21=[r25],16;; | |
1297 | ldf.fill.nta f22=[r25],16;; | |
1298 | ldf.fill.nta f23=[r25],16;; | |
1299 | ldf.fill.nta f24=[r25],16;; | |
1300 | ldf.fill.nta f25=[r25],16;; | |
1301 | ldf.fill.nta f26=[r25],16;; | |
1302 | ldf.fill.nta f27=[r25],16;; | |
1303 | ldf.fill.nta f28=[r25],16;; | |
1304 | ldf.fill.nta f29=[r25],16;; | |
1305 | ldf.fill.nta f30=[r25],16;; | |
1306 | ldf.fill.nta f31=[r25],16;; | |
1307 | ||
1308 | /* | |
1309 | * Now that we have done all the register restores | |
1310 | * we are now ready for the big DIVE to SAL Land | |
1311 | */ | |
1312 | ssm psr.ic;; | |
1313 | srlz.d;; | |
1314 | br.ret.sptk.many b0;; | |
1315 | END(ia64_jump_to_sal) | |
1316 | #endif /* CONFIG_HOTPLUG_CPU */ | |
1317 | ||
1da177e4 | 1318 | #endif /* CONFIG_SMP */ |