[deliverable/linux.git] / arch / x86 / power / cpu_64.c

/*
 * Suspend and hibernation support for x86-64
 *
 * Distribute under GPLv2
 *
 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
 * Copyright (c) 2002 Pavel Machek <pavel@suse.cz>
 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
 */

#include <linux/suspend.h>
#include <linux/smp.h>

#include <asm/pgtable.h>
#include <asm/proto.h>
#include <asm/mtrr.h>
#include <asm/page.h>
#include <asm/mce.h>
#include <asm/xcr.h>
#include <asm/suspend.h>

#ifdef CONFIG_X86_32
static struct saved_context saved_context;

unsigned long saved_context_ebx;
unsigned long saved_context_esp, saved_context_ebp;
unsigned long saved_context_esi, saved_context_edi;
unsigned long saved_context_eflags;
#else
/* CONFIG_X86_64 */
struct saved_context saved_context;
#endif

/**
 *	__save_processor_state - save CPU registers before creating a
 *		hibernation image and before restoring the memory state from it
 *	@ctxt - structure to store the registers contents in
 *
 *	NOTE: If there is a CPU register the modification of which by the
 *	boot kernel (ie. the kernel used for loading the hibernation image)
 *	might affect the operations of the restored target kernel (ie. the one
 *	saved in the hibernation image), then its contents must be saved by this
 *	function.  In other words, if kernel A is hibernated and different
 *	kernel B is used for loading the hibernation image into memory, the
 *	kernel A's __save_processor_state() function must save all registers
 *	needed by kernel A, so that it can operate correctly after the resume
 *	regardless of what kernel B does in the meantime.
 */
static void __save_processor_state(struct saved_context *ctxt)
{
#ifdef CONFIG_X86_32
	mtrr_save_fixed_ranges(NULL);
#endif
	kernel_fpu_begin();

	/*
	 * descriptor tables
	 */
#ifdef CONFIG_X86_32
	store_gdt(&ctxt->gdt);
	store_idt(&ctxt->idt);
#else
/* CONFIG_X86_64 */
	store_gdt((struct desc_ptr *)&ctxt->gdt_limit);
	store_idt((struct desc_ptr *)&ctxt->idt_limit);
#endif
	store_tr(ctxt->tr);

	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
	/*
	 * segment registers
	 */
#ifdef CONFIG_X86_32
	savesegment(es, ctxt->es);
	savesegment(fs, ctxt->fs);
	savesegment(gs, ctxt->gs);
	savesegment(ss, ctxt->ss);
#else
/* CONFIG_X86_64 */
	asm volatile ("movw %%ds, %0" : "=m" (ctxt->ds));
	asm volatile ("movw %%es, %0" : "=m" (ctxt->es));
	asm volatile ("movw %%fs, %0" : "=m" (ctxt->fs));
	asm volatile ("movw %%gs, %0" : "=m" (ctxt->gs));
	asm volatile ("movw %%ss, %0" : "=m" (ctxt->ss));

	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
	rdmsrl(MSR_GS_BASE, ctxt->gs_base);
	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
	mtrr_save_fixed_ranges(NULL);

	rdmsrl(MSR_EFER, ctxt->efer);
#endif

	/*
	 * control registers
	 */
	ctxt->cr0 = read_cr0();
	ctxt->cr2 = read_cr2();
	ctxt->cr3 = read_cr3();
#ifdef CONFIG_X86_32
	ctxt->cr4 = read_cr4_safe();
#else
/* CONFIG_X86_64 */
	ctxt->cr4 = read_cr4();
	ctxt->cr8 = read_cr8();
#endif
}

/* Needed by apm.c */
void save_processor_state(void)
{
	__save_processor_state(&saved_context);
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(save_processor_state);
#endif

static void do_fpu_end(void)
{
	/*
	 * Restore FPU regs if necessary.
	 */
	kernel_fpu_end();
}

static void fix_processor_context(void)
{
	int cpu = smp_processor_id();
	struct tss_struct *t = &per_cpu(init_tss, cpu);

	set_tss_desc(cpu, t);	/*
				 * This just modifies memory; should not be
				 * necessary. But... This is necessary, because
				 * 386 hardware has concept of busy TSS or some
				 * similar stupidity.
				 */

#ifdef CONFIG_X86_64
	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;

	syscall_init();				/* This sets MSR_*STAR and related */
#endif
	load_TR_desc();				/* This does ltr */
	load_LDT(&current->active_mm->context);	/* This does lldt */

	/*
	 * Now maybe reload the debug registers
	 */
	if (current->thread.debugreg7) {
#ifdef CONFIG_X86_32
		set_debugreg(current->thread.debugreg0, 0);
		set_debugreg(current->thread.debugreg1, 1);
		set_debugreg(current->thread.debugreg2, 2);
		set_debugreg(current->thread.debugreg3, 3);
		/* no 4 and 5 */
		set_debugreg(current->thread.debugreg6, 6);
		set_debugreg(current->thread.debugreg7, 7);
#else
		/* CONFIG_X86_64 */
		loaddebug(&current->thread, 0);
		loaddebug(&current->thread, 1);
		loaddebug(&current->thread, 2);
		loaddebug(&current->thread, 3);
		/* no 4 and 5 */
		loaddebug(&current->thread, 6);
		loaddebug(&current->thread, 7);
#endif
	}

}

/**
 *	__restore_processor_state - restore the contents of CPU registers saved
 *		by __save_processor_state()
 *	@ctxt - structure to load the registers contents from
 */
static void __restore_processor_state(struct saved_context *ctxt)
{
	/*
	 * control registers
	 */
	/* cr4 was introduced in the Pentium CPU */
#ifdef CONFIG_X86_32
	if (ctxt->cr4)
		write_cr4(ctxt->cr4);
#else
/* CONFIG X86_64 */
	wrmsrl(MSR_EFER, ctxt->efer);
	write_cr8(ctxt->cr8);
	write_cr4(ctxt->cr4);
#endif
	write_cr3(ctxt->cr3);
	write_cr2(ctxt->cr2);
	write_cr0(ctxt->cr0);

	/*
	 * now restore the descriptor tables to their proper values
	 * ltr is done i fix_processor_context().
	 */
#ifdef CONFIG_X86_32
	load_gdt(&ctxt->gdt);
	load_idt(&ctxt->idt);
#else
/* CONFIG_X86_64 */
	load_gdt((const struct desc_ptr *)&ctxt->gdt_limit);
	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
#endif

	/*
	 * segment registers
	 */
#ifdef CONFIG_X86_32
	loadsegment(es, ctxt->es);
	loadsegment(fs, ctxt->fs);
	loadsegment(gs, ctxt->gs);
	loadsegment(ss, ctxt->ss);

	/*
	 * sysenter MSRs
	 */
	if (boot_cpu_has(X86_FEATURE_SEP))
		enable_sep_cpu();
#else
/* CONFIG_X86_64 */
	asm volatile ("movw %0, %%ds" :: "r" (ctxt->ds));
	asm volatile ("movw %0, %%es" :: "r" (ctxt->es));
	asm volatile ("movw %0, %%fs" :: "r" (ctxt->fs));
	load_gs_index(ctxt->gs);
	asm volatile ("movw %0, %%ss" :: "r" (ctxt->ss));

	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
	wrmsrl(MSR_GS_BASE, ctxt->gs_base);
	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
#endif

	/*
	 * restore XCR0 for xsave capable cpu's.
	 */
	if (cpu_has_xsave)
		xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);

	fix_processor_context();

	do_fpu_end();
	mtrr_ap_init();

#ifdef CONFIG_X86_32
	mcheck_init(&boot_cpu_data);
#endif
}

/* Needed by apm.c */
void restore_processor_state(void)
{
	__restore_processor_state(&saved_context);
}
#ifdef CONFIG_X86_32
EXPORT_SYMBOL(restore_processor_state);
#endif
Commit	Line	Data
1da177e4	1	/*
cf7700fe	2	* Suspend and hibernation support for x86-64
1da177e4 LT	3	*
	4	* Distribute under GPLv2
	5	*
cf7700fe	6	* Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
1da177e4 LT	7	* Copyright (c) 2002 Pavel Machek <pavel@suse.cz>
	8	* Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
	9	*/
	10
1da177e4	11	#include <linux/suspend.h>
f6783d20 SL	12	#include <linux/smp.h>
f6783d20 SL	13
3dd08325	14	#include <asm/pgtable.h>
f6783d20	15	#include <asm/proto.h>
3ebad590	16	#include <asm/mtrr.h>
f6783d20 SL	17	#include <asm/page.h>
f6783d20 SL	18	#include <asm/mce.h>
83b8e28b	19	#include <asm/xcr.h>
a8af7898	20	#include <asm/suspend.h>
1da177e4	21
833b2ca0 SL	22	#ifdef CONFIG_X86_32
	23	static struct saved_context saved_context;
	24
	25	unsigned long saved_context_ebx;
	26	unsigned long saved_context_esp, saved_context_ebp;
	27	unsigned long saved_context_esi, saved_context_edi;
	28	unsigned long saved_context_eflags;
	29	#else
	30	/* CONFIG_X86_64 */
1da177e4	31	struct saved_context saved_context;
833b2ca0	32	#endif
1da177e4	33
5c9c9bec RW	34	/**
	35	* __save_processor_state - save CPU registers before creating a
	36	* hibernation image and before restoring the memory state from it
	37	* @ctxt - structure to store the registers contents in
	38	*
	39	* NOTE: If there is a CPU register the modification of which by the
	40	* boot kernel (ie. the kernel used for loading the hibernation image)
	41	* might affect the operations of the restored target kernel (ie. the one
	42	* saved in the hibernation image), then its contents must be saved by this
	43	* function. In other words, if kernel A is hibernated and different
	44	* kernel B is used for loading the hibernation image into memory, the
	45	* kernel A's __save_processor_state() function must save all registers
	46	* needed by kernel A, so that it can operate correctly after the resume
	47	* regardless of what kernel B does in the meantime.
	48	*/
cae45957	49	static void __save_processor_state(struct saved_context *ctxt)
1da177e4	50	{
f9ebbe53 SL	51	#ifdef CONFIG_X86_32
	52	mtrr_save_fixed_ranges(NULL);
	53	#endif
1da177e4 LT	54	kernel_fpu_begin();
	55
	56	/*
	57	* descriptor tables
	58	*/
f9ebbe53 SL	59	#ifdef CONFIG_X86_32
	60	store_gdt(&ctxt->gdt);
	61	store_idt(&ctxt->idt);
	62	#else
	63	/* CONFIG_X86_64 */
9d1c6e7c GOC	64	store_gdt((struct desc_ptr *)&ctxt->gdt_limit);
9d1c6e7c GOC	65	store_idt((struct desc_ptr *)&ctxt->idt_limit);
f9ebbe53	66	#endif
9d1c6e7c	67	store_tr(ctxt->tr);
1da177e4 LT	68
1da177e4 LT	69	/* XMM0..XMM15 should be handled by kernel_fpu_begin(). */
1da177e4 LT	70	/*
	71	* segment registers
	72	*/
f9ebbe53 SL	73	#ifdef CONFIG_X86_32
	74	savesegment(es, ctxt->es);
	75	savesegment(fs, ctxt->fs);
	76	savesegment(gs, ctxt->gs);
	77	savesegment(ss, ctxt->ss);
	78	#else
	79	/* CONFIG_X86_64 */
1da177e4 LT	80	asm volatile ("movw %%ds, %0" : "=m" (ctxt->ds));
	81	asm volatile ("movw %%es, %0" : "=m" (ctxt->es));
	82	asm volatile ("movw %%fs, %0" : "=m" (ctxt->fs));
	83	asm volatile ("movw %%gs, %0" : "=m" (ctxt->gs));
	84	asm volatile ("movw %%ss, %0" : "=m" (ctxt->ss));
	85
	86	rdmsrl(MSR_FS_BASE, ctxt->fs_base);
	87	rdmsrl(MSR_GS_BASE, ctxt->gs_base);
	88	rdmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
3ebad590	89	mtrr_save_fixed_ranges(NULL);
1da177e4	90
f9ebbe53 SL	91	rdmsrl(MSR_EFER, ctxt->efer);
	92	#endif
	93
1da177e4	94	/*
cf7700fe	95	* control registers
1da177e4	96	*/
f51c9452 GOC	97	ctxt->cr0 = read_cr0();
	98	ctxt->cr2 = read_cr2();
	99	ctxt->cr3 = read_cr3();
f9ebbe53 SL	100	#ifdef CONFIG_X86_32
	101	ctxt->cr4 = read_cr4_safe();
	102	#else
	103	/* CONFIG_X86_64 */
f51c9452 GOC	104	ctxt->cr4 = read_cr4();
f51c9452 GOC	105	ctxt->cr8 = read_cr8();
f9ebbe53	106	#endif
1da177e4 LT	107	}
1da177e4 LT	108
f9ebbe53	109	/* Needed by apm.c */
1da177e4 LT	110	void save_processor_state(void)
	111	{
	112	__save_processor_state(&saved_context);
	113	}
f9ebbe53 SL	114	#ifdef CONFIG_X86_32
	115	EXPORT_SYMBOL(save_processor_state);
	116	#endif
1da177e4	117
08967f94	118	static void do_fpu_end(void)
1da177e4	119	{
08967f94	120	/*
3134d04b	121	* Restore FPU regs if necessary.
08967f94 SL	122	*/
08967f94 SL	123	kernel_fpu_end();
1da177e4 LT	124	}
1da177e4 LT	125
3134d04b SL	126	static void fix_processor_context(void)
	127	{
	128	int cpu = smp_processor_id();
	129	struct tss_struct *t = &per_cpu(init_tss, cpu);
	130
	131	set_tss_desc(cpu, t); /*
	132	* This just modifies memory; should not be
	133	* necessary. But... This is necessary, because
	134	* 386 hardware has concept of busy TSS or some
	135	* similar stupidity.
	136	*/
	137
	138	#ifdef CONFIG_X86_64
	139	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;
	140
	141	syscall_init(); /* This sets MSR_STAR and related /
	142	#endif
	143	load_TR_desc(); /* This does ltr */
	144	load_LDT(&current->active_mm->context); /* This does lldt */
	145
	146	/*
	147	* Now maybe reload the debug registers
	148	*/
	149	if (current->thread.debugreg7) {
	150	#ifdef CONFIG_X86_32
	151	set_debugreg(current->thread.debugreg0, 0);
	152	set_debugreg(current->thread.debugreg1, 1);
	153	set_debugreg(current->thread.debugreg2, 2);
	154	set_debugreg(current->thread.debugreg3, 3);
	155	/* no 4 and 5 */
	156	set_debugreg(current->thread.debugreg6, 6);
	157	set_debugreg(current->thread.debugreg7, 7);
	158	#else
	159	/* CONFIG_X86_64 */
	160	loaddebug(&current->thread, 0);
	161	loaddebug(&current->thread, 1);
	162	loaddebug(&current->thread, 2);
	163	loaddebug(&current->thread, 3);
	164	/* no 4 and 5 */
	165	loaddebug(&current->thread, 6);
	166	loaddebug(&current->thread, 7);
	167	#endif
	168	}
	169
	170	}
	171
5c9c9bec RW	172	/**
	173	* __restore_processor_state - restore the contents of CPU registers saved
	174	* by __save_processor_state()
	175	* @ctxt - structure to load the registers contents from
	176	*/
cae45957	177	static void __restore_processor_state(struct saved_context *ctxt)
1da177e4 LT	178	{
	179	/*
	180	* control registers
	181	*/
3134d04b SL	182	/* cr4 was introduced in the Pentium CPU */
	183	#ifdef CONFIG_X86_32
	184	if (ctxt->cr4)
	185	write_cr4(ctxt->cr4);
	186	#else
	187	/* CONFIG X86_64 */
3c321bce	188	wrmsrl(MSR_EFER, ctxt->efer);
f51c9452 GOC	189	write_cr8(ctxt->cr8);
f51c9452 GOC	190	write_cr4(ctxt->cr4);
3134d04b	191	#endif
f51c9452 GOC	192	write_cr3(ctxt->cr3);
	193	write_cr2(ctxt->cr2);
	194	write_cr0(ctxt->cr0);
1da177e4	195
8d783b3e PM	196	/*
	197	* now restore the descriptor tables to their proper values
	198	* ltr is done i fix_processor_context().
	199	*/
3134d04b SL	200	#ifdef CONFIG_X86_32
	201	load_gdt(&ctxt->gdt);
	202	load_idt(&ctxt->idt);
	203	#else
	204	/* CONFIG_X86_64 */
9d1c6e7c GOC	205	load_gdt((const struct desc_ptr *)&ctxt->gdt_limit);
9d1c6e7c GOC	206	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
3134d04b	207	#endif
8d783b3e	208
1da177e4 LT	209	/*
	210	* segment registers
	211	*/
3134d04b SL	212	#ifdef CONFIG_X86_32
	213	loadsegment(es, ctxt->es);
	214	loadsegment(fs, ctxt->fs);
	215	loadsegment(gs, ctxt->gs);
	216	loadsegment(ss, ctxt->ss);
	217
	218	/*
	219	* sysenter MSRs
	220	*/
	221	if (boot_cpu_has(X86_FEATURE_SEP))
	222	enable_sep_cpu();
	223	#else
	224	/* CONFIG_X86_64 */
1da177e4 LT	225	asm volatile ("movw %0, %%ds" :: "r" (ctxt->ds));
	226	asm volatile ("movw %0, %%es" :: "r" (ctxt->es));
	227	asm volatile ("movw %0, %%fs" :: "r" (ctxt->fs));
	228	load_gs_index(ctxt->gs);
	229	asm volatile ("movw %0, %%ss" :: "r" (ctxt->ss));
	230
	231	wrmsrl(MSR_FS_BASE, ctxt->fs_base);
	232	wrmsrl(MSR_GS_BASE, ctxt->gs_base);
	233	wrmsrl(MSR_KERNEL_GS_BASE, ctxt->gs_kernel_base);
3134d04b	234	#endif
1da177e4	235
83b8e28b SS	236	/*
	237	* restore XCR0 for xsave capable cpu's.
	238	*/
	239	if (cpu_has_xsave)
	240	xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
	241
1da177e4 LT	242	fix_processor_context();
	243
	244	do_fpu_end();
3b520b23	245	mtrr_ap_init();
3134d04b SL	246
	247	#ifdef CONFIG_X86_32
	248	mcheck_init(&boot_cpu_data);
	249	#endif
1da177e4 LT	250	}
1da177e4 LT	251
3134d04b	252	/* Needed by apm.c */
1da177e4 LT	253	void restore_processor_state(void)
	254	{
	255	__restore_processor_state(&saved_context);
	256	}
3134d04b SL	257	#ifdef CONFIG_X86_32
	258	EXPORT_SYMBOL(restore_processor_state);
	259	#endif