[deliverable/linux.git] / arch / i386 / kernel / alternative.c

#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/list.h>
#include <asm/alternative.h>
#include <asm/sections.h>

#define DEBUG 0
#if DEBUG
# define DPRINTK(fmt, args...) printk(fmt, args)
#else
# define DPRINTK(fmt, args...)
#endif

/* Use inline assembly to define this because the nops are defined
   as inline assembly strings in the include files and we cannot
   get them easily into strings. */
asm("\t.data\nintelnops: "
	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	GENERIC_NOP7 GENERIC_NOP8);
asm("\t.data\nk8nops: "
	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	K8_NOP7 K8_NOP8);
asm("\t.data\nk7nops: "
	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	K7_NOP7 K7_NOP8);

extern unsigned char intelnops[], k8nops[], k7nops[];
static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	NULL,
	intelnops,
	intelnops + 1,
	intelnops + 1 + 2,
	intelnops + 1 + 2 + 3,
	intelnops + 1 + 2 + 3 + 4,
	intelnops + 1 + 2 + 3 + 4 + 5,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	NULL,
	k8nops,
	k8nops + 1,
	k8nops + 1 + 2,
	k8nops + 1 + 2 + 3,
	k8nops + 1 + 2 + 3 + 4,
	k8nops + 1 + 2 + 3 + 4 + 5,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	NULL,
	k7nops,
	k7nops + 1,
	k7nops + 1 + 2,
	k7nops + 1 + 2 + 3,
	k7nops + 1 + 2 + 3 + 4,
	k7nops + 1 + 2 + 3 + 4 + 5,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
};
static struct nop {
	int cpuid;
	unsigned char **noptable;
} noptypes[] = {
	{ X86_FEATURE_K8, k8_nops },
	{ X86_FEATURE_K7, k7_nops },
	{ -1, NULL }
};


extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
extern u8 *__smp_locks[], *__smp_locks_end[];

extern u8 __smp_alt_begin[], __smp_alt_end[];


static unsigned char** find_nop_table(void)
{
	unsigned char **noptable = intel_nops;
	int i;

	for (i = 0; noptypes[i].cpuid >= 0; i++) {
		if (boot_cpu_has(noptypes[i].cpuid)) {
			noptable = noptypes[i].noptable;
			break;
		}
	}
	return noptable;
}

/* Replace instructions with better alternatives for this CPU type.
   This runs before SMP is initialized to avoid SMP problems with
   self modifying code. This implies that assymetric systems where
   APs have less capabilities than the boot processor are not handled.
   Tough. Make sure you disable such features by hand. */

void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
{
	unsigned char **noptable = find_nop_table();
	struct alt_instr *a;
	int diff, i, k;

	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		BUG_ON(a->replacementlen > a->instrlen);
		if (!boot_cpu_has(a->cpuid))
			continue;
		memcpy(a->instr, a->replacement, a->replacementlen);
		diff = a->instrlen - a->replacementlen;
		/* Pad the rest with nops */
		for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
			k = diff;
			if (k > ASM_NOP_MAX)
				k = ASM_NOP_MAX;
			memcpy(a->instr + i, noptable[k], k);
		}
	}
}

static void alternatives_smp_save(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;

	DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
	for (a = start; a < end; a++) {
		memcpy(a->replacement + a->replacementlen,
		       a->instr,
		       a->instrlen);
	}
}

static void alternatives_smp_apply(struct alt_instr *start, struct alt_instr *end)
{
	struct alt_instr *a;

	for (a = start; a < end; a++) {
		memcpy(a->instr,
		       a->replacement + a->replacementlen,
		       a->instrlen);
	}
}

static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		**ptr = 0xf0; /* lock prefix */
	};
}

static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
{
	unsigned char **noptable = find_nop_table();
	u8 **ptr;

	for (ptr = start; ptr < end; ptr++) {
		if (*ptr < text)
			continue;
		if (*ptr > text_end)
			continue;
		**ptr = noptable[1][0];
	};
}

struct smp_alt_module {
	/* what is this ??? */
	struct module	*mod;
	char		*name;

	/* ptrs to lock prefixes */
	u8		**locks;
	u8		**locks_end;

	/* .text segment, needed to avoid patching init code ;) */
	u8		*text;
	u8		*text_end;

	struct list_head next;
};
static LIST_HEAD(smp_alt_modules);
static DEFINE_SPINLOCK(smp_alt);

static int smp_alt_once = 0;
static int __init bootonly(char *str)
{
	smp_alt_once = 1;
	return 1;
}
__setup("smp-alt-boot", bootonly);

void alternatives_smp_module_add(struct module *mod, char *name,
				 void *locks, void *locks_end,
				 void *text,  void *text_end)
{
	struct smp_alt_module *smp;
	unsigned long flags;

	if (smp_alt_once) {
		if (boot_cpu_has(X86_FEATURE_UP))
			alternatives_smp_unlock(locks, locks_end,
						text, text_end);
		return;
	}

	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
	if (NULL == smp)
		return; /* we'll run the (safe but slow) SMP code then ... */

	smp->mod	= mod;
	smp->name	= name;
	smp->locks	= locks;
	smp->locks_end	= locks_end;
	smp->text	= text;
	smp->text_end	= text_end;
	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
		__FUNCTION__, smp->locks, smp->locks_end,
		smp->text, smp->text_end, smp->name);

	spin_lock_irqsave(&smp_alt, flags);
	list_add_tail(&smp->next, &smp_alt_modules);
	if (boot_cpu_has(X86_FEATURE_UP))
		alternatives_smp_unlock(smp->locks, smp->locks_end,
					smp->text, smp->text_end);
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_module_del(struct module *mod)
{
	struct smp_alt_module *item;
	unsigned long flags;

	if (smp_alt_once)
		return;

	spin_lock_irqsave(&smp_alt, flags);
	list_for_each_entry(item, &smp_alt_modules, next) {
		if (mod != item->mod)
			continue;
		list_del(&item->next);
		spin_unlock_irqrestore(&smp_alt, flags);
		DPRINTK("%s: %s\n", __FUNCTION__, item->name);
		kfree(item);
		return;
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void alternatives_smp_switch(int smp)
{
	struct smp_alt_module *mod;
	unsigned long flags;

	if (smp_alt_once)
		return;
	BUG_ON(!smp && (num_online_cpus() > 1));

	spin_lock_irqsave(&smp_alt, flags);
	if (smp) {
		printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
		clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		alternatives_smp_apply(__smp_alt_instructions,
				       __smp_alt_instructions_end);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_lock(mod->locks, mod->locks_end,
					      mod->text, mod->text_end);
	} else {
		printk(KERN_INFO "SMP alternatives: switching to UP code\n");
		set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
		set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
		apply_alternatives(__smp_alt_instructions,
				   __smp_alt_instructions_end);
		list_for_each_entry(mod, &smp_alt_modules, next)
			alternatives_smp_unlock(mod->locks, mod->locks_end,
						mod->text, mod->text_end);
	}
	spin_unlock_irqrestore(&smp_alt, flags);
}

void __init alternative_instructions(void)
{
	apply_alternatives(__alt_instructions, __alt_instructions_end);

	/* switch to patch-once-at-boottime-only mode and free the
	 * tables in case we know the number of CPUs will never ever
	 * change */
#ifdef CONFIG_HOTPLUG_CPU
	if (num_possible_cpus() < 2)
		smp_alt_once = 1;
#else
	smp_alt_once = 1;
#endif

	if (smp_alt_once) {
		if (1 == num_possible_cpus()) {
			printk(KERN_INFO "SMP alternatives: switching to UP code\n");
			set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
			set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
			apply_alternatives(__smp_alt_instructions,
					   __smp_alt_instructions_end);
			alternatives_smp_unlock(__smp_locks, __smp_locks_end,
						_text, _etext);
		}
		free_init_pages("SMP alternatives",
				(unsigned long)__smp_alt_begin,
				(unsigned long)__smp_alt_end);
	} else {
		alternatives_smp_save(__smp_alt_instructions,
				      __smp_alt_instructions_end);
		alternatives_smp_module_add(NULL, "core kernel",
					    __smp_locks, __smp_locks_end,
					    _text, _etext);
		alternatives_smp_switch(0);
	}
}
Commit	Line	Data
9a0b5817 GH	1	#include <linux/module.h>
	2	#include <linux/spinlock.h>
	3	#include <linux/list.h>
	4	#include <asm/alternative.h>
	5	#include <asm/sections.h>
	6
	7	#define DEBUG 0
	8	#if DEBUG
	9	# define DPRINTK(fmt, args...) printk(fmt, args)
	10	#else
	11	# define DPRINTK(fmt, args...)
	12	#endif
	13
	14	/* Use inline assembly to define this because the nops are defined
	15	as inline assembly strings in the include files and we cannot
	16	get them easily into strings. */
	17	asm("\t.data\nintelnops: "
	18	GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
	19	GENERIC_NOP7 GENERIC_NOP8);
	20	asm("\t.data\nk8nops: "
	21	K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
	22	K8_NOP7 K8_NOP8);
	23	asm("\t.data\nk7nops: "
	24	K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
	25	K7_NOP7 K7_NOP8);
	26
	27	extern unsigned char intelnops[], k8nops[], k7nops[];
	28	static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
	29	NULL,
	30	intelnops,
	31	intelnops + 1,
	32	intelnops + 1 + 2,
	33	intelnops + 1 + 2 + 3,
	34	intelnops + 1 + 2 + 3 + 4,
	35	intelnops + 1 + 2 + 3 + 4 + 5,
	36	intelnops + 1 + 2 + 3 + 4 + 5 + 6,
	37	intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	38	};
	39	static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
	40	NULL,
	41	k8nops,
	42	k8nops + 1,
	43	k8nops + 1 + 2,
	44	k8nops + 1 + 2 + 3,
	45	k8nops + 1 + 2 + 3 + 4,
	46	k8nops + 1 + 2 + 3 + 4 + 5,
	47	k8nops + 1 + 2 + 3 + 4 + 5 + 6,
	48	k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	49	};
	50	static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
	51	NULL,
	52	k7nops,
	53	k7nops + 1,
	54	k7nops + 1 + 2,
	55	k7nops + 1 + 2 + 3,
	56	k7nops + 1 + 2 + 3 + 4,
	57	k7nops + 1 + 2 + 3 + 4 + 5,
	58	k7nops + 1 + 2 + 3 + 4 + 5 + 6,
	59	k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
	60	};
	61	static struct nop {
	62	int cpuid;
	63	unsigned char **noptable;
	64	} noptypes[] = {
65	{ X86_FEATURE_K8, k8_nops },
66	{ X86_FEATURE_K7, k7_nops },
67	{ -1, NULL }
68	};
69
70
71	extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
72	extern struct alt_instr __smp_alt_instructions[], __smp_alt_instructions_end[];
73	extern u8 __smp_locks[], __smp_locks_end[];
74
75	extern u8 __smp_alt_begin[], __smp_alt_end[];
76
77
78	static unsigned char** find_nop_table(void)
79	{
80	unsigned char **noptable = intel_nops;
81	int i;
82
83	for (i = 0; noptypes[i].cpuid >= 0; i++) {
84	if (boot_cpu_has(noptypes[i].cpuid)) {
85	noptable = noptypes[i].noptable;
86	break;
87	}
88	}
89	return noptable;
90	}
91
92	/* Replace instructions with better alternatives for this CPU type.
93	This runs before SMP is initialized to avoid SMP problems with
94	self modifying code. This implies that assymetric systems where
95	APs have less capabilities than the boot processor are not handled.
96	Tough. Make sure you disable such features by hand. */
97
98	void apply_alternatives(struct alt_instr start, struct alt_instr end)
99	{
100	unsigned char **noptable = find_nop_table();
101	struct alt_instr *a;
102	int diff, i, k;
103
104	DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
105	for (a = start; a < end; a++) {
106	BUG_ON(a->replacementlen > a->instrlen);
107	if (!boot_cpu_has(a->cpuid))
108	continue;
109	memcpy(a->instr, a->replacement, a->replacementlen);
110	diff = a->instrlen - a->replacementlen;
111	/* Pad the rest with nops */
112	for (i = a->replacementlen; diff > 0; diff -= k, i += k) {
113	k = diff;
114	if (k > ASM_NOP_MAX)
115	k = ASM_NOP_MAX;
116	memcpy(a->instr + i, noptable[k], k);
117	}
118	}
119	}
120
121	static void alternatives_smp_save(struct alt_instr start, struct alt_instr end)
122	{
123	struct alt_instr *a;
124
125	DPRINTK("%s: alt table %p-%p\n", __FUNCTION__, start, end);
126	for (a = start; a < end; a++) {
127	memcpy(a->replacement + a->replacementlen,
128	a->instr,
129	a->instrlen);
130	}
131	}
132
133	static void alternatives_smp_apply(struct alt_instr start, struct alt_instr end)
134	{
135	struct alt_instr *a;
136
137	for (a = start; a < end; a++) {
138	memcpy(a->instr,
139	a->replacement + a->replacementlen,
140	a->instrlen);
141	}
142	}
143
144	static void alternatives_smp_lock(u8 start, u8 end, u8 text, u8 text_end)
145	{
146	u8 **ptr;
147
148	for (ptr = start; ptr < end; ptr++) {
149	if (*ptr < text)
150	continue;
151	if (*ptr > text_end)
152	continue;
153	*ptr = 0xf0; / lock prefix */
154	};
155	}
156
157	static void alternatives_smp_unlock(u8 start, u8 end, u8 text, u8 text_end)
158	{
159	unsigned char **noptable = find_nop_table();
160	u8 **ptr;
161
162	for (ptr = start; ptr < end; ptr++) {
163	if (*ptr < text)
164	continue;
165	if (*ptr > text_end)
166	continue;
167	**ptr = noptable[1][0];
168	};
169	}
170
171	struct smp_alt_module {
172	/* what is this ??? */
173	struct module *mod;
174	char *name;
175
176	/* ptrs to lock prefixes */
177	u8 **locks;
178	u8 **locks_end;
179
180	/* .text segment, needed to avoid patching init code ;) */
181	u8 *text;
182	u8 *text_end;
183
184	struct list_head next;
185	};
186	static LIST_HEAD(smp_alt_modules);
187	static DEFINE_SPINLOCK(smp_alt);
188
189	static int smp_alt_once = 0;
190	static int __init bootonly(char *str)
191	{
192	smp_alt_once = 1;
193	return 1;
194	}
195	__setup("smp-alt-boot", bootonly);
196
197	void alternatives_smp_module_add(struct module mod, char name,
198	void locks, void locks_end,
199	void text, void text_end)
200	{
201	struct smp_alt_module *smp;
202	unsigned long flags;
203
204	if (smp_alt_once) {
205	if (boot_cpu_has(X86_FEATURE_UP))
206	alternatives_smp_unlock(locks, locks_end,
207	text, text_end);
208	return;
209	}
210
211	smp = kzalloc(sizeof(*smp), GFP_KERNEL);
212	if (NULL == smp)
213	return; /* we'll run the (safe but slow) SMP code then ... */
214
215	smp->mod = mod;
216	smp->name = name;
217	smp->locks = locks;
218	smp->locks_end = locks_end;
219	smp->text = text;
220	smp->text_end = text_end;
221	DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
222	__FUNCTION__, smp->locks, smp->locks_end,
223	smp->text, smp->text_end, smp->name);
224
225	spin_lock_irqsave(&smp_alt, flags);
226	list_add_tail(&smp->next, &smp_alt_modules);
227	if (boot_cpu_has(X86_FEATURE_UP))
228	alternatives_smp_unlock(smp->locks, smp->locks_end,
229	smp->text, smp->text_end);
230	spin_unlock_irqrestore(&smp_alt, flags);
231	}
232
233	void alternatives_smp_module_del(struct module *mod)
234	{
235	struct smp_alt_module *item;
236	unsigned long flags;
237
238	if (smp_alt_once)
239	return;
240
241	spin_lock_irqsave(&smp_alt, flags);
242	list_for_each_entry(item, &smp_alt_modules, next) {
243	if (mod != item->mod)
244	continue;
245	list_del(&item->next);
246	spin_unlock_irqrestore(&smp_alt, flags);
247	DPRINTK("%s: %s\n", __FUNCTION__, item->name);
248	kfree(item);
249	return;
250	}
251	spin_unlock_irqrestore(&smp_alt, flags);
252	}
253
254	void alternatives_smp_switch(int smp)
255	{
256	struct smp_alt_module *mod;
257	unsigned long flags;
258
259	if (smp_alt_once)
260	return;
261	BUG_ON(!smp && (num_online_cpus() > 1));
262
263	spin_lock_irqsave(&smp_alt, flags);
264	if (smp) {
265	printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
266	clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
267	clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
268	alternatives_smp_apply(__smp_alt_instructions,
269	__smp_alt_instructions_end);
270	list_for_each_entry(mod, &smp_alt_modules, next)
271	alternatives_smp_lock(mod->locks, mod->locks_end,
272	mod->text, mod->text_end);
273	} else {
274	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
275	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
276	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
277	apply_alternatives(__smp_alt_instructions,
278	__smp_alt_instructions_end);
279	list_for_each_entry(mod, &smp_alt_modules, next)
280	alternatives_smp_unlock(mod->locks, mod->locks_end,
281	mod->text, mod->text_end);
282	}
283	spin_unlock_irqrestore(&smp_alt, flags);
284	}
285
286	void __init alternative_instructions(void)
287	{
288	apply_alternatives(__alt_instructions, __alt_instructions_end);
289
290	/* switch to patch-once-at-boottime-only mode and free the
291	* tables in case we know the number of CPUs will never ever
292	* change */
293	#ifdef CONFIG_HOTPLUG_CPU
294	if (num_possible_cpus() < 2)
295	smp_alt_once = 1;
296	#else
297	smp_alt_once = 1;
298	#endif
299
300	if (smp_alt_once) {
301	if (1 == num_possible_cpus()) {
302	printk(KERN_INFO "SMP alternatives: switching to UP code\n");
303	set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
304	set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
305	apply_alternatives(__smp_alt_instructions,
306	__smp_alt_instructions_end);
307	alternatives_smp_unlock(__smp_locks, __smp_locks_end,
308	_text, _etext);
309	}
310	free_init_pages("SMP alternatives",
311	(unsigned long)__smp_alt_begin,
312	(unsigned long)__smp_alt_end);
313	} else {
314	alternatives_smp_save(__smp_alt_instructions,
315	__smp_alt_instructions_end);
316	alternatives_smp_module_add(NULL, "core kernel",
317	__smp_locks, __smp_locks_end,
318	_text, _etext);
319	alternatives_smp_switch(0);
320	}
321	}