Commit | Line | Data |
---|---|---|
eee3af4a MM |
1 | /* |
2 | * Debug Store support | |
3 | * | |
4 | * This provides a low-level interface to the hardware's Debug Store | |
93fa7636 | 5 | * feature that is used for branch trace store (BTS) and |
eee3af4a MM |
6 | * precise-event based sampling (PEBS). |
7 | * | |
93fa7636 MM |
8 | * It manages: |
9 | * - per-thread and per-cpu allocation of BTS and PEBS | |
10 | * - buffer memory allocation (optional) | |
11 | * - buffer overflow handling | |
12 | * - buffer access | |
eee3af4a | 13 | * |
93fa7636 MM |
14 | * It assumes: |
15 | * - get_task_struct on all parameter tasks | |
16 | * - current is allowed to trace parameter tasks | |
eee3af4a | 17 | * |
eee3af4a | 18 | * |
93fa7636 MM |
19 | * Copyright (C) 2007-2008 Intel Corporation. |
20 | * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008 | |
eee3af4a MM |
21 | */ |
22 | ||
93fa7636 MM |
23 | |
24 | #ifdef CONFIG_X86_DS | |
25 | ||
eee3af4a MM |
26 | #include <asm/ds.h> |
27 | ||
28 | #include <linux/errno.h> | |
29 | #include <linux/string.h> | |
30 | #include <linux/slab.h> | |
93fa7636 | 31 | #include <linux/sched.h> |
3c933904 | 32 | #include <linux/mm.h> |
93fa7636 MM |
33 | |
34 | ||
35 | /* | |
36 | * The configuration for a particular DS hardware implementation. | |
37 | */ | |
38 | struct ds_configuration { | |
39 | /* the size of the DS structure in bytes */ | |
40 | unsigned char sizeof_ds; | |
41 | /* the size of one pointer-typed field in the DS structure in bytes; | |
42 | this covers the first 8 fields related to buffer management. */ | |
43 | unsigned char sizeof_field; | |
44 | /* the size of a BTS/PEBS record in bytes */ | |
45 | unsigned char sizeof_rec[2]; | |
46 | }; | |
47 | static struct ds_configuration ds_cfg; | |
eee3af4a MM |
48 | |
49 | ||
50 | /* | |
51 | * Debug Store (DS) save area configuration (see Intel64 and IA32 | |
52 | * Architectures Software Developer's Manual, section 18.5) | |
53 | * | |
54 | * The DS configuration consists of the following fields; different | |
55 | * architetures vary in the size of those fields. | |
56 | * - double-word aligned base linear address of the BTS buffer | |
57 | * - write pointer into the BTS buffer | |
58 | * - end linear address of the BTS buffer (one byte beyond the end of | |
59 | * the buffer) | |
60 | * - interrupt pointer into BTS buffer | |
61 | * (interrupt occurs when write pointer passes interrupt pointer) | |
62 | * - double-word aligned base linear address of the PEBS buffer | |
63 | * - write pointer into the PEBS buffer | |
64 | * - end linear address of the PEBS buffer (one byte beyond the end of | |
65 | * the buffer) | |
66 | * - interrupt pointer into PEBS buffer | |
67 | * (interrupt occurs when write pointer passes interrupt pointer) | |
68 | * - value to which counter is reset following counter overflow | |
69 | * | |
93fa7636 MM |
70 | * Later architectures use 64bit pointers throughout, whereas earlier |
71 | * architectures use 32bit pointers in 32bit mode. | |
eee3af4a | 72 | * |
eee3af4a | 73 | * |
93fa7636 MM |
74 | * We compute the base address for the first 8 fields based on: |
75 | * - the field size stored in the DS configuration | |
76 | * - the relative field position | |
77 | * - an offset giving the start of the respective region | |
eee3af4a | 78 | * |
93fa7636 MM |
79 | * This offset is further used to index various arrays holding |
80 | * information for BTS and PEBS at the respective index. | |
eee3af4a | 81 | * |
93fa7636 MM |
82 | * On later 32bit processors, we only access the lower 32bit of the |
83 | * 64bit pointer fields. The upper halves will be zeroed out. | |
eee3af4a MM |
84 | */ |
85 | ||
93fa7636 MM |
86 | enum ds_field { |
87 | ds_buffer_base = 0, | |
88 | ds_index, | |
89 | ds_absolute_maximum, | |
90 | ds_interrupt_threshold, | |
91 | }; | |
eee3af4a | 92 | |
93fa7636 MM |
93 | enum ds_qualifier { |
94 | ds_bts = 0, | |
95 | ds_pebs | |
eee3af4a MM |
96 | }; |
97 | ||
93fa7636 MM |
98 | static inline unsigned long ds_get(const unsigned char *base, |
99 | enum ds_qualifier qual, enum ds_field field) | |
100 | { | |
101 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | |
102 | return *(unsigned long *)base; | |
103 | } | |
104 | ||
105 | static inline void ds_set(unsigned char *base, enum ds_qualifier qual, | |
106 | enum ds_field field, unsigned long value) | |
107 | { | |
108 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | |
109 | (*(unsigned long *)base) = value; | |
110 | } | |
111 | ||
112 | ||
eee3af4a | 113 | /* |
93fa7636 MM |
114 | * Locking is done only for allocating BTS or PEBS resources and for |
115 | * guarding context and buffer memory allocation. | |
116 | * | |
117 | * Most functions require the current task to own the ds context part | |
118 | * they are going to access. All the locking is done when validating | |
119 | * access to the context. | |
eee3af4a | 120 | */ |
93fa7636 | 121 | static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock); |
eee3af4a MM |
122 | |
123 | /* | |
93fa7636 MM |
124 | * Validate that the current task is allowed to access the BTS/PEBS |
125 | * buffer of the parameter task. | |
126 | * | |
127 | * Returns 0, if access is granted; -Eerrno, otherwise. | |
eee3af4a | 128 | */ |
93fa7636 MM |
129 | static inline int ds_validate_access(struct ds_context *context, |
130 | enum ds_qualifier qual) | |
131 | { | |
132 | if (!context) | |
133 | return -EPERM; | |
134 | ||
135 | if (context->owner[qual] == current) | |
136 | return 0; | |
137 | ||
138 | return -EPERM; | |
139 | } | |
140 | ||
eee3af4a MM |
141 | |
142 | /* | |
93fa7636 MM |
143 | * We either support (system-wide) per-cpu or per-thread allocation. |
144 | * We distinguish the two based on the task_struct pointer, where a | |
145 | * NULL pointer indicates per-cpu allocation for the current cpu. | |
146 | * | |
147 | * Allocations are use-counted. As soon as resources are allocated, | |
148 | * further allocations must be of the same type (per-cpu or | |
149 | * per-thread). We model this by counting allocations (i.e. the number | |
150 | * of tracers of a certain type) for one type negatively: | |
151 | * =0 no tracers | |
152 | * >0 number of per-thread tracers | |
153 | * <0 number of per-cpu tracers | |
154 | * | |
155 | * The below functions to get and put tracers and to check the | |
156 | * allocation type require the ds_lock to be held by the caller. | |
157 | * | |
158 | * Tracers essentially gives the number of ds contexts for a certain | |
159 | * type of allocation. | |
eee3af4a | 160 | */ |
93fa7636 MM |
161 | static long tracers; |
162 | ||
163 | static inline void get_tracer(struct task_struct *task) | |
eee3af4a | 164 | { |
93fa7636 | 165 | tracers += (task ? 1 : -1); |
eee3af4a | 166 | } |
93fa7636 MM |
167 | |
168 | static inline void put_tracer(struct task_struct *task) | |
eee3af4a | 169 | { |
93fa7636 | 170 | tracers -= (task ? 1 : -1); |
eee3af4a | 171 | } |
93fa7636 MM |
172 | |
173 | static inline int check_tracer(struct task_struct *task) | |
eee3af4a | 174 | { |
93fa7636 | 175 | return (task ? (tracers >= 0) : (tracers <= 0)); |
eee3af4a | 176 | } |
93fa7636 MM |
177 | |
178 | ||
179 | /* | |
180 | * The DS context is either attached to a thread or to a cpu: | |
181 | * - in the former case, the thread_struct contains a pointer to the | |
182 | * attached context. | |
183 | * - in the latter case, we use a static array of per-cpu context | |
184 | * pointers. | |
185 | * | |
186 | * Contexts are use-counted. They are allocated on first access and | |
187 | * deallocated when the last user puts the context. | |
188 | * | |
189 | * We distinguish between an allocating and a non-allocating get of a | |
190 | * context: | |
191 | * - the allocating get is used for requesting BTS/PEBS resources. It | |
192 | * requires the caller to hold the global ds_lock. | |
193 | * - the non-allocating get is used for all other cases. A | |
194 | * non-existing context indicates an error. It acquires and releases | |
195 | * the ds_lock itself for obtaining the context. | |
196 | * | |
197 | * A context and its DS configuration are allocated and deallocated | |
198 | * together. A context always has a DS configuration of the | |
199 | * appropriate size. | |
200 | */ | |
201 | static DEFINE_PER_CPU(struct ds_context *, system_context); | |
202 | ||
203 | #define this_system_context per_cpu(system_context, smp_processor_id()) | |
204 | ||
205 | /* | |
206 | * Returns the pointer to the parameter task's context or to the | |
207 | * system-wide context, if task is NULL. | |
208 | * | |
209 | * Increases the use count of the returned context, if not NULL. | |
210 | */ | |
211 | static inline struct ds_context *ds_get_context(struct task_struct *task) | |
eee3af4a | 212 | { |
93fa7636 MM |
213 | struct ds_context *context; |
214 | ||
215 | spin_lock(&ds_lock); | |
216 | ||
217 | context = (task ? task->thread.ds_ctx : this_system_context); | |
218 | if (context) | |
219 | context->count++; | |
220 | ||
221 | spin_unlock(&ds_lock); | |
222 | ||
223 | return context; | |
eee3af4a | 224 | } |
93fa7636 MM |
225 | |
226 | /* | |
227 | * Same as ds_get_context, but allocates the context and it's DS | |
228 | * structure, if necessary; returns NULL; if out of memory. | |
229 | * | |
230 | * pre: requires ds_lock to be held | |
231 | */ | |
232 | static inline struct ds_context *ds_alloc_context(struct task_struct *task) | |
eee3af4a | 233 | { |
93fa7636 MM |
234 | struct ds_context **p_context = |
235 | (task ? &task->thread.ds_ctx : &this_system_context); | |
236 | struct ds_context *context = *p_context; | |
237 | ||
238 | if (!context) { | |
239 | context = kzalloc(sizeof(*context), GFP_KERNEL); | |
240 | ||
241 | if (!context) | |
573da422 | 242 | return NULL; |
93fa7636 MM |
243 | |
244 | context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL); | |
245 | if (!context->ds) { | |
246 | kfree(context); | |
573da422 | 247 | return NULL; |
93fa7636 MM |
248 | } |
249 | ||
250 | *p_context = context; | |
251 | ||
252 | context->this = p_context; | |
253 | context->task = task; | |
254 | ||
255 | if (task) | |
256 | set_tsk_thread_flag(task, TIF_DS_AREA_MSR); | |
257 | ||
258 | if (!task || (task == current)) | |
259 | wrmsr(MSR_IA32_DS_AREA, (unsigned long)context->ds, 0); | |
260 | ||
261 | get_tracer(task); | |
262 | } | |
263 | ||
264 | context->count++; | |
265 | ||
266 | return context; | |
eee3af4a | 267 | } |
93fa7636 MM |
268 | |
269 | /* | |
270 | * Decreases the use count of the parameter context, if not NULL. | |
271 | * Deallocates the context, if the use count reaches zero. | |
272 | */ | |
273 | static inline void ds_put_context(struct ds_context *context) | |
eee3af4a | 274 | { |
93fa7636 MM |
275 | if (!context) |
276 | return; | |
277 | ||
278 | spin_lock(&ds_lock); | |
279 | ||
280 | if (--context->count) | |
281 | goto out; | |
282 | ||
573da422 | 283 | *(context->this) = NULL; |
93fa7636 MM |
284 | |
285 | if (context->task) | |
286 | clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR); | |
287 | ||
288 | if (!context->task || (context->task == current)) | |
289 | wrmsrl(MSR_IA32_DS_AREA, 0); | |
290 | ||
291 | put_tracer(context->task); | |
292 | ||
293 | /* free any leftover buffers from tracers that did not | |
294 | * deallocate them properly. */ | |
295 | kfree(context->buffer[ds_bts]); | |
296 | kfree(context->buffer[ds_pebs]); | |
297 | kfree(context->ds); | |
298 | kfree(context); | |
299 | out: | |
300 | spin_unlock(&ds_lock); | |
eee3af4a | 301 | } |
93fa7636 MM |
302 | |
303 | ||
304 | /* | |
305 | * Handle a buffer overflow | |
306 | * | |
307 | * task: the task whose buffers are overflowing; | |
308 | * NULL for a buffer overflow on the current cpu | |
309 | * context: the ds context | |
310 | * qual: the buffer type | |
311 | */ | |
312 | static void ds_overflow(struct task_struct *task, struct ds_context *context, | |
313 | enum ds_qualifier qual) | |
eee3af4a | 314 | { |
93fa7636 MM |
315 | if (!context) |
316 | return; | |
317 | ||
318 | if (context->callback[qual]) | |
319 | (*context->callback[qual])(task); | |
320 | ||
321 | /* todo: do some more overflow handling */ | |
eee3af4a | 322 | } |
93fa7636 MM |
323 | |
324 | ||
325 | /* | |
326 | * Allocate a non-pageable buffer of the parameter size. | |
327 | * Checks the memory and the locked memory rlimit. | |
328 | * | |
329 | * Returns the buffer, if successful; | |
330 | * NULL, if out of memory or rlimit exceeded. | |
331 | * | |
332 | * size: the requested buffer size in bytes | |
333 | * pages (out): if not NULL, contains the number of pages reserved | |
334 | */ | |
335 | static inline void *ds_allocate_buffer(size_t size, unsigned int *pages) | |
eee3af4a | 336 | { |
93fa7636 MM |
337 | unsigned long rlim, vm, pgsz; |
338 | void *buffer; | |
339 | ||
340 | pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT; | |
341 | ||
342 | rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT; | |
343 | vm = current->mm->total_vm + pgsz; | |
344 | if (rlim < vm) | |
573da422 | 345 | return NULL; |
93fa7636 MM |
346 | |
347 | rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; | |
348 | vm = current->mm->locked_vm + pgsz; | |
349 | if (rlim < vm) | |
573da422 | 350 | return NULL; |
93fa7636 MM |
351 | |
352 | buffer = kzalloc(size, GFP_KERNEL); | |
353 | if (!buffer) | |
573da422 | 354 | return NULL; |
93fa7636 MM |
355 | |
356 | current->mm->total_vm += pgsz; | |
357 | current->mm->locked_vm += pgsz; | |
358 | ||
359 | if (pages) | |
360 | *pages = pgsz; | |
361 | ||
362 | return buffer; | |
eee3af4a | 363 | } |
93fa7636 MM |
364 | |
365 | static int ds_request(struct task_struct *task, void *base, size_t size, | |
366 | ds_ovfl_callback_t ovfl, enum ds_qualifier qual) | |
eee3af4a | 367 | { |
93fa7636 MM |
368 | struct ds_context *context; |
369 | unsigned long buffer, adj; | |
370 | const unsigned long alignment = (1 << 3); | |
371 | int error = 0; | |
372 | ||
373 | if (!ds_cfg.sizeof_ds) | |
374 | return -EOPNOTSUPP; | |
375 | ||
376 | /* we require some space to do alignment adjustments below */ | |
377 | if (size < (alignment + ds_cfg.sizeof_rec[qual])) | |
378 | return -EINVAL; | |
379 | ||
380 | /* buffer overflow notification is not yet implemented */ | |
381 | if (ovfl) | |
382 | return -EOPNOTSUPP; | |
383 | ||
384 | ||
385 | spin_lock(&ds_lock); | |
386 | ||
387 | if (!check_tracer(task)) | |
388 | return -EPERM; | |
389 | ||
390 | error = -ENOMEM; | |
391 | context = ds_alloc_context(task); | |
392 | if (!context) | |
393 | goto out_unlock; | |
394 | ||
395 | error = -EALREADY; | |
396 | if (context->owner[qual] == current) | |
397 | goto out_unlock; | |
398 | error = -EPERM; | |
573da422 | 399 | if (context->owner[qual] != NULL) |
93fa7636 MM |
400 | goto out_unlock; |
401 | context->owner[qual] = current; | |
402 | ||
403 | spin_unlock(&ds_lock); | |
404 | ||
405 | ||
406 | error = -ENOMEM; | |
407 | if (!base) { | |
408 | base = ds_allocate_buffer(size, &context->pages[qual]); | |
409 | if (!base) | |
410 | goto out_release; | |
411 | ||
412 | context->buffer[qual] = base; | |
413 | } | |
414 | error = 0; | |
415 | ||
416 | context->callback[qual] = ovfl; | |
417 | ||
418 | /* adjust the buffer address and size to meet alignment | |
419 | * constraints: | |
420 | * - buffer is double-word aligned | |
421 | * - size is multiple of record size | |
422 | * | |
423 | * We checked the size at the very beginning; we have enough | |
424 | * space to do the adjustment. | |
425 | */ | |
426 | buffer = (unsigned long)base; | |
427 | ||
428 | adj = ALIGN(buffer, alignment) - buffer; | |
429 | buffer += adj; | |
430 | size -= adj; | |
431 | ||
432 | size /= ds_cfg.sizeof_rec[qual]; | |
433 | size *= ds_cfg.sizeof_rec[qual]; | |
434 | ||
435 | ds_set(context->ds, qual, ds_buffer_base, buffer); | |
436 | ds_set(context->ds, qual, ds_index, buffer); | |
437 | ds_set(context->ds, qual, ds_absolute_maximum, buffer + size); | |
438 | ||
439 | if (ovfl) { | |
440 | /* todo: select a suitable interrupt threshold */ | |
441 | } else | |
442 | ds_set(context->ds, qual, | |
443 | ds_interrupt_threshold, buffer + size + 1); | |
444 | ||
445 | /* we keep the context until ds_release */ | |
446 | return error; | |
447 | ||
448 | out_release: | |
573da422 | 449 | context->owner[qual] = NULL; |
93fa7636 MM |
450 | ds_put_context(context); |
451 | return error; | |
452 | ||
453 | out_unlock: | |
454 | spin_unlock(&ds_lock); | |
455 | ds_put_context(context); | |
456 | return error; | |
eee3af4a | 457 | } |
93fa7636 MM |
458 | |
459 | int ds_request_bts(struct task_struct *task, void *base, size_t size, | |
460 | ds_ovfl_callback_t ovfl) | |
eee3af4a | 461 | { |
93fa7636 | 462 | return ds_request(task, base, size, ovfl, ds_bts); |
eee3af4a | 463 | } |
93fa7636 MM |
464 | |
465 | int ds_request_pebs(struct task_struct *task, void *base, size_t size, | |
466 | ds_ovfl_callback_t ovfl) | |
eee3af4a | 467 | { |
93fa7636 | 468 | return ds_request(task, base, size, ovfl, ds_pebs); |
eee3af4a | 469 | } |
93fa7636 MM |
470 | |
471 | static int ds_release(struct task_struct *task, enum ds_qualifier qual) | |
eee3af4a | 472 | { |
93fa7636 MM |
473 | struct ds_context *context; |
474 | int error; | |
475 | ||
476 | context = ds_get_context(task); | |
477 | error = ds_validate_access(context, qual); | |
478 | if (error < 0) | |
479 | goto out; | |
480 | ||
481 | kfree(context->buffer[qual]); | |
493cd912 | 482 | context->buffer[qual] = NULL; |
93fa7636 MM |
483 | |
484 | current->mm->total_vm -= context->pages[qual]; | |
485 | current->mm->locked_vm -= context->pages[qual]; | |
486 | context->pages[qual] = 0; | |
493cd912 | 487 | context->owner[qual] = NULL; |
93fa7636 MM |
488 | |
489 | /* | |
490 | * we put the context twice: | |
491 | * once for the ds_get_context | |
492 | * once for the corresponding ds_request | |
493 | */ | |
494 | ds_put_context(context); | |
495 | out: | |
496 | ds_put_context(context); | |
497 | return error; | |
eee3af4a | 498 | } |
93fa7636 MM |
499 | |
500 | int ds_release_bts(struct task_struct *task) | |
eee3af4a | 501 | { |
93fa7636 | 502 | return ds_release(task, ds_bts); |
eee3af4a | 503 | } |
93fa7636 MM |
504 | |
505 | int ds_release_pebs(struct task_struct *task) | |
eee3af4a | 506 | { |
93fa7636 | 507 | return ds_release(task, ds_pebs); |
eee3af4a | 508 | } |
93fa7636 MM |
509 | |
510 | static int ds_get_index(struct task_struct *task, size_t *pos, | |
511 | enum ds_qualifier qual) | |
eee3af4a | 512 | { |
93fa7636 MM |
513 | struct ds_context *context; |
514 | unsigned long base, index; | |
515 | int error; | |
516 | ||
517 | context = ds_get_context(task); | |
518 | error = ds_validate_access(context, qual); | |
519 | if (error < 0) | |
520 | goto out; | |
521 | ||
522 | base = ds_get(context->ds, qual, ds_buffer_base); | |
523 | index = ds_get(context->ds, qual, ds_index); | |
524 | ||
525 | error = ((index - base) / ds_cfg.sizeof_rec[qual]); | |
526 | if (pos) | |
527 | *pos = error; | |
528 | out: | |
529 | ds_put_context(context); | |
530 | return error; | |
eee3af4a | 531 | } |
93fa7636 MM |
532 | |
533 | int ds_get_bts_index(struct task_struct *task, size_t *pos) | |
eee3af4a | 534 | { |
93fa7636 | 535 | return ds_get_index(task, pos, ds_bts); |
eee3af4a MM |
536 | } |
537 | ||
93fa7636 MM |
538 | int ds_get_pebs_index(struct task_struct *task, size_t *pos) |
539 | { | |
540 | return ds_get_index(task, pos, ds_pebs); | |
541 | } | |
eee3af4a | 542 | |
93fa7636 MM |
543 | static int ds_get_end(struct task_struct *task, size_t *pos, |
544 | enum ds_qualifier qual) | |
eee3af4a | 545 | { |
93fa7636 MM |
546 | struct ds_context *context; |
547 | unsigned long base, end; | |
548 | int error; | |
549 | ||
550 | context = ds_get_context(task); | |
551 | error = ds_validate_access(context, qual); | |
552 | if (error < 0) | |
553 | goto out; | |
554 | ||
555 | base = ds_get(context->ds, qual, ds_buffer_base); | |
556 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
557 | ||
558 | error = ((end - base) / ds_cfg.sizeof_rec[qual]); | |
559 | if (pos) | |
560 | *pos = error; | |
561 | out: | |
562 | ds_put_context(context); | |
563 | return error; | |
564 | } | |
eee3af4a | 565 | |
93fa7636 MM |
566 | int ds_get_bts_end(struct task_struct *task, size_t *pos) |
567 | { | |
568 | return ds_get_end(task, pos, ds_bts); | |
569 | } | |
eee3af4a | 570 | |
93fa7636 MM |
571 | int ds_get_pebs_end(struct task_struct *task, size_t *pos) |
572 | { | |
573 | return ds_get_end(task, pos, ds_pebs); | |
574 | } | |
eee3af4a | 575 | |
93fa7636 MM |
576 | static int ds_access(struct task_struct *task, size_t index, |
577 | const void **record, enum ds_qualifier qual) | |
578 | { | |
579 | struct ds_context *context; | |
580 | unsigned long base, idx; | |
581 | int error; | |
eee3af4a | 582 | |
93fa7636 | 583 | if (!record) |
eee3af4a MM |
584 | return -EINVAL; |
585 | ||
93fa7636 MM |
586 | context = ds_get_context(task); |
587 | error = ds_validate_access(context, qual); | |
588 | if (error < 0) | |
589 | goto out; | |
eee3af4a | 590 | |
93fa7636 MM |
591 | base = ds_get(context->ds, qual, ds_buffer_base); |
592 | idx = base + (index * ds_cfg.sizeof_rec[qual]); | |
eee3af4a | 593 | |
93fa7636 MM |
594 | error = -EINVAL; |
595 | if (idx > ds_get(context->ds, qual, ds_absolute_maximum)) | |
596 | goto out; | |
eee3af4a | 597 | |
93fa7636 MM |
598 | *record = (const void *)idx; |
599 | error = ds_cfg.sizeof_rec[qual]; | |
600 | out: | |
601 | ds_put_context(context); | |
602 | return error; | |
eee3af4a MM |
603 | } |
604 | ||
93fa7636 | 605 | int ds_access_bts(struct task_struct *task, size_t index, const void **record) |
eee3af4a | 606 | { |
93fa7636 | 607 | return ds_access(task, index, record, ds_bts); |
eee3af4a MM |
608 | } |
609 | ||
93fa7636 | 610 | int ds_access_pebs(struct task_struct *task, size_t index, const void **record) |
eee3af4a | 611 | { |
93fa7636 | 612 | return ds_access(task, index, record, ds_pebs); |
a95d67f8 MM |
613 | } |
614 | ||
93fa7636 MM |
615 | static int ds_write(struct task_struct *task, const void *record, size_t size, |
616 | enum ds_qualifier qual, int force) | |
a95d67f8 | 617 | { |
93fa7636 MM |
618 | struct ds_context *context; |
619 | int error; | |
eee3af4a | 620 | |
93fa7636 MM |
621 | if (!record) |
622 | return -EINVAL; | |
eee3af4a | 623 | |
93fa7636 MM |
624 | error = -EPERM; |
625 | context = ds_get_context(task); | |
626 | if (!context) | |
627 | goto out; | |
eee3af4a | 628 | |
93fa7636 MM |
629 | if (!force) { |
630 | error = ds_validate_access(context, qual); | |
631 | if (error < 0) | |
632 | goto out; | |
633 | } | |
eee3af4a | 634 | |
93fa7636 MM |
635 | error = 0; |
636 | while (size) { | |
637 | unsigned long base, index, end, write_end, int_th; | |
638 | unsigned long write_size, adj_write_size; | |
639 | ||
640 | /* | |
641 | * write as much as possible without producing an | |
642 | * overflow interrupt. | |
643 | * | |
644 | * interrupt_threshold must either be | |
645 | * - bigger than absolute_maximum or | |
646 | * - point to a record between buffer_base and absolute_maximum | |
647 | * | |
648 | * index points to a valid record. | |
649 | */ | |
650 | base = ds_get(context->ds, qual, ds_buffer_base); | |
651 | index = ds_get(context->ds, qual, ds_index); | |
652 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
653 | int_th = ds_get(context->ds, qual, ds_interrupt_threshold); | |
654 | ||
655 | write_end = min(end, int_th); | |
656 | ||
657 | /* if we are already beyond the interrupt threshold, | |
658 | * we fill the entire buffer */ | |
659 | if (write_end <= index) | |
660 | write_end = end; | |
661 | ||
662 | if (write_end <= index) | |
663 | goto out; | |
664 | ||
665 | write_size = min((unsigned long) size, write_end - index); | |
666 | memcpy((void *)index, record, write_size); | |
667 | ||
668 | record = (const char *)record + write_size; | |
669 | size -= write_size; | |
670 | error += write_size; | |
671 | ||
672 | adj_write_size = write_size / ds_cfg.sizeof_rec[qual]; | |
673 | adj_write_size *= ds_cfg.sizeof_rec[qual]; | |
674 | ||
675 | /* zero out trailing bytes */ | |
676 | memset((char *)index + write_size, 0, | |
677 | adj_write_size - write_size); | |
678 | index += adj_write_size; | |
679 | ||
680 | if (index >= end) | |
681 | index = base; | |
682 | ds_set(context->ds, qual, ds_index, index); | |
683 | ||
684 | if (index >= int_th) | |
685 | ds_overflow(task, context, qual); | |
686 | } | |
eee3af4a | 687 | |
93fa7636 MM |
688 | out: |
689 | ds_put_context(context); | |
690 | return error; | |
eee3af4a MM |
691 | } |
692 | ||
93fa7636 | 693 | int ds_write_bts(struct task_struct *task, const void *record, size_t size) |
a95d67f8 | 694 | { |
93fa7636 | 695 | return ds_write(task, record, size, ds_bts, /* force = */ 0); |
a95d67f8 MM |
696 | } |
697 | ||
93fa7636 | 698 | int ds_write_pebs(struct task_struct *task, const void *record, size_t size) |
a95d67f8 | 699 | { |
93fa7636 | 700 | return ds_write(task, record, size, ds_pebs, /* force = */ 0); |
a95d67f8 MM |
701 | } |
702 | ||
93fa7636 MM |
703 | int ds_unchecked_write_bts(struct task_struct *task, |
704 | const void *record, size_t size) | |
a95d67f8 | 705 | { |
93fa7636 | 706 | return ds_write(task, record, size, ds_bts, /* force = */ 1); |
a95d67f8 MM |
707 | } |
708 | ||
93fa7636 MM |
709 | int ds_unchecked_write_pebs(struct task_struct *task, |
710 | const void *record, size_t size) | |
eee3af4a | 711 | { |
93fa7636 MM |
712 | return ds_write(task, record, size, ds_pebs, /* force = */ 1); |
713 | } | |
eee3af4a | 714 | |
93fa7636 MM |
715 | static int ds_reset_or_clear(struct task_struct *task, |
716 | enum ds_qualifier qual, int clear) | |
717 | { | |
718 | struct ds_context *context; | |
719 | unsigned long base, end; | |
720 | int error; | |
eee3af4a | 721 | |
93fa7636 MM |
722 | context = ds_get_context(task); |
723 | error = ds_validate_access(context, qual); | |
724 | if (error < 0) | |
725 | goto out; | |
eee3af4a | 726 | |
93fa7636 MM |
727 | base = ds_get(context->ds, qual, ds_buffer_base); |
728 | end = ds_get(context->ds, qual, ds_absolute_maximum); | |
eee3af4a | 729 | |
93fa7636 MM |
730 | if (clear) |
731 | memset((void *)base, 0, end - base); | |
eee3af4a | 732 | |
93fa7636 | 733 | ds_set(context->ds, qual, ds_index, base); |
eee3af4a | 734 | |
93fa7636 MM |
735 | error = 0; |
736 | out: | |
737 | ds_put_context(context); | |
738 | return error; | |
eee3af4a MM |
739 | } |
740 | ||
93fa7636 | 741 | int ds_reset_bts(struct task_struct *task) |
eee3af4a | 742 | { |
93fa7636 MM |
743 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 0); |
744 | } | |
eee3af4a | 745 | |
93fa7636 MM |
746 | int ds_reset_pebs(struct task_struct *task) |
747 | { | |
748 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0); | |
749 | } | |
eee3af4a | 750 | |
93fa7636 MM |
751 | int ds_clear_bts(struct task_struct *task) |
752 | { | |
753 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 1); | |
754 | } | |
eee3af4a | 755 | |
93fa7636 MM |
756 | int ds_clear_pebs(struct task_struct *task) |
757 | { | |
758 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1); | |
759 | } | |
eee3af4a | 760 | |
93fa7636 MM |
761 | int ds_get_pebs_reset(struct task_struct *task, u64 *value) |
762 | { | |
763 | struct ds_context *context; | |
764 | int error; | |
eee3af4a | 765 | |
93fa7636 | 766 | if (!value) |
eee3af4a | 767 | return -EINVAL; |
eee3af4a | 768 | |
93fa7636 MM |
769 | context = ds_get_context(task); |
770 | error = ds_validate_access(context, ds_pebs); | |
771 | if (error < 0) | |
772 | goto out; | |
eee3af4a | 773 | |
93fa7636 MM |
774 | *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)); |
775 | ||
776 | error = 0; | |
777 | out: | |
778 | ds_put_context(context); | |
779 | return error; | |
eee3af4a MM |
780 | } |
781 | ||
93fa7636 | 782 | int ds_set_pebs_reset(struct task_struct *task, u64 value) |
eee3af4a | 783 | { |
93fa7636 MM |
784 | struct ds_context *context; |
785 | int error; | |
eee3af4a | 786 | |
93fa7636 MM |
787 | context = ds_get_context(task); |
788 | error = ds_validate_access(context, ds_pebs); | |
789 | if (error < 0) | |
790 | goto out; | |
eee3af4a | 791 | |
93fa7636 MM |
792 | *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value; |
793 | ||
794 | error = 0; | |
795 | out: | |
796 | ds_put_context(context); | |
797 | return error; | |
798 | } | |
799 | ||
800 | static const struct ds_configuration ds_cfg_var = { | |
801 | .sizeof_ds = sizeof(long) * 12, | |
802 | .sizeof_field = sizeof(long), | |
803 | .sizeof_rec[ds_bts] = sizeof(long) * 3, | |
804 | .sizeof_rec[ds_pebs] = sizeof(long) * 10 | |
eee3af4a | 805 | }; |
93fa7636 MM |
806 | static const struct ds_configuration ds_cfg_64 = { |
807 | .sizeof_ds = 8 * 12, | |
808 | .sizeof_field = 8, | |
809 | .sizeof_rec[ds_bts] = 8 * 3, | |
810 | .sizeof_rec[ds_pebs] = 8 * 10 | |
eee3af4a MM |
811 | }; |
812 | ||
813 | static inline void | |
814 | ds_configure(const struct ds_configuration *cfg) | |
815 | { | |
816 | ds_cfg = *cfg; | |
817 | } | |
818 | ||
819 | void __cpuinit ds_init_intel(struct cpuinfo_x86 *c) | |
820 | { | |
821 | switch (c->x86) { | |
822 | case 0x6: | |
823 | switch (c->x86_model) { | |
eee3af4a MM |
824 | case 0xD: |
825 | case 0xE: /* Pentium M */ | |
93fa7636 | 826 | ds_configure(&ds_cfg_var); |
eee3af4a | 827 | break; |
eee3af4a | 828 | case 0xF: /* Core2 */ |
573da422 | 829 | case 0x1C: /* Atom */ |
93fa7636 | 830 | ds_configure(&ds_cfg_64); |
eee3af4a MM |
831 | break; |
832 | default: | |
833 | /* sorry, don't know about them */ | |
834 | break; | |
835 | } | |
836 | break; | |
837 | case 0xF: | |
838 | switch (c->x86_model) { | |
eee3af4a MM |
839 | case 0x0: |
840 | case 0x1: | |
841 | case 0x2: /* Netburst */ | |
93fa7636 | 842 | ds_configure(&ds_cfg_var); |
eee3af4a | 843 | break; |
eee3af4a MM |
844 | default: |
845 | /* sorry, don't know about them */ | |
846 | break; | |
847 | } | |
848 | break; | |
849 | default: | |
850 | /* sorry, don't know about them */ | |
851 | break; | |
852 | } | |
853 | } | |
93fa7636 MM |
854 | |
855 | void ds_free(struct ds_context *context) | |
856 | { | |
857 | /* This is called when the task owning the parameter context | |
858 | * is dying. There should not be any user of that context left | |
859 | * to disturb us, anymore. */ | |
860 | unsigned long leftovers = context->count; | |
861 | while (leftovers--) | |
862 | ds_put_context(context); | |
863 | } | |
864 | #endif /* CONFIG_X86_DS */ |