4 * COMEDI - Linux Control and Measurement Device Interface
5 * Copyright (C) 1997-2000 David A. Schleef <ds@schleef.org>
6 * Copyright (C) 2002 Frank Mori Hess <fmhess@users.sourceforge.net>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
19 #include <linux/vmalloc.h>
20 #include <linux/slab.h>
22 #include "comedidev.h"
23 #include "comedi_internal.h"
25 #ifdef PAGE_KERNEL_NOCACHE
26 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL_NOCACHE
28 #define COMEDI_PAGE_PROTECTION PAGE_KERNEL
31 static void comedi_buf_map_kref_release(struct kref
*kref
)
33 struct comedi_buf_map
*bm
=
34 container_of(kref
, struct comedi_buf_map
, refcount
);
35 struct comedi_buf_page
*buf
;
39 for (i
= 0; i
< bm
->n_pages
; i
++) {
40 buf
= &bm
->page_list
[i
];
41 clear_bit(PG_reserved
,
42 &(virt_to_page(buf
->virt_addr
)->flags
));
43 if (bm
->dma_dir
!= DMA_NONE
) {
45 dma_free_coherent(bm
->dma_hw_dev
,
51 free_page((unsigned long)buf
->virt_addr
);
56 if (bm
->dma_dir
!= DMA_NONE
)
57 put_device(bm
->dma_hw_dev
);
61 static void __comedi_buf_free(struct comedi_device
*dev
,
62 struct comedi_subdevice
*s
)
64 struct comedi_async
*async
= s
->async
;
65 struct comedi_buf_map
*bm
;
68 if (async
->prealloc_buf
) {
69 vunmap(async
->prealloc_buf
);
70 async
->prealloc_buf
= NULL
;
71 async
->prealloc_bufsz
= 0;
74 spin_lock_irqsave(&s
->spin_lock
, flags
);
76 async
->buf_map
= NULL
;
77 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
78 comedi_buf_map_put(bm
);
81 static void __comedi_buf_alloc(struct comedi_device
*dev
,
82 struct comedi_subdevice
*s
,
85 struct comedi_async
*async
= s
->async
;
86 struct page
**pages
= NULL
;
87 struct comedi_buf_map
*bm
;
88 struct comedi_buf_page
*buf
;
92 if (!IS_ENABLED(CONFIG_HAS_DMA
) && s
->async_dma_dir
!= DMA_NONE
) {
93 dev_err(dev
->class_dev
,
94 "dma buffer allocation not supported\n");
98 bm
= kzalloc(sizeof(*async
->buf_map
), GFP_KERNEL
);
102 kref_init(&bm
->refcount
);
103 spin_lock_irqsave(&s
->spin_lock
, flags
);
105 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
106 bm
->dma_dir
= s
->async_dma_dir
;
107 if (bm
->dma_dir
!= DMA_NONE
)
108 /* Need ref to hardware device to free buffer later. */
109 bm
->dma_hw_dev
= get_device(dev
->hw_dev
);
111 bm
->page_list
= vzalloc(sizeof(*buf
) * n_pages
);
113 pages
= vmalloc(sizeof(struct page
*) * n_pages
);
118 for (i
= 0; i
< n_pages
; i
++) {
119 buf
= &bm
->page_list
[i
];
120 if (bm
->dma_dir
!= DMA_NONE
)
121 #ifdef CONFIG_HAS_DMA
122 buf
->virt_addr
= dma_alloc_coherent(bm
->dma_hw_dev
,
131 buf
->virt_addr
= (void *)get_zeroed_page(GFP_KERNEL
);
135 set_bit(PG_reserved
, &(virt_to_page(buf
->virt_addr
)->flags
));
137 pages
[i
] = virt_to_page(buf
->virt_addr
);
139 spin_lock_irqsave(&s
->spin_lock
, flags
);
141 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
143 /* vmap the prealloc_buf if all the pages were allocated */
145 async
->prealloc_buf
= vmap(pages
, n_pages
, VM_MAP
,
146 COMEDI_PAGE_PROTECTION
);
151 void comedi_buf_map_get(struct comedi_buf_map
*bm
)
154 kref_get(&bm
->refcount
);
157 int comedi_buf_map_put(struct comedi_buf_map
*bm
)
160 return kref_put(&bm
->refcount
, comedi_buf_map_kref_release
);
164 /* returns s->async->buf_map and increments its kref refcount */
165 struct comedi_buf_map
*
166 comedi_buf_map_from_subdev_get(struct comedi_subdevice
*s
)
168 struct comedi_async
*async
= s
->async
;
169 struct comedi_buf_map
*bm
= NULL
;
175 spin_lock_irqsave(&s
->spin_lock
, flags
);
177 /* only want it if buffer pages allocated */
178 if (bm
&& bm
->n_pages
)
179 comedi_buf_map_get(bm
);
182 spin_unlock_irqrestore(&s
->spin_lock
, flags
);
187 bool comedi_buf_is_mmapped(struct comedi_subdevice
*s
)
189 struct comedi_buf_map
*bm
= s
->async
->buf_map
;
191 return bm
&& (atomic_read(&bm
->refcount
.refcount
) > 1);
194 int comedi_buf_alloc(struct comedi_device
*dev
, struct comedi_subdevice
*s
,
195 unsigned long new_size
)
197 struct comedi_async
*async
= s
->async
;
199 /* Round up new_size to multiple of PAGE_SIZE */
200 new_size
= (new_size
+ PAGE_SIZE
- 1) & PAGE_MASK
;
202 /* if no change is required, do nothing */
203 if (async
->prealloc_buf
&& async
->prealloc_bufsz
== new_size
)
206 /* deallocate old buffer */
207 __comedi_buf_free(dev
, s
);
209 /* allocate new buffer */
211 unsigned n_pages
= new_size
>> PAGE_SHIFT
;
213 __comedi_buf_alloc(dev
, s
, n_pages
);
215 if (!async
->prealloc_buf
) {
216 /* allocation failed */
217 __comedi_buf_free(dev
, s
);
221 async
->prealloc_bufsz
= new_size
;
226 void comedi_buf_reset(struct comedi_subdevice
*s
)
228 struct comedi_async
*async
= s
->async
;
230 async
->buf_write_alloc_count
= 0;
231 async
->buf_write_count
= 0;
232 async
->buf_read_alloc_count
= 0;
233 async
->buf_read_count
= 0;
235 async
->buf_write_ptr
= 0;
236 async
->buf_read_ptr
= 0;
239 async
->scans_done
= 0;
240 async
->scan_progress
= 0;
241 async
->munge_chan
= 0;
242 async
->munge_count
= 0;
243 async
->munge_ptr
= 0;
248 static unsigned int comedi_buf_write_n_unalloc(struct comedi_subdevice
*s
)
250 struct comedi_async
*async
= s
->async
;
251 unsigned int free_end
= async
->buf_read_count
+ async
->prealloc_bufsz
;
253 return free_end
- async
->buf_write_alloc_count
;
256 unsigned int comedi_buf_write_n_available(struct comedi_subdevice
*s
)
258 struct comedi_async
*async
= s
->async
;
259 unsigned int free_end
= async
->buf_read_count
+ async
->prealloc_bufsz
;
261 return free_end
- async
->buf_write_count
;
265 * comedi_buf_write_alloc() - Reserve buffer space for writing
266 * @s: COMEDI subdevice.
267 * @nbytes: Maximum space to reserve in bytes.
269 * Reserve up to @nbytes bytes of space to be written in the COMEDI acquisition
270 * data buffer associated with the subdevice. The amount reserved is limited
271 * by the space available.
273 * Return: The amount of space reserved in bytes.
275 unsigned int comedi_buf_write_alloc(struct comedi_subdevice
*s
,
278 struct comedi_async
*async
= s
->async
;
279 unsigned int unalloc
= comedi_buf_write_n_unalloc(s
);
281 if (nbytes
> unalloc
)
284 async
->buf_write_alloc_count
+= nbytes
;
287 * ensure the async buffer 'counts' are read and updated
288 * before we write data to the write-alloc'ed buffer space
294 EXPORT_SYMBOL_GPL(comedi_buf_write_alloc
);
297 * munging is applied to data by core as it passes between user
300 static unsigned int comedi_buf_munge(struct comedi_subdevice
*s
,
301 unsigned int num_bytes
)
303 struct comedi_async
*async
= s
->async
;
304 unsigned int count
= 0;
305 const unsigned num_sample_bytes
= comedi_bytes_per_sample(s
);
307 if (!s
->munge
|| (async
->cmd
.flags
& CMDF_RAWDATA
)) {
308 async
->munge_count
+= num_bytes
;
311 /* don't munge partial samples */
312 num_bytes
-= num_bytes
% num_sample_bytes
;
313 while (count
< num_bytes
) {
314 int block_size
= num_bytes
- count
;
315 unsigned int buf_end
;
317 buf_end
= async
->prealloc_bufsz
- async
->munge_ptr
;
318 if (block_size
> buf_end
)
319 block_size
= buf_end
;
321 s
->munge(s
->device
, s
,
322 async
->prealloc_buf
+ async
->munge_ptr
,
323 block_size
, async
->munge_chan
);
326 * ensure data is munged in buffer before the
327 * async buffer munge_count is incremented
331 async
->munge_chan
+= block_size
/ num_sample_bytes
;
332 async
->munge_chan
%= async
->cmd
.chanlist_len
;
333 async
->munge_count
+= block_size
;
334 async
->munge_ptr
+= block_size
;
335 async
->munge_ptr
%= async
->prealloc_bufsz
;
343 unsigned int comedi_buf_write_n_allocated(struct comedi_subdevice
*s
)
345 struct comedi_async
*async
= s
->async
;
347 return async
->buf_write_alloc_count
- async
->buf_write_count
;
351 * comedi_buf_write_free() - Free buffer space after it is written
352 * @s: COMEDI subdevice.
353 * @nbytes: Maximum space to free in bytes.
355 * Free up to @nbytes bytes of space previously reserved for writing in the
356 * COMEDI acquisition data buffer associated with the subdevice. The amount of
357 * space freed is limited to the amount that was reserved. The freed space is
358 * assumed to have been filled with sample data by the writer.
360 * If the samples in the freed space need to be "munged", do so here. The
361 * freed space becomes available for allocation by the reader.
363 * Return: The amount of space freed in bytes.
365 unsigned int comedi_buf_write_free(struct comedi_subdevice
*s
,
368 struct comedi_async
*async
= s
->async
;
369 unsigned int allocated
= comedi_buf_write_n_allocated(s
);
371 if (nbytes
> allocated
)
374 async
->buf_write_count
+= nbytes
;
375 async
->buf_write_ptr
+= nbytes
;
376 comedi_buf_munge(s
, async
->buf_write_count
- async
->munge_count
);
377 if (async
->buf_write_ptr
>= async
->prealloc_bufsz
)
378 async
->buf_write_ptr
%= async
->prealloc_bufsz
;
382 EXPORT_SYMBOL_GPL(comedi_buf_write_free
);
385 * comedi_buf_read_n_available() - Determine amount of readable buffer space
386 * @s: COMEDI subdevice.
388 * Determine the amount of readable buffer space in the COMEDI acquisition data
389 * buffer associated with the subdevice. The readable buffer space is that
390 * which has been freed by the writer and "munged" to the sample data format
391 * expected by COMEDI if necessary.
393 * Return: The amount of readable buffer space.
395 unsigned int comedi_buf_read_n_available(struct comedi_subdevice
*s
)
397 struct comedi_async
*async
= s
->async
;
403 num_bytes
= async
->munge_count
- async
->buf_read_count
;
406 * ensure the async buffer 'counts' are read before we
407 * attempt to read data from the buffer
413 EXPORT_SYMBOL_GPL(comedi_buf_read_n_available
);
416 * comedi_buf_read_alloc() - Reserve buffer space for reading
417 * @s: COMEDI subdevice.
418 * @nbytes: Maximum space to reserve in bytes.
420 * Reserve up to @nbytes bytes of previously written and "munged" buffer space
421 * for reading in the COMEDI acquisition data buffer associated with the
422 * subdevice. The amount reserved is limited to the space available. The
423 * reader can read from the reserved space and then free it. A reader is also
424 * allowed to read from the space before reserving it as long as it determines
425 * the amount of readable data available, but the space needs to be marked as
426 * reserved before it can be freed.
428 * Return: The amount of space reserved in bytes.
430 unsigned int comedi_buf_read_alloc(struct comedi_subdevice
*s
,
433 struct comedi_async
*async
= s
->async
;
434 unsigned int available
;
436 available
= async
->munge_count
- async
->buf_read_alloc_count
;
437 if (nbytes
> available
)
440 async
->buf_read_alloc_count
+= nbytes
;
443 * ensure the async buffer 'counts' are read before we
444 * attempt to read data from the read-alloc'ed buffer space
450 EXPORT_SYMBOL_GPL(comedi_buf_read_alloc
);
452 static unsigned int comedi_buf_read_n_allocated(struct comedi_async
*async
)
454 return async
->buf_read_alloc_count
- async
->buf_read_count
;
458 * comedi_buf_read_free() - Free buffer space after it has been read
459 * @s: COMEDI subdevice.
460 * @nbytes: Maximum space to free in bytes.
462 * Free up to @nbytes bytes of buffer space previously reserved for reading in
463 * the COMEDI acquisition data buffer associated with the subdevice. The
464 * amount of space freed is limited to the amount that was reserved.
466 * The freed space becomes available for allocation by the writer.
468 * Return: The amount of space freed in bytes.
470 unsigned int comedi_buf_read_free(struct comedi_subdevice
*s
,
473 struct comedi_async
*async
= s
->async
;
474 unsigned int allocated
;
477 * ensure data has been read out of buffer before
478 * the async read count is incremented
482 allocated
= comedi_buf_read_n_allocated(async
);
483 if (nbytes
> allocated
)
486 async
->buf_read_count
+= nbytes
;
487 async
->buf_read_ptr
+= nbytes
;
488 async
->buf_read_ptr
%= async
->prealloc_bufsz
;
491 EXPORT_SYMBOL_GPL(comedi_buf_read_free
);
493 static void comedi_buf_memcpy_to(struct comedi_subdevice
*s
,
494 const void *data
, unsigned int num_bytes
)
496 struct comedi_async
*async
= s
->async
;
497 unsigned int write_ptr
= async
->buf_write_ptr
;
500 unsigned int block_size
;
502 if (write_ptr
+ num_bytes
> async
->prealloc_bufsz
)
503 block_size
= async
->prealloc_bufsz
- write_ptr
;
505 block_size
= num_bytes
;
507 memcpy(async
->prealloc_buf
+ write_ptr
, data
, block_size
);
510 num_bytes
-= block_size
;
516 static void comedi_buf_memcpy_from(struct comedi_subdevice
*s
,
517 void *dest
, unsigned int nbytes
)
520 struct comedi_async
*async
= s
->async
;
521 unsigned int read_ptr
= async
->buf_read_ptr
;
524 unsigned int block_size
;
526 src
= async
->prealloc_buf
+ read_ptr
;
528 if (nbytes
>= async
->prealloc_bufsz
- read_ptr
)
529 block_size
= async
->prealloc_bufsz
- read_ptr
;
533 memcpy(dest
, src
, block_size
);
534 nbytes
-= block_size
;
541 * comedi_buf_write_samples() - Write sample data to COMEDI buffer
542 * @s: COMEDI subdevice.
543 * @data: Pointer to source samples.
544 * @nsamples: Number of samples to write.
546 * Write up to @nsamples samples to the COMEDI acquisition data buffer
547 * associated with the subdevice, mark it as written and update the
548 * acquisition scan progress. If there is not enough room for the specified
549 * number of samples, the number of samples written is limited to the number
550 * that will fit and the %COMEDI_CB_OVERFLOW event flag is set to cause the
551 * acquisition to terminate with an overrun error. Set the %COMEDI_CB_BLOCK
552 * event flag if any samples are written to cause waiting tasks to be woken
553 * when the event flags are processed.
555 * Return: The amount of data written in bytes.
557 unsigned int comedi_buf_write_samples(struct comedi_subdevice
*s
,
558 const void *data
, unsigned int nsamples
)
560 unsigned int max_samples
;
564 * Make sure there is enough room in the buffer for all the samples.
565 * If not, clamp the nsamples to the number that will fit, flag the
566 * buffer overrun and add the samples that fit.
568 max_samples
= comedi_bytes_to_samples(s
, comedi_buf_write_n_unalloc(s
));
569 if (nsamples
> max_samples
) {
570 dev_warn(s
->device
->class_dev
, "buffer overrun\n");
571 s
->async
->events
|= COMEDI_CB_OVERFLOW
;
572 nsamples
= max_samples
;
578 nbytes
= comedi_buf_write_alloc(s
,
579 comedi_samples_to_bytes(s
, nsamples
));
580 comedi_buf_memcpy_to(s
, data
, nbytes
);
581 comedi_buf_write_free(s
, nbytes
);
582 comedi_inc_scan_progress(s
, nbytes
);
583 s
->async
->events
|= COMEDI_CB_BLOCK
;
587 EXPORT_SYMBOL_GPL(comedi_buf_write_samples
);
590 * comedi_buf_read_samples() - Read sample data from COMEDI buffer
591 * @s: COMEDI subdevice.
592 * @data: Pointer to destination.
593 * @nsamples: Maximum number of samples to read.
595 * Read up to @nsamples samples from the COMEDI acquisition data buffer
596 * associated with the subdevice, mark it as read and update the acquisition
597 * scan progress. Limit the number of samples read to the number available.
598 * Set the %COMEDI_CB_BLOCK event flag if any samples are read to cause waiting
599 * tasks to be woken when the event flags are processed.
601 * Return: The amount of data read in bytes.
603 unsigned int comedi_buf_read_samples(struct comedi_subdevice
*s
,
604 void *data
, unsigned int nsamples
)
606 unsigned int max_samples
;
609 /* clamp nsamples to the number of full samples available */
610 max_samples
= comedi_bytes_to_samples(s
,
611 comedi_buf_read_n_available(s
));
612 if (nsamples
> max_samples
)
613 nsamples
= max_samples
;
618 nbytes
= comedi_buf_read_alloc(s
,
619 comedi_samples_to_bytes(s
, nsamples
));
620 comedi_buf_memcpy_from(s
, data
, nbytes
);
621 comedi_buf_read_free(s
, nbytes
);
622 comedi_inc_scan_progress(s
, nbytes
);
623 s
->async
->events
|= COMEDI_CB_BLOCK
;
627 EXPORT_SYMBOL_GPL(comedi_buf_read_samples
);