3 * Copyright (c) 2009, Microsoft Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16 * Place - Suite 330, Boston, MA 02111-1307 USA.
19 * Haiyang Zhang <haiyangz@microsoft.com>
20 * Hank Janssen <hjanssen@microsoft.com>
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
25 #include <linux/kernel.h>
28 #include "ring_buffer.h"
34 /* Amount of space to write to */
35 #define BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))
41 get_ringbuffer_availbytes()
44 Get number of bytes available to read and to write to
45 for the specified ring buffer
49 get_ringbuffer_availbytes(struct hv_ring_buffer_info
*rbi
,
50 u32
*read
, u32
*write
)
52 u32 read_loc
, write_loc
;
54 /* Capture the read/write indices before they changed */
55 read_loc
= rbi
->ring_buffer
->read_index
;
56 write_loc
= rbi
->ring_buffer
->write_index
;
58 *write
= BYTES_AVAIL_TO_WRITE(read_loc
, write_loc
, rbi
->ring_datasize
);
59 *read
= rbi
->ring_datasize
- *write
;
65 get_next_write_location()
68 Get the next write location for the specified ring buffer
72 get_next_write_location(struct hv_ring_buffer_info
*ring_info
)
74 u32 next
= ring_info
->ring_buffer
->write_index
;
76 /* ASSERT(next < ring_info->RingDataSize); */
84 set_next_write_location()
87 Set the next write location for the specified ring buffer
91 set_next_write_location(struct hv_ring_buffer_info
*ring_info
,
92 u32 next_write_location
)
94 ring_info
->ring_buffer
->write_index
= next_write_location
;
100 get_next_read_location()
103 Get the next read location for the specified ring buffer
107 get_next_read_location(struct hv_ring_buffer_info
*ring_info
)
109 u32 next
= ring_info
->ring_buffer
->read_index
;
111 /* ASSERT(next < ring_info->RingDataSize); */
119 get_next_readlocation_withoffset()
122 Get the next read location + offset for the specified ring buffer.
123 This allows the caller to skip
127 get_next_readlocation_withoffset(struct hv_ring_buffer_info
*ring_info
,
130 u32 next
= ring_info
->ring_buffer
->read_index
;
132 /* ASSERT(next < ring_info->RingDataSize); */
134 next
%= ring_info
->ring_datasize
;
142 set_next_read_location()
145 Set the next read location for the specified ring buffer
149 set_next_read_location(struct hv_ring_buffer_info
*ring_info
,
150 u32 next_read_location
)
152 ring_info
->ring_buffer
->read_index
= next_read_location
;
162 Get the start of the ring buffer
166 get_ring_buffer(struct hv_ring_buffer_info
*ring_info
)
168 return (void *)ring_info
->ring_buffer
->buffer
;
175 get_ring_buffersize()
178 Get the size of the ring buffer
182 get_ring_buffersize(struct hv_ring_buffer_info
*ring_info
)
184 return ring_info
->ring_datasize
;
190 get_ring_bufferindices()
193 Get the read and write indices as u64 of the specified ring buffer
197 get_ring_bufferindices(struct hv_ring_buffer_info
*ring_info
)
199 return (u64
)ring_info
->ring_buffer
->write_index
<< 32;
209 Dump out to console the ring buffer info
212 void dump_ring_info(struct hv_ring_buffer_info
*ring_info
, char *prefix
)
214 u32 bytes_avail_towrite
;
215 u32 bytes_avail_toread
;
217 get_ringbuffer_availbytes(ring_info
,
219 &bytes_avail_towrite
);
223 "%s <<ringinfo %p buffer %p avail write %u "
224 "avail read %u read idx %u write idx %u>>",
227 ring_info
->ring_buffer
->buffer
,
230 ring_info
->ring_buffer
->read_index
,
231 ring_info
->ring_buffer
->write_index
);
235 /* Internal routines */
239 struct hv_ring_buffer_info
*ring_info
,
240 u32 start_write_offset
,
246 struct hv_ring_buffer_info
*ring_info
,
249 u32 start_read_offset
);
256 ringbuffer_get_debuginfo()
259 Get various debug metrics for the specified ring buffer
262 void ringbuffer_get_debuginfo(struct hv_ring_buffer_info
*ring_info
,
263 struct hv_ring_buffer_debug_info
*debug_info
)
265 u32 bytes_avail_towrite
;
266 u32 bytes_avail_toread
;
268 if (ring_info
->ring_buffer
) {
269 get_ringbuffer_availbytes(ring_info
,
271 &bytes_avail_towrite
);
273 debug_info
->bytes_avail_toread
= bytes_avail_toread
;
274 debug_info
->bytes_avail_towrite
= bytes_avail_towrite
;
275 debug_info
->current_read_index
=
276 ring_info
->ring_buffer
->read_index
;
277 debug_info
->current_write_index
=
278 ring_info
->ring_buffer
->write_index
;
279 debug_info
->current_interrupt_mask
=
280 ring_info
->ring_buffer
->interrupt_mask
;
288 get_ringbuffer_interrupt_mask()
291 Get the interrupt mask for the specified ring buffer
294 u32
get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info
*rbi
)
296 return rbi
->ring_buffer
->interrupt_mask
;
305 Initialize the ring buffer
308 int hv_ringbuffer_init(struct hv_ring_buffer_info
*ring_info
,
309 void *buffer
, u32 buflen
)
311 if (sizeof(struct hv_ring_buffer
) != PAGE_SIZE
)
314 memset(ring_info
, 0, sizeof(struct hv_ring_buffer_info
));
316 ring_info
->ring_buffer
= (struct hv_ring_buffer
*)buffer
;
317 ring_info
->ring_buffer
->read_index
=
318 ring_info
->ring_buffer
->write_index
= 0;
320 ring_info
->ring_size
= buflen
;
321 ring_info
->ring_datasize
= buflen
- sizeof(struct hv_ring_buffer
);
323 spin_lock_init(&ring_info
->ring_lock
);
334 Cleanup the ring buffer
337 void ringbuffer_cleanup(struct hv_ring_buffer_info
*ring_info
)
347 Write to the ring buffer
350 int ringbuffer_write(struct hv_ring_buffer_info
*outring_info
,
351 struct scatterlist
*sglist
, u32 sgcount
)
354 u32 bytes_avail_towrite
;
355 u32 bytes_avail_toread
;
356 u32 totalbytes_towrite
= 0;
358 struct scatterlist
*sg
;
359 volatile u32 next_write_location
;
360 u64 prev_indices
= 0;
363 for_each_sg(sglist
, sg
, sgcount
, i
)
365 totalbytes_towrite
+= sg
->length
;
368 totalbytes_towrite
+= sizeof(u64
);
370 spin_lock_irqsave(&outring_info
->ring_lock
, flags
);
372 get_ringbuffer_availbytes(outring_info
,
374 &bytes_avail_towrite
);
376 /* Dumpring_info(Outring_info, "BEFORE "); */
378 /* If there is only room for the packet, assume it is full. */
379 /* Otherwise, the next time around, we think the ring buffer */
380 /* is empty since the read index == write index */
381 if (bytes_avail_towrite
<= totalbytes_towrite
) {
382 spin_unlock_irqrestore(&outring_info
->ring_lock
, flags
);
386 /* Write to the ring buffer */
387 next_write_location
= get_next_write_location(outring_info
);
389 for_each_sg(sglist
, sg
, sgcount
, i
)
391 next_write_location
= copyto_ringbuffer(outring_info
,
397 /* Set previous packet start */
398 prev_indices
= get_ring_bufferindices(outring_info
);
400 next_write_location
= copyto_ringbuffer(outring_info
,
405 /* Make sure we flush all writes before updating the writeIndex */
408 /* Now, update the write location */
409 set_next_write_location(outring_info
, next_write_location
);
411 /* Dumpring_info(Outring_info, "AFTER "); */
413 spin_unlock_irqrestore(&outring_info
->ring_lock
, flags
);
424 Read without advancing the read index
427 int ringbuffer_peek(struct hv_ring_buffer_info
*Inring_info
,
428 void *Buffer
, u32 buflen
)
430 u32 bytes_avail_towrite
;
431 u32 bytes_avail_toread
;
432 u32 next_read_location
= 0;
435 spin_lock_irqsave(&Inring_info
->ring_lock
, flags
);
437 get_ringbuffer_availbytes(Inring_info
,
439 &bytes_avail_towrite
);
441 /* Make sure there is something to read */
442 if (bytes_avail_toread
< buflen
) {
444 "got callback but not enough to read "
445 "<avail to read %d read size %d>!!",
449 spin_unlock_irqrestore(&Inring_info
->ring_lock
, flags
);
454 /* Convert to byte offset */
455 next_read_location
= get_next_read_location(Inring_info
);
457 next_read_location
= copyfrom_ringbuffer(Inring_info
,
462 spin_unlock_irqrestore(&Inring_info
->ring_lock
, flags
);
474 Read and advance the read index
477 int ringbuffer_read(struct hv_ring_buffer_info
*inring_info
, void *buffer
,
478 u32 buflen
, u32 offset
)
480 u32 bytes_avail_towrite
;
481 u32 bytes_avail_toread
;
482 u32 next_read_location
= 0;
483 u64 prev_indices
= 0;
489 spin_lock_irqsave(&inring_info
->ring_lock
, flags
);
491 get_ringbuffer_availbytes(inring_info
,
493 &bytes_avail_towrite
);
495 /* Dumpring_info(Inring_info, "BEFORE "); */
497 /* Make sure there is something to read */
498 if (bytes_avail_toread
< buflen
) {
499 spin_unlock_irqrestore(&inring_info
->ring_lock
, flags
);
505 get_next_readlocation_withoffset(inring_info
, offset
);
507 next_read_location
= copyfrom_ringbuffer(inring_info
,
512 next_read_location
= copyfrom_ringbuffer(inring_info
,
517 /* Make sure all reads are done before we update the read index since */
518 /* the writer may start writing to the read area once the read index */
522 /* Update the read index */
523 set_next_read_location(inring_info
, next_read_location
);
525 /* Dumpring_info(Inring_info, "AFTER "); */
527 spin_unlock_irqrestore(&inring_info
->ring_lock
, flags
);
539 Helper routine to copy from source to ring buffer.
540 Assume there is enough room. Handles wrap-around in dest case only!!
545 struct hv_ring_buffer_info
*ring_info
,
546 u32 start_write_offset
,
550 void *ring_buffer
= get_ring_buffer(ring_info
);
551 u32 ring_buffer_size
= get_ring_buffersize(ring_info
);
554 /* wrap-around detected! */
555 if (srclen
> ring_buffer_size
- start_write_offset
) {
556 frag_len
= ring_buffer_size
- start_write_offset
;
557 memcpy(ring_buffer
+ start_write_offset
, src
, frag_len
);
558 memcpy(ring_buffer
, src
+ frag_len
, srclen
- frag_len
);
560 memcpy(ring_buffer
+ start_write_offset
, src
, srclen
);
562 start_write_offset
+= srclen
;
563 start_write_offset
%= ring_buffer_size
;
565 return start_write_offset
;
572 copyfrom_ringbuffer()
575 Helper routine to copy to source from ring buffer.
576 Assume there is enough room. Handles wrap-around in src case only!!
581 struct hv_ring_buffer_info
*ring_info
,
584 u32 start_read_offset
)
586 void *ring_buffer
= get_ring_buffer(ring_info
);
587 u32 ring_buffer_size
= get_ring_buffersize(ring_info
);
591 /* wrap-around detected at the src */
592 if (destlen
> ring_buffer_size
- start_read_offset
) {
593 frag_len
= ring_buffer_size
- start_read_offset
;
595 memcpy(dest
, ring_buffer
+ start_read_offset
, frag_len
);
596 memcpy(dest
+ frag_len
, ring_buffer
, destlen
- frag_len
);
599 memcpy(dest
, ring_buffer
+ start_read_offset
, destlen
);
602 start_read_offset
+= destlen
;
603 start_read_offset
%= ring_buffer_size
;
605 return start_read_offset
;