Commit | Line | Data |
---|---|---|
3e7ee490 HJ |
1 | /* |
2 | * | |
3 | * Copyright (c) 2009, Microsoft Corporation. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms and conditions of the GNU General Public License, | |
7 | * version 2, as published by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 Temple | |
16 | * Place - Suite 330, Boston, MA 02111-1307 USA. | |
17 | * | |
18 | * Authors: | |
19 | * Haiyang Zhang <haiyangz@microsoft.com> | |
20 | * Hank Janssen <hjanssen@microsoft.com> | |
b2a5a585 | 21 | * K. Y. Srinivasan <kys@microsoft.com> |
3e7ee490 HJ |
22 | * |
23 | */ | |
0a46618d | 24 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
3e7ee490 | 25 | |
a0086dc5 GKH |
26 | #include <linux/kernel.h> |
27 | #include <linux/mm.h> | |
46a97191 | 28 | #include <linux/hyperv.h> |
3f335ea2 | 29 | |
0f2a6619 | 30 | #include "hyperv_vmbus.h" |
3e7ee490 | 31 | |
3e7ee490 | 32 | |
454f18a9 BP |
33 | /* #defines */ |
34 | ||
35 | ||
36 | /* Amount of space to write to */ | |
b737b2e0 S |
37 | #define BYTES_AVAIL_TO_WRITE(r, w, z) \ |
38 | ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w)) | |
3e7ee490 HJ |
39 | |
40 | ||
b2a5a585 S |
41 | /* |
42 | * | |
43 | * hv_get_ringbuffer_availbytes() | |
44 | * | |
45 | * Get number of bytes available to read and to write to | |
46 | * for the specified ring buffer | |
47 | */ | |
3e7ee490 | 48 | static inline void |
2b8a912e | 49 | hv_get_ringbuffer_availbytes(struct hv_ring_buffer_info *rbi, |
1ac58644 | 50 | u32 *read, u32 *write) |
3e7ee490 | 51 | { |
4408f531 | 52 | u32 read_loc, write_loc; |
3e7ee490 | 53 | |
df2a4a71 S |
54 | smp_read_barrier_depends(); |
55 | ||
454f18a9 | 56 | /* Capture the read/write indices before they changed */ |
82f8bd40 HZ |
57 | read_loc = rbi->ring_buffer->read_index; |
58 | write_loc = rbi->ring_buffer->write_index; | |
3e7ee490 | 59 | |
82f8bd40 HZ |
60 | *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->ring_datasize); |
61 | *read = rbi->ring_datasize - *write; | |
3e7ee490 HJ |
62 | } |
63 | ||
b2a5a585 S |
64 | /* |
65 | * hv_get_next_write_location() | |
66 | * | |
67 | * Get the next write location for the specified ring buffer | |
68 | * | |
69 | */ | |
4d643114 | 70 | static inline u32 |
2b8a912e | 71 | hv_get_next_write_location(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 72 | { |
fc8c72eb | 73 | u32 next = ring_info->ring_buffer->write_index; |
3e7ee490 | 74 | |
3e7ee490 HJ |
75 | return next; |
76 | } | |
77 | ||
b2a5a585 S |
78 | /* |
79 | * hv_set_next_write_location() | |
80 | * | |
81 | * Set the next write location for the specified ring buffer | |
82 | * | |
83 | */ | |
3e7ee490 | 84 | static inline void |
2b8a912e | 85 | hv_set_next_write_location(struct hv_ring_buffer_info *ring_info, |
fc8c72eb | 86 | u32 next_write_location) |
3e7ee490 | 87 | { |
fc8c72eb | 88 | ring_info->ring_buffer->write_index = next_write_location; |
3e7ee490 HJ |
89 | } |
90 | ||
b2a5a585 S |
91 | /* |
92 | * hv_get_next_read_location() | |
93 | * | |
94 | * Get the next read location for the specified ring buffer | |
95 | */ | |
4d643114 | 96 | static inline u32 |
2b8a912e | 97 | hv_get_next_read_location(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 98 | { |
fc8c72eb | 99 | u32 next = ring_info->ring_buffer->read_index; |
3e7ee490 | 100 | |
3e7ee490 HJ |
101 | return next; |
102 | } | |
103 | ||
b2a5a585 S |
104 | /* |
105 | * hv_get_next_readlocation_withoffset() | |
106 | * | |
107 | * Get the next read location + offset for the specified ring buffer. | |
108 | * This allows the caller to skip | |
109 | */ | |
4d643114 | 110 | static inline u32 |
2b8a912e | 111 | hv_get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info, |
1ac58644 | 112 | u32 offset) |
3e7ee490 | 113 | { |
fc8c72eb | 114 | u32 next = ring_info->ring_buffer->read_index; |
3e7ee490 | 115 | |
fc8c72eb HZ |
116 | next += offset; |
117 | next %= ring_info->ring_datasize; | |
3e7ee490 HJ |
118 | |
119 | return next; | |
120 | } | |
121 | ||
b2a5a585 S |
122 | /* |
123 | * | |
124 | * hv_set_next_read_location() | |
125 | * | |
126 | * Set the next read location for the specified ring buffer | |
127 | * | |
128 | */ | |
3e7ee490 | 129 | static inline void |
2b8a912e | 130 | hv_set_next_read_location(struct hv_ring_buffer_info *ring_info, |
fc8c72eb | 131 | u32 next_read_location) |
3e7ee490 | 132 | { |
fc8c72eb | 133 | ring_info->ring_buffer->read_index = next_read_location; |
3e7ee490 HJ |
134 | } |
135 | ||
136 | ||
b2a5a585 S |
137 | /* |
138 | * | |
139 | * hv_get_ring_buffer() | |
140 | * | |
141 | * Get the start of the ring buffer | |
142 | */ | |
8282c400 | 143 | static inline void * |
2b8a912e | 144 | hv_get_ring_buffer(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 145 | { |
fc8c72eb | 146 | return (void *)ring_info->ring_buffer->buffer; |
3e7ee490 HJ |
147 | } |
148 | ||
149 | ||
b2a5a585 S |
150 | /* |
151 | * | |
152 | * hv_get_ring_buffersize() | |
153 | * | |
154 | * Get the size of the ring buffer | |
155 | */ | |
4d643114 | 156 | static inline u32 |
2b8a912e | 157 | hv_get_ring_buffersize(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 158 | { |
fc8c72eb | 159 | return ring_info->ring_datasize; |
3e7ee490 HJ |
160 | } |
161 | ||
b2a5a585 S |
162 | /* |
163 | * | |
164 | * hv_get_ring_bufferindices() | |
165 | * | |
166 | * Get the read and write indices as u64 of the specified ring buffer | |
167 | * | |
168 | */ | |
59471438 | 169 | static inline u64 |
2b8a912e | 170 | hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 171 | { |
fc8c72eb | 172 | return (u64)ring_info->ring_buffer->write_index << 32; |
3e7ee490 HJ |
173 | } |
174 | ||
8f1136ae S |
175 | /* |
176 | * | |
177 | * hv_copyfrom_ringbuffer() | |
178 | * | |
179 | * Helper routine to copy to source from ring buffer. | |
180 | * Assume there is enough room. Handles wrap-around in src case only!! | |
181 | * | |
182 | */ | |
183 | static u32 hv_copyfrom_ringbuffer( | |
184 | struct hv_ring_buffer_info *ring_info, | |
185 | void *dest, | |
186 | u32 destlen, | |
187 | u32 start_read_offset) | |
188 | { | |
189 | void *ring_buffer = hv_get_ring_buffer(ring_info); | |
190 | u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); | |
191 | ||
192 | u32 frag_len; | |
193 | ||
194 | /* wrap-around detected at the src */ | |
195 | if (destlen > ring_buffer_size - start_read_offset) { | |
196 | frag_len = ring_buffer_size - start_read_offset; | |
197 | ||
198 | memcpy(dest, ring_buffer + start_read_offset, frag_len); | |
199 | memcpy(dest + frag_len, ring_buffer, destlen - frag_len); | |
200 | } else | |
201 | ||
202 | memcpy(dest, ring_buffer + start_read_offset, destlen); | |
203 | ||
204 | ||
205 | start_read_offset += destlen; | |
206 | start_read_offset %= ring_buffer_size; | |
207 | ||
208 | return start_read_offset; | |
209 | } | |
210 | ||
211 | ||
7581578d S |
212 | /* |
213 | * | |
214 | * hv_copyto_ringbuffer() | |
215 | * | |
216 | * Helper routine to copy from source to ring buffer. | |
217 | * Assume there is enough room. Handles wrap-around in dest case only!! | |
218 | * | |
219 | */ | |
220 | static u32 hv_copyto_ringbuffer( | |
fc8c72eb HZ |
221 | struct hv_ring_buffer_info *ring_info, |
222 | u32 start_write_offset, | |
223 | void *src, | |
7581578d S |
224 | u32 srclen) |
225 | { | |
226 | void *ring_buffer = hv_get_ring_buffer(ring_info); | |
227 | u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); | |
228 | u32 frag_len; | |
229 | ||
230 | /* wrap-around detected! */ | |
231 | if (srclen > ring_buffer_size - start_write_offset) { | |
232 | frag_len = ring_buffer_size - start_write_offset; | |
233 | memcpy(ring_buffer + start_write_offset, src, frag_len); | |
234 | memcpy(ring_buffer, src + frag_len, srclen - frag_len); | |
235 | } else | |
236 | memcpy(ring_buffer + start_write_offset, src, srclen); | |
3e7ee490 | 237 | |
7581578d S |
238 | start_write_offset += srclen; |
239 | start_write_offset %= ring_buffer_size; | |
240 | ||
241 | return start_write_offset; | |
242 | } | |
3e7ee490 | 243 | |
b2a5a585 S |
244 | /* |
245 | * | |
246 | * hv_ringbuffer_get_debuginfo() | |
247 | * | |
248 | * Get various debug metrics for the specified ring buffer | |
249 | * | |
250 | */ | |
a75b61d5 | 251 | void hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info, |
80682b7a | 252 | struct hv_ring_buffer_debug_info *debug_info) |
3e7ee490 | 253 | { |
fc8c72eb HZ |
254 | u32 bytes_avail_towrite; |
255 | u32 bytes_avail_toread; | |
3e7ee490 | 256 | |
fc8c72eb | 257 | if (ring_info->ring_buffer) { |
2b8a912e | 258 | hv_get_ringbuffer_availbytes(ring_info, |
fc8c72eb HZ |
259 | &bytes_avail_toread, |
260 | &bytes_avail_towrite); | |
3e7ee490 | 261 | |
fc8c72eb HZ |
262 | debug_info->bytes_avail_toread = bytes_avail_toread; |
263 | debug_info->bytes_avail_towrite = bytes_avail_towrite; | |
82f8bd40 | 264 | debug_info->current_read_index = |
fc8c72eb | 265 | ring_info->ring_buffer->read_index; |
82f8bd40 | 266 | debug_info->current_write_index = |
fc8c72eb | 267 | ring_info->ring_buffer->write_index; |
82f8bd40 | 268 | debug_info->current_interrupt_mask = |
fc8c72eb | 269 | ring_info->ring_buffer->interrupt_mask; |
3e7ee490 HJ |
270 | } |
271 | } | |
272 | ||
273 | ||
b2a5a585 S |
274 | /* |
275 | * | |
276 | * hv_get_ringbuffer_interrupt_mask() | |
277 | * | |
278 | * Get the interrupt mask for the specified ring buffer | |
279 | * | |
280 | */ | |
decc49da | 281 | u32 hv_get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info *rbi) |
3e7ee490 | 282 | { |
82f8bd40 | 283 | return rbi->ring_buffer->interrupt_mask; |
3e7ee490 HJ |
284 | } |
285 | ||
b2a5a585 S |
286 | /* |
287 | * | |
288 | * hv_ringbuffer_init() | |
289 | * | |
290 | *Initialize the ring buffer | |
291 | * | |
292 | */ | |
72a95cbc | 293 | int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, |
fc8c72eb | 294 | void *buffer, u32 buflen) |
3e7ee490 | 295 | { |
4a1b3acc | 296 | if (sizeof(struct hv_ring_buffer) != PAGE_SIZE) |
3324fb40 | 297 | return -EINVAL; |
3e7ee490 | 298 | |
fc8c72eb | 299 | memset(ring_info, 0, sizeof(struct hv_ring_buffer_info)); |
3e7ee490 | 300 | |
fc8c72eb HZ |
301 | ring_info->ring_buffer = (struct hv_ring_buffer *)buffer; |
302 | ring_info->ring_buffer->read_index = | |
303 | ring_info->ring_buffer->write_index = 0; | |
3e7ee490 | 304 | |
fc8c72eb HZ |
305 | ring_info->ring_size = buflen; |
306 | ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer); | |
3e7ee490 | 307 | |
fc8c72eb | 308 | spin_lock_init(&ring_info->ring_lock); |
3e7ee490 HJ |
309 | |
310 | return 0; | |
311 | } | |
312 | ||
b2a5a585 S |
313 | /* |
314 | * | |
315 | * hv_ringbuffer_cleanup() | |
316 | * | |
317 | * Cleanup the ring buffer | |
318 | * | |
319 | */ | |
2dba688b | 320 | void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) |
3e7ee490 | 321 | { |
3e7ee490 HJ |
322 | } |
323 | ||
b2a5a585 S |
324 | /* |
325 | * | |
326 | * hv_ringbuffer_write() | |
327 | * | |
328 | * Write to the ring buffer | |
329 | * | |
330 | */ | |
633c4dce | 331 | int hv_ringbuffer_write(struct hv_ring_buffer_info *outring_info, |
3523a805 | 332 | struct scatterlist *sglist, u32 sgcount) |
3e7ee490 | 333 | { |
4408f531 | 334 | int i = 0; |
fc8c72eb HZ |
335 | u32 bytes_avail_towrite; |
336 | u32 bytes_avail_toread; | |
337 | u32 totalbytes_towrite = 0; | |
3e7ee490 | 338 | |
b219b3f7 | 339 | struct scatterlist *sg; |
66a60543 | 340 | u32 next_write_location; |
fc8c72eb | 341 | u64 prev_indices = 0; |
a98f96ee | 342 | unsigned long flags; |
3e7ee490 | 343 | |
b219b3f7 | 344 | for_each_sg(sglist, sg, sgcount, i) |
3e7ee490 | 345 | { |
fc8c72eb | 346 | totalbytes_towrite += sg->length; |
3e7ee490 HJ |
347 | } |
348 | ||
fc8c72eb | 349 | totalbytes_towrite += sizeof(u64); |
3e7ee490 | 350 | |
fc8c72eb | 351 | spin_lock_irqsave(&outring_info->ring_lock, flags); |
3e7ee490 | 352 | |
2b8a912e | 353 | hv_get_ringbuffer_availbytes(outring_info, |
fc8c72eb HZ |
354 | &bytes_avail_toread, |
355 | &bytes_avail_towrite); | |
3e7ee490 | 356 | |
3e7ee490 | 357 | |
4408f531 B |
358 | /* If there is only room for the packet, assume it is full. */ |
359 | /* Otherwise, the next time around, we think the ring buffer */ | |
454f18a9 | 360 | /* is empty since the read index == write index */ |
fc8c72eb | 361 | if (bytes_avail_towrite <= totalbytes_towrite) { |
fc8c72eb | 362 | spin_unlock_irqrestore(&outring_info->ring_lock, flags); |
d2598f01 | 363 | return -EAGAIN; |
3e7ee490 HJ |
364 | } |
365 | ||
454f18a9 | 366 | /* Write to the ring buffer */ |
2b8a912e | 367 | next_write_location = hv_get_next_write_location(outring_info); |
3e7ee490 | 368 | |
b219b3f7 | 369 | for_each_sg(sglist, sg, sgcount, i) |
3e7ee490 | 370 | { |
2b8a912e | 371 | next_write_location = hv_copyto_ringbuffer(outring_info, |
fc8c72eb | 372 | next_write_location, |
b219b3f7 NP |
373 | sg_virt(sg), |
374 | sg->length); | |
3e7ee490 HJ |
375 | } |
376 | ||
454f18a9 | 377 | /* Set previous packet start */ |
2b8a912e | 378 | prev_indices = hv_get_ring_bufferindices(outring_info); |
3e7ee490 | 379 | |
2b8a912e | 380 | next_write_location = hv_copyto_ringbuffer(outring_info, |
fc8c72eb HZ |
381 | next_write_location, |
382 | &prev_indices, | |
b219b3f7 | 383 | sizeof(u64)); |
3e7ee490 | 384 | |
454f18a9 | 385 | /* Make sure we flush all writes before updating the writeIndex */ |
e690b5a9 | 386 | smp_wmb(); |
3e7ee490 | 387 | |
454f18a9 | 388 | /* Now, update the write location */ |
2b8a912e | 389 | hv_set_next_write_location(outring_info, next_write_location); |
3e7ee490 | 390 | |
3e7ee490 | 391 | |
fc8c72eb | 392 | spin_unlock_irqrestore(&outring_info->ring_lock, flags); |
3e7ee490 HJ |
393 | return 0; |
394 | } | |
395 | ||
396 | ||
b2a5a585 S |
397 | /* |
398 | * | |
399 | * hv_ringbuffer_peek() | |
400 | * | |
401 | * Read without advancing the read index | |
402 | * | |
403 | */ | |
a89186c2 | 404 | int hv_ringbuffer_peek(struct hv_ring_buffer_info *Inring_info, |
fc8c72eb | 405 | void *Buffer, u32 buflen) |
3e7ee490 | 406 | { |
fc8c72eb HZ |
407 | u32 bytes_avail_towrite; |
408 | u32 bytes_avail_toread; | |
409 | u32 next_read_location = 0; | |
a98f96ee | 410 | unsigned long flags; |
3e7ee490 | 411 | |
fc8c72eb | 412 | spin_lock_irqsave(&Inring_info->ring_lock, flags); |
3e7ee490 | 413 | |
2b8a912e | 414 | hv_get_ringbuffer_availbytes(Inring_info, |
fc8c72eb HZ |
415 | &bytes_avail_toread, |
416 | &bytes_avail_towrite); | |
3e7ee490 | 417 | |
454f18a9 | 418 | /* Make sure there is something to read */ |
fc8c72eb | 419 | if (bytes_avail_toread < buflen) { |
3e7ee490 | 420 | |
fc8c72eb | 421 | spin_unlock_irqrestore(&Inring_info->ring_lock, flags); |
3e7ee490 | 422 | |
d2598f01 | 423 | return -EAGAIN; |
3e7ee490 HJ |
424 | } |
425 | ||
454f18a9 | 426 | /* Convert to byte offset */ |
2b8a912e | 427 | next_read_location = hv_get_next_read_location(Inring_info); |
3e7ee490 | 428 | |
2b8a912e | 429 | next_read_location = hv_copyfrom_ringbuffer(Inring_info, |
4408f531 | 430 | Buffer, |
fc8c72eb HZ |
431 | buflen, |
432 | next_read_location); | |
3e7ee490 | 433 | |
fc8c72eb | 434 | spin_unlock_irqrestore(&Inring_info->ring_lock, flags); |
3e7ee490 HJ |
435 | |
436 | return 0; | |
437 | } | |
438 | ||
439 | ||
b2a5a585 S |
440 | /* |
441 | * | |
442 | * hv_ringbuffer_read() | |
443 | * | |
444 | * Read and advance the read index | |
445 | * | |
446 | */ | |
38397c8a | 447 | int hv_ringbuffer_read(struct hv_ring_buffer_info *inring_info, void *buffer, |
fc8c72eb | 448 | u32 buflen, u32 offset) |
3e7ee490 | 449 | { |
fc8c72eb HZ |
450 | u32 bytes_avail_towrite; |
451 | u32 bytes_avail_toread; | |
452 | u32 next_read_location = 0; | |
453 | u64 prev_indices = 0; | |
a98f96ee | 454 | unsigned long flags; |
3e7ee490 | 455 | |
fc8c72eb | 456 | if (buflen <= 0) |
a16e1485 | 457 | return -EINVAL; |
3e7ee490 | 458 | |
fc8c72eb | 459 | spin_lock_irqsave(&inring_info->ring_lock, flags); |
3e7ee490 | 460 | |
2b8a912e | 461 | hv_get_ringbuffer_availbytes(inring_info, |
fc8c72eb HZ |
462 | &bytes_avail_toread, |
463 | &bytes_avail_towrite); | |
3e7ee490 | 464 | |
454f18a9 | 465 | /* Make sure there is something to read */ |
fc8c72eb | 466 | if (bytes_avail_toread < buflen) { |
fc8c72eb | 467 | spin_unlock_irqrestore(&inring_info->ring_lock, flags); |
3e7ee490 | 468 | |
d2598f01 | 469 | return -EAGAIN; |
3e7ee490 HJ |
470 | } |
471 | ||
1ac58644 | 472 | next_read_location = |
2b8a912e | 473 | hv_get_next_readlocation_withoffset(inring_info, offset); |
3e7ee490 | 474 | |
2b8a912e | 475 | next_read_location = hv_copyfrom_ringbuffer(inring_info, |
fc8c72eb HZ |
476 | buffer, |
477 | buflen, | |
478 | next_read_location); | |
3e7ee490 | 479 | |
2b8a912e | 480 | next_read_location = hv_copyfrom_ringbuffer(inring_info, |
fc8c72eb | 481 | &prev_indices, |
4408f531 | 482 | sizeof(u64), |
fc8c72eb | 483 | next_read_location); |
3e7ee490 | 484 | |
454f18a9 | 485 | /* Make sure all reads are done before we update the read index since */ |
4408f531 B |
486 | /* the writer may start writing to the read area once the read index */ |
487 | /*is updated */ | |
ef0d5b23 | 488 | smp_mb(); |
3e7ee490 | 489 | |
454f18a9 | 490 | /* Update the read index */ |
2b8a912e | 491 | hv_set_next_read_location(inring_info, next_read_location); |
3e7ee490 | 492 | |
fc8c72eb | 493 | spin_unlock_irqrestore(&inring_info->ring_lock, flags); |
3e7ee490 HJ |
494 | |
495 | return 0; | |
496 | } |