Commit | Line | Data |
---|---|---|
225c7b1f RD |
1 | /* |
2 | * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. | |
51a379d0 | 3 | * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. |
225c7b1f RD |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
32 | */ | |
33 | ||
34 | #include <linux/errno.h> | |
35 | #include <linux/slab.h> | |
6526128c | 36 | #include <linux/mm.h> |
ee40fa06 | 37 | #include <linux/export.h> |
225c7b1f | 38 | #include <linux/bitmap.h> |
9cbe05c7 | 39 | #include <linux/dma-mapping.h> |
29c27112 | 40 | #include <linux/vmalloc.h> |
225c7b1f RD |
41 | |
42 | #include "mlx4.h" | |
43 | ||
44 | u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap) | |
45 | { | |
46 | u32 obj; | |
47 | ||
48 | spin_lock(&bitmap->lock); | |
49 | ||
50 | obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last); | |
51 | if (obj >= bitmap->max) { | |
93fc9e1b YP |
52 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
53 | & bitmap->mask; | |
225c7b1f RD |
54 | obj = find_first_zero_bit(bitmap->table, bitmap->max); |
55 | } | |
56 | ||
57 | if (obj < bitmap->max) { | |
58 | set_bit(obj, bitmap->table); | |
93fc9e1b YP |
59 | bitmap->last = (obj + 1); |
60 | if (bitmap->last == bitmap->max) | |
61 | bitmap->last = 0; | |
225c7b1f | 62 | obj |= bitmap->top; |
225c7b1f RD |
63 | } else |
64 | obj = -1; | |
65 | ||
42d1e017 EC |
66 | if (obj != -1) |
67 | --bitmap->avail; | |
68 | ||
225c7b1f RD |
69 | spin_unlock(&bitmap->lock); |
70 | ||
71 | return obj; | |
72 | } | |
73 | ||
7c6d74d2 | 74 | void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr) |
225c7b1f | 75 | { |
7c6d74d2 | 76 | mlx4_bitmap_free_range(bitmap, obj, 1, use_rr); |
a3cdcbfa YP |
77 | } |
78 | ||
ddae0349 EE |
79 | static unsigned long find_aligned_range(unsigned long *bitmap, |
80 | u32 start, u32 nbits, | |
81 | int len, int align, u32 skip_mask) | |
82 | { | |
83 | unsigned long end, i; | |
84 | ||
85 | again: | |
86 | start = ALIGN(start, align); | |
87 | ||
88 | while ((start < nbits) && (test_bit(start, bitmap) || | |
89 | (start & skip_mask))) | |
90 | start += align; | |
91 | ||
92 | if (start >= nbits) | |
93 | return -1; | |
94 | ||
95 | end = start+len; | |
96 | if (end > nbits) | |
97 | return -1; | |
98 | ||
99 | for (i = start + 1; i < end; i++) { | |
100 | if (test_bit(i, bitmap) || ((u32)i & skip_mask)) { | |
101 | start = i + 1; | |
102 | goto again; | |
103 | } | |
104 | } | |
105 | ||
106 | return start; | |
107 | } | |
108 | ||
109 | u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, | |
110 | int align, u32 skip_mask) | |
a3cdcbfa | 111 | { |
e27cd4f8 | 112 | u32 obj; |
a3cdcbfa | 113 | |
ddae0349 | 114 | if (likely(cnt == 1 && align == 1 && !skip_mask)) |
a3cdcbfa YP |
115 | return mlx4_bitmap_alloc(bitmap); |
116 | ||
117 | spin_lock(&bitmap->lock); | |
118 | ||
ddae0349 EE |
119 | obj = find_aligned_range(bitmap->table, bitmap->last, |
120 | bitmap->max, cnt, align, skip_mask); | |
a3cdcbfa | 121 | if (obj >= bitmap->max) { |
93fc9e1b YP |
122 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) |
123 | & bitmap->mask; | |
ddae0349 EE |
124 | obj = find_aligned_range(bitmap->table, 0, bitmap->max, |
125 | cnt, align, skip_mask); | |
a3cdcbfa YP |
126 | } |
127 | ||
128 | if (obj < bitmap->max) { | |
e27cd4f8 | 129 | bitmap_set(bitmap->table, obj, cnt); |
a3cdcbfa YP |
130 | if (obj == bitmap->last) { |
131 | bitmap->last = (obj + cnt); | |
132 | if (bitmap->last >= bitmap->max) | |
133 | bitmap->last = 0; | |
134 | } | |
135 | obj |= bitmap->top; | |
136 | } else | |
137 | obj = -1; | |
138 | ||
42d1e017 EC |
139 | if (obj != -1) |
140 | bitmap->avail -= cnt; | |
141 | ||
a3cdcbfa YP |
142 | spin_unlock(&bitmap->lock); |
143 | ||
144 | return obj; | |
145 | } | |
146 | ||
42d1e017 EC |
147 | u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap) |
148 | { | |
149 | return bitmap->avail; | |
150 | } | |
151 | ||
7a89399f MB |
152 | static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj) |
153 | { | |
154 | return obj & (bitmap->max + bitmap->reserved_top - 1); | |
155 | } | |
156 | ||
7c6d74d2 JM |
157 | void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt, |
158 | int use_rr) | |
a3cdcbfa | 159 | { |
93fc9e1b | 160 | obj &= bitmap->max + bitmap->reserved_top - 1; |
225c7b1f RD |
161 | |
162 | spin_lock(&bitmap->lock); | |
7c6d74d2 JM |
163 | if (!use_rr) { |
164 | bitmap->last = min(bitmap->last, obj); | |
165 | bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top) | |
166 | & bitmap->mask; | |
167 | } | |
e27cd4f8 | 168 | bitmap_clear(bitmap->table, obj, cnt); |
42d1e017 | 169 | bitmap->avail += cnt; |
225c7b1f RD |
170 | spin_unlock(&bitmap->lock); |
171 | } | |
172 | ||
93fc9e1b YP |
173 | int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask, |
174 | u32 reserved_bot, u32 reserved_top) | |
225c7b1f | 175 | { |
225c7b1f RD |
176 | /* num must be a power of 2 */ |
177 | if (num != roundup_pow_of_two(num)) | |
178 | return -EINVAL; | |
179 | ||
180 | bitmap->last = 0; | |
181 | bitmap->top = 0; | |
93fc9e1b | 182 | bitmap->max = num - reserved_top; |
225c7b1f | 183 | bitmap->mask = mask; |
93fc9e1b | 184 | bitmap->reserved_top = reserved_top; |
42d1e017 | 185 | bitmap->avail = num - reserved_top - reserved_bot; |
7a89399f | 186 | bitmap->effective_len = bitmap->avail; |
225c7b1f | 187 | spin_lock_init(&bitmap->lock); |
93fc9e1b YP |
188 | bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) * |
189 | sizeof (long), GFP_KERNEL); | |
225c7b1f RD |
190 | if (!bitmap->table) |
191 | return -ENOMEM; | |
192 | ||
e27cd4f8 | 193 | bitmap_set(bitmap->table, 0, reserved_bot); |
225c7b1f RD |
194 | |
195 | return 0; | |
196 | } | |
197 | ||
198 | void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap) | |
199 | { | |
200 | kfree(bitmap->table); | |
201 | } | |
202 | ||
7a89399f MB |
203 | struct mlx4_zone_allocator { |
204 | struct list_head entries; | |
205 | struct list_head prios; | |
206 | u32 last_uid; | |
207 | u32 mask; | |
208 | /* protect the zone_allocator from concurrent accesses */ | |
209 | spinlock_t lock; | |
210 | enum mlx4_zone_alloc_flags flags; | |
211 | }; | |
212 | ||
213 | struct mlx4_zone_entry { | |
214 | struct list_head list; | |
215 | struct list_head prio_list; | |
216 | u32 uid; | |
217 | struct mlx4_zone_allocator *allocator; | |
218 | struct mlx4_bitmap *bitmap; | |
219 | int use_rr; | |
220 | int priority; | |
221 | int offset; | |
222 | enum mlx4_zone_flags flags; | |
223 | }; | |
224 | ||
225 | struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags) | |
226 | { | |
227 | struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL); | |
228 | ||
229 | if (NULL == zones) | |
230 | return NULL; | |
231 | ||
232 | INIT_LIST_HEAD(&zones->entries); | |
233 | INIT_LIST_HEAD(&zones->prios); | |
234 | spin_lock_init(&zones->lock); | |
235 | zones->last_uid = 0; | |
236 | zones->mask = 0; | |
237 | zones->flags = flags; | |
238 | ||
239 | return zones; | |
240 | } | |
241 | ||
242 | int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc, | |
243 | struct mlx4_bitmap *bitmap, | |
244 | u32 flags, | |
245 | int priority, | |
246 | int offset, | |
247 | u32 *puid) | |
248 | { | |
249 | u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1); | |
250 | struct mlx4_zone_entry *it; | |
251 | struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL); | |
252 | ||
253 | if (NULL == zone) | |
254 | return -ENOMEM; | |
255 | ||
256 | zone->flags = flags; | |
257 | zone->bitmap = bitmap; | |
258 | zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0; | |
259 | zone->priority = priority; | |
260 | zone->offset = offset; | |
261 | ||
262 | spin_lock(&zone_alloc->lock); | |
263 | ||
264 | zone->uid = zone_alloc->last_uid++; | |
265 | zone->allocator = zone_alloc; | |
266 | ||
267 | if (zone_alloc->mask < mask) | |
268 | zone_alloc->mask = mask; | |
269 | ||
270 | list_for_each_entry(it, &zone_alloc->prios, prio_list) | |
271 | if (it->priority >= priority) | |
272 | break; | |
273 | ||
274 | if (&it->prio_list == &zone_alloc->prios || it->priority > priority) | |
275 | list_add_tail(&zone->prio_list, &it->prio_list); | |
276 | list_add_tail(&zone->list, &it->list); | |
277 | ||
278 | spin_unlock(&zone_alloc->lock); | |
279 | ||
280 | *puid = zone->uid; | |
281 | ||
282 | return 0; | |
283 | } | |
284 | ||
285 | /* Should be called under a lock */ | |
286 | static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry) | |
287 | { | |
288 | struct mlx4_zone_allocator *zone_alloc = entry->allocator; | |
289 | ||
290 | if (!list_empty(&entry->prio_list)) { | |
291 | /* Check if we need to add an alternative node to the prio list */ | |
292 | if (!list_is_last(&entry->list, &zone_alloc->entries)) { | |
293 | struct mlx4_zone_entry *next = list_first_entry(&entry->list, | |
294 | typeof(*next), | |
295 | list); | |
296 | ||
297 | if (next->priority == entry->priority) | |
298 | list_add_tail(&next->prio_list, &entry->prio_list); | |
299 | } | |
300 | ||
301 | list_del(&entry->prio_list); | |
302 | } | |
303 | ||
304 | list_del(&entry->list); | |
305 | ||
306 | if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) { | |
307 | u32 mask = 0; | |
308 | struct mlx4_zone_entry *it; | |
309 | ||
310 | list_for_each_entry(it, &zone_alloc->prios, prio_list) { | |
311 | u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1); | |
312 | ||
313 | if (mask < cur_mask) | |
314 | mask = cur_mask; | |
315 | } | |
316 | zone_alloc->mask = mask; | |
317 | } | |
318 | ||
319 | return 0; | |
320 | } | |
321 | ||
322 | void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc) | |
323 | { | |
324 | struct mlx4_zone_entry *zone, *tmp; | |
325 | ||
326 | spin_lock(&zone_alloc->lock); | |
327 | ||
328 | list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) { | |
329 | list_del(&zone->list); | |
330 | list_del(&zone->prio_list); | |
331 | kfree(zone); | |
332 | } | |
333 | ||
334 | spin_unlock(&zone_alloc->lock); | |
335 | kfree(zone_alloc); | |
336 | } | |
337 | ||
338 | /* Should be called under a lock */ | |
339 | static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count, | |
340 | int align, u32 skip_mask, u32 *puid) | |
341 | { | |
342 | u32 uid; | |
343 | u32 res; | |
344 | struct mlx4_zone_allocator *zone_alloc = zone->allocator; | |
345 | struct mlx4_zone_entry *curr_node; | |
346 | ||
347 | res = mlx4_bitmap_alloc_range(zone->bitmap, count, | |
348 | align, skip_mask); | |
349 | ||
350 | if (res != (u32)-1) { | |
351 | res += zone->offset; | |
352 | uid = zone->uid; | |
353 | goto out; | |
354 | } | |
355 | ||
356 | list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) { | |
357 | if (unlikely(curr_node->priority == zone->priority)) | |
358 | break; | |
359 | } | |
360 | ||
361 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) { | |
362 | struct mlx4_zone_entry *it = curr_node; | |
363 | ||
364 | list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) { | |
365 | res = mlx4_bitmap_alloc_range(it->bitmap, count, | |
366 | align, skip_mask); | |
367 | if (res != (u32)-1) { | |
368 | res += it->offset; | |
369 | uid = it->uid; | |
370 | goto out; | |
371 | } | |
372 | } | |
373 | } | |
374 | ||
375 | if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) { | |
376 | struct mlx4_zone_entry *it = curr_node; | |
377 | ||
378 | list_for_each_entry_from(it, &zone_alloc->entries, list) { | |
379 | if (unlikely(it == zone)) | |
380 | continue; | |
381 | ||
382 | if (unlikely(it->priority != curr_node->priority)) | |
383 | break; | |
384 | ||
385 | res = mlx4_bitmap_alloc_range(it->bitmap, count, | |
386 | align, skip_mask); | |
387 | if (res != (u32)-1) { | |
388 | res += it->offset; | |
389 | uid = it->uid; | |
390 | goto out; | |
391 | } | |
392 | } | |
393 | } | |
394 | ||
395 | if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) { | |
396 | if (list_is_last(&curr_node->prio_list, &zone_alloc->prios)) | |
397 | goto out; | |
398 | ||
399 | curr_node = list_first_entry(&curr_node->prio_list, | |
400 | typeof(*curr_node), | |
401 | prio_list); | |
402 | ||
403 | list_for_each_entry_from(curr_node, &zone_alloc->entries, list) { | |
404 | res = mlx4_bitmap_alloc_range(curr_node->bitmap, count, | |
405 | align, skip_mask); | |
406 | if (res != (u32)-1) { | |
407 | res += curr_node->offset; | |
408 | uid = curr_node->uid; | |
409 | goto out; | |
410 | } | |
411 | } | |
412 | } | |
413 | ||
414 | out: | |
415 | if (NULL != puid && res != (u32)-1) | |
416 | *puid = uid; | |
417 | return res; | |
418 | } | |
419 | ||
420 | /* Should be called under a lock */ | |
421 | static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj, | |
422 | u32 count) | |
423 | { | |
424 | mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr); | |
425 | } | |
426 | ||
427 | /* Should be called under a lock */ | |
428 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid( | |
429 | struct mlx4_zone_allocator *zones, u32 uid) | |
430 | { | |
431 | struct mlx4_zone_entry *zone; | |
432 | ||
433 | list_for_each_entry(zone, &zones->entries, list) { | |
434 | if (zone->uid == uid) | |
435 | return zone; | |
436 | } | |
437 | ||
438 | return NULL; | |
439 | } | |
440 | ||
441 | struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid) | |
442 | { | |
443 | struct mlx4_zone_entry *zone; | |
444 | struct mlx4_bitmap *bitmap; | |
445 | ||
446 | spin_lock(&zones->lock); | |
447 | ||
448 | zone = __mlx4_find_zone_by_uid(zones, uid); | |
449 | ||
450 | bitmap = zone == NULL ? NULL : zone->bitmap; | |
451 | ||
452 | spin_unlock(&zones->lock); | |
453 | ||
454 | return bitmap; | |
455 | } | |
456 | ||
457 | int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid) | |
458 | { | |
459 | struct mlx4_zone_entry *zone; | |
460 | int res; | |
461 | ||
462 | spin_lock(&zones->lock); | |
463 | ||
464 | zone = __mlx4_find_zone_by_uid(zones, uid); | |
465 | ||
466 | if (NULL == zone) { | |
467 | res = -1; | |
468 | goto out; | |
469 | } | |
470 | ||
471 | res = __mlx4_zone_remove_one_entry(zone); | |
472 | ||
473 | out: | |
474 | spin_unlock(&zones->lock); | |
475 | kfree(zone); | |
476 | ||
477 | return res; | |
478 | } | |
479 | ||
480 | /* Should be called under a lock */ | |
481 | static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique( | |
482 | struct mlx4_zone_allocator *zones, u32 obj) | |
483 | { | |
484 | struct mlx4_zone_entry *zone, *zone_candidate = NULL; | |
485 | u32 dist = (u32)-1; | |
486 | ||
487 | /* Search for the smallest zone that this obj could be | |
488 | * allocated from. This is done in order to handle | |
489 | * situations when small bitmaps are allocated from bigger | |
490 | * bitmaps (and the allocated space is marked as reserved in | |
491 | * the bigger bitmap. | |
492 | */ | |
493 | list_for_each_entry(zone, &zones->entries, list) { | |
494 | if (obj >= zone->offset) { | |
495 | u32 mobj = (obj - zone->offset) & zones->mask; | |
496 | ||
497 | if (mobj < zone->bitmap->max) { | |
498 | u32 curr_dist = zone->bitmap->effective_len; | |
499 | ||
500 | if (curr_dist < dist) { | |
501 | dist = curr_dist; | |
502 | zone_candidate = zone; | |
503 | } | |
504 | } | |
505 | } | |
506 | } | |
507 | ||
508 | return zone_candidate; | |
509 | } | |
510 | ||
511 | u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count, | |
512 | int align, u32 skip_mask, u32 *puid) | |
513 | { | |
514 | struct mlx4_zone_entry *zone; | |
515 | int res = -1; | |
516 | ||
517 | spin_lock(&zones->lock); | |
518 | ||
519 | zone = __mlx4_find_zone_by_uid(zones, uid); | |
520 | ||
521 | if (NULL == zone) | |
522 | goto out; | |
523 | ||
524 | res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid); | |
525 | ||
526 | out: | |
527 | spin_unlock(&zones->lock); | |
528 | ||
529 | return res; | |
530 | } | |
531 | ||
532 | u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count) | |
533 | { | |
534 | struct mlx4_zone_entry *zone; | |
535 | int res = 0; | |
536 | ||
537 | spin_lock(&zones->lock); | |
538 | ||
539 | zone = __mlx4_find_zone_by_uid(zones, uid); | |
540 | ||
541 | if (NULL == zone) { | |
542 | res = -1; | |
543 | goto out; | |
544 | } | |
545 | ||
546 | __mlx4_free_from_zone(zone, obj, count); | |
547 | ||
548 | out: | |
549 | spin_unlock(&zones->lock); | |
550 | ||
551 | return res; | |
552 | } | |
553 | ||
554 | u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count) | |
555 | { | |
556 | struct mlx4_zone_entry *zone; | |
557 | int res; | |
558 | ||
559 | if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP)) | |
560 | return -EFAULT; | |
561 | ||
562 | spin_lock(&zones->lock); | |
563 | ||
564 | zone = __mlx4_find_zone_by_uid_unique(zones, obj); | |
565 | ||
566 | if (NULL == zone) { | |
567 | res = -1; | |
568 | goto out; | |
569 | } | |
570 | ||
571 | __mlx4_free_from_zone(zone, obj, count); | |
572 | res = 0; | |
573 | ||
574 | out: | |
575 | spin_unlock(&zones->lock); | |
576 | ||
577 | return res; | |
578 | } | |
225c7b1f RD |
579 | /* |
580 | * Handling for queue buffers -- we allocate a bunch of memory and | |
581 | * register it in a memory region at HCA virtual address 0. If the | |
582 | * requested size is > max_direct, we split the allocation into | |
583 | * multiple pages, so we don't require too much contiguous memory. | |
584 | */ | |
585 | ||
586 | int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct, | |
40f2287b | 587 | struct mlx4_buf *buf, gfp_t gfp) |
225c7b1f RD |
588 | { |
589 | dma_addr_t t; | |
590 | ||
591 | if (size <= max_direct) { | |
592 | buf->nbufs = 1; | |
593 | buf->npages = 1; | |
594 | buf->page_shift = get_order(size) + PAGE_SHIFT; | |
b57aacfa | 595 | buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, |
40f2287b | 596 | size, &t, gfp); |
b57aacfa | 597 | if (!buf->direct.buf) |
225c7b1f RD |
598 | return -ENOMEM; |
599 | ||
b57aacfa | 600 | buf->direct.map = t; |
225c7b1f RD |
601 | |
602 | while (t & ((1 << buf->page_shift) - 1)) { | |
603 | --buf->page_shift; | |
604 | buf->npages *= 2; | |
605 | } | |
606 | ||
b57aacfa | 607 | memset(buf->direct.buf, 0, size); |
225c7b1f RD |
608 | } else { |
609 | int i; | |
610 | ||
030b4b33 | 611 | buf->direct.buf = NULL; |
225c7b1f RD |
612 | buf->nbufs = (size + PAGE_SIZE - 1) / PAGE_SIZE; |
613 | buf->npages = buf->nbufs; | |
614 | buf->page_shift = PAGE_SHIFT; | |
baeb2ffa | 615 | buf->page_list = kcalloc(buf->nbufs, sizeof(*buf->page_list), |
40f2287b | 616 | gfp); |
b57aacfa | 617 | if (!buf->page_list) |
225c7b1f RD |
618 | return -ENOMEM; |
619 | ||
620 | for (i = 0; i < buf->nbufs; ++i) { | |
b57aacfa | 621 | buf->page_list[i].buf = |
225c7b1f | 622 | dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, |
40f2287b | 623 | &t, gfp); |
b57aacfa | 624 | if (!buf->page_list[i].buf) |
225c7b1f RD |
625 | goto err_free; |
626 | ||
b57aacfa | 627 | buf->page_list[i].map = t; |
225c7b1f | 628 | |
b57aacfa | 629 | memset(buf->page_list[i].buf, 0, PAGE_SIZE); |
225c7b1f | 630 | } |
313abe55 JM |
631 | |
632 | if (BITS_PER_LONG == 64) { | |
633 | struct page **pages; | |
40f2287b | 634 | pages = kmalloc(sizeof *pages * buf->nbufs, gfp); |
313abe55 JM |
635 | if (!pages) |
636 | goto err_free; | |
637 | for (i = 0; i < buf->nbufs; ++i) | |
b57aacfa RD |
638 | pages[i] = virt_to_page(buf->page_list[i].buf); |
639 | buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL); | |
313abe55 | 640 | kfree(pages); |
b57aacfa | 641 | if (!buf->direct.buf) |
313abe55 JM |
642 | goto err_free; |
643 | } | |
225c7b1f RD |
644 | } |
645 | ||
646 | return 0; | |
647 | ||
648 | err_free: | |
649 | mlx4_buf_free(dev, size, buf); | |
650 | ||
651 | return -ENOMEM; | |
652 | } | |
653 | EXPORT_SYMBOL_GPL(mlx4_buf_alloc); | |
654 | ||
655 | void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf) | |
656 | { | |
657 | int i; | |
658 | ||
659 | if (buf->nbufs == 1) | |
b57aacfa RD |
660 | dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, |
661 | buf->direct.map); | |
225c7b1f | 662 | else { |
030b4b33 | 663 | if (BITS_PER_LONG == 64 && buf->direct.buf) |
b57aacfa | 664 | vunmap(buf->direct.buf); |
313abe55 | 665 | |
225c7b1f | 666 | for (i = 0; i < buf->nbufs; ++i) |
b57aacfa | 667 | if (buf->page_list[i].buf) |
3bba11e5 | 668 | dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, |
b57aacfa RD |
669 | buf->page_list[i].buf, |
670 | buf->page_list[i].map); | |
671 | kfree(buf->page_list); | |
225c7b1f RD |
672 | } |
673 | } | |
674 | EXPORT_SYMBOL_GPL(mlx4_buf_free); | |
6296883c | 675 | |
40f2287b JK |
676 | static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device, |
677 | gfp_t gfp) | |
6296883c YP |
678 | { |
679 | struct mlx4_db_pgdir *pgdir; | |
680 | ||
40f2287b | 681 | pgdir = kzalloc(sizeof *pgdir, gfp); |
6296883c YP |
682 | if (!pgdir) |
683 | return NULL; | |
684 | ||
685 | bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2); | |
686 | pgdir->bits[0] = pgdir->order0; | |
687 | pgdir->bits[1] = pgdir->order1; | |
688 | pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE, | |
40f2287b | 689 | &pgdir->db_dma, gfp); |
6296883c YP |
690 | if (!pgdir->db_page) { |
691 | kfree(pgdir); | |
692 | return NULL; | |
693 | } | |
694 | ||
695 | return pgdir; | |
696 | } | |
697 | ||
698 | static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir, | |
699 | struct mlx4_db *db, int order) | |
700 | { | |
701 | int o; | |
702 | int i; | |
703 | ||
704 | for (o = order; o <= 1; ++o) { | |
705 | i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o); | |
706 | if (i < MLX4_DB_PER_PAGE >> o) | |
707 | goto found; | |
708 | } | |
709 | ||
710 | return -ENOMEM; | |
711 | ||
712 | found: | |
713 | clear_bit(i, pgdir->bits[o]); | |
714 | ||
715 | i <<= o; | |
716 | ||
717 | if (o > order) | |
718 | set_bit(i ^ 1, pgdir->bits[order]); | |
719 | ||
720 | db->u.pgdir = pgdir; | |
721 | db->index = i; | |
722 | db->db = pgdir->db_page + db->index; | |
723 | db->dma = pgdir->db_dma + db->index * 4; | |
724 | db->order = order; | |
725 | ||
726 | return 0; | |
727 | } | |
728 | ||
40f2287b | 729 | int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp) |
6296883c YP |
730 | { |
731 | struct mlx4_priv *priv = mlx4_priv(dev); | |
732 | struct mlx4_db_pgdir *pgdir; | |
733 | int ret = 0; | |
734 | ||
735 | mutex_lock(&priv->pgdir_mutex); | |
736 | ||
737 | list_for_each_entry(pgdir, &priv->pgdir_list, list) | |
738 | if (!mlx4_alloc_db_from_pgdir(pgdir, db, order)) | |
739 | goto out; | |
740 | ||
40f2287b | 741 | pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp); |
6296883c YP |
742 | if (!pgdir) { |
743 | ret = -ENOMEM; | |
744 | goto out; | |
745 | } | |
746 | ||
747 | list_add(&pgdir->list, &priv->pgdir_list); | |
748 | ||
749 | /* This should never fail -- we just allocated an empty page: */ | |
750 | WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order)); | |
751 | ||
752 | out: | |
753 | mutex_unlock(&priv->pgdir_mutex); | |
754 | ||
755 | return ret; | |
756 | } | |
757 | EXPORT_SYMBOL_GPL(mlx4_db_alloc); | |
758 | ||
759 | void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db) | |
760 | { | |
761 | struct mlx4_priv *priv = mlx4_priv(dev); | |
762 | int o; | |
763 | int i; | |
764 | ||
765 | mutex_lock(&priv->pgdir_mutex); | |
766 | ||
767 | o = db->order; | |
768 | i = db->index; | |
769 | ||
770 | if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) { | |
771 | clear_bit(i ^ 1, db->u.pgdir->order0); | |
772 | ++o; | |
773 | } | |
774 | i >>= o; | |
775 | set_bit(i, db->u.pgdir->bits[o]); | |
776 | ||
777 | if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) { | |
778 | dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, | |
779 | db->u.pgdir->db_page, db->u.pgdir->db_dma); | |
780 | list_del(&db->u.pgdir->list); | |
781 | kfree(db->u.pgdir); | |
782 | } | |
783 | ||
784 | mutex_unlock(&priv->pgdir_mutex); | |
785 | } | |
786 | EXPORT_SYMBOL_GPL(mlx4_db_free); | |
38ae6a53 YP |
787 | |
788 | int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, | |
789 | int size, int max_direct) | |
790 | { | |
791 | int err; | |
792 | ||
40f2287b | 793 | err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL); |
38ae6a53 YP |
794 | if (err) |
795 | return err; | |
796 | ||
797 | *wqres->db.db = 0; | |
798 | ||
40f2287b | 799 | err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL); |
38ae6a53 YP |
800 | if (err) |
801 | goto err_db; | |
802 | ||
803 | err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift, | |
804 | &wqres->mtt); | |
805 | if (err) | |
806 | goto err_buf; | |
807 | ||
40f2287b | 808 | err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL); |
38ae6a53 YP |
809 | if (err) |
810 | goto err_mtt; | |
811 | ||
812 | return 0; | |
813 | ||
814 | err_mtt: | |
815 | mlx4_mtt_cleanup(dev, &wqres->mtt); | |
816 | err_buf: | |
817 | mlx4_buf_free(dev, size, &wqres->buf); | |
818 | err_db: | |
819 | mlx4_db_free(dev, &wqres->db); | |
820 | ||
821 | return err; | |
822 | } | |
823 | EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res); | |
824 | ||
825 | void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres, | |
826 | int size) | |
827 | { | |
828 | mlx4_mtt_cleanup(dev, &wqres->mtt); | |
829 | mlx4_buf_free(dev, size, &wqres->buf); | |
830 | mlx4_db_free(dev, &wqres->db); | |
831 | } | |
832 | EXPORT_SYMBOL_GPL(mlx4_free_hwq_res); |