drivers/net/ethernet/mellanox/mlx4/alloc.c

   1 /*
   2  * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/errno.h>
  35 #include <linux/slab.h>
  36 #include <linux/mm.h>
  37 #include <linux/export.h>
  38 #include <linux/bitmap.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/vmalloc.h>
  41
  42 #include "mlx4.h"
  43
  44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)
  45 {
  46         u32 obj;
  47
  48         spin_lock(&bitmap->lock);
  49
  50         obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);
  51         if (obj >= bitmap->max) {
  52                 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
  53                                 & bitmap->mask;
  54                 obj = find_first_zero_bit(bitmap->table, bitmap->max);
  55         }
  56
  57         if (obj < bitmap->max) {
  58                 set_bit(obj, bitmap->table);
  59                 bitmap->last = (obj + 1);
  60                 if (bitmap->last == bitmap->max)
  61                         bitmap->last = 0;
  62                 obj |= bitmap->top;
  63         } else
  64                 obj = -1;
  65
  66         if (obj != -1)
  67                 --bitmap->avail;
  68
  69         spin_unlock(&bitmap->lock);
  70
  71         return obj;
  72 }
  73
  74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr)
  75 {
  76         mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);
  77 }
  78
  79 static unsigned long find_aligned_range(unsigned long *bitmap,
  80                                         u32 start, u32 nbits,
  81                                         int len, int align, u32 skip_mask)
  82 {
  83         unsigned long end, i;
  84
  85 again:
  86         start = ALIGN(start, align);
  87
  88         while ((start < nbits) && (test_bit(start, bitmap) ||
  89                                    (start & skip_mask)))
  90                 start += align;
  91
  92         if (start >= nbits)
  93                 return -1;
  94
  95         end = start+len;
  96         if (end > nbits)
  97                 return -1;
  98
  99         for (i = start + 1; i < end; i++) {
 100                 if (test_bit(i, bitmap) || ((u32)i & skip_mask)) {
 101                         start = i + 1;
 102                         goto again;
 103                 }
 104         }
 105
 106         return start;
 107 }
 108
 109 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,
 110                             int align, u32 skip_mask)
 111 {
 112         u32 obj;
 113
 114         if (likely(cnt == 1 && align == 1 && !skip_mask))
 115                 return mlx4_bitmap_alloc(bitmap);
 116
 117         spin_lock(&bitmap->lock);
 118
 119         obj = find_aligned_range(bitmap->table, bitmap->last,
 120                                  bitmap->max, cnt, align, skip_mask);
 121         if (obj >= bitmap->max) {
 122                 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
 123                                 & bitmap->mask;
 124                 obj = find_aligned_range(bitmap->table, 0, bitmap->max,
 125                                          cnt, align, skip_mask);
 126         }
 127
 128         if (obj < bitmap->max) {
 129                 bitmap_set(bitmap->table, obj, cnt);
 130                 if (obj == bitmap->last) {
 131                         bitmap->last = (obj + cnt);
 132                         if (bitmap->last >= bitmap->max)
 133                                 bitmap->last = 0;
 134                 }
 135                 obj |= bitmap->top;
 136         } else
 137                 obj = -1;
 138
 139         if (obj != -1)
 140                 bitmap->avail -= cnt;
 141
 142         spin_unlock(&bitmap->lock);
 143
 144         return obj;
 145 }
 146
 147 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)
 148 {
 149         return bitmap->avail;
 150 }
 151
 152 static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj)
 153 {
 154         return obj & (bitmap->max + bitmap->reserved_top - 1);
 155 }
 156
 157 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,
 158                             int use_rr)
 159 {
 160         obj &= bitmap->max + bitmap->reserved_top - 1;
 161
 162         spin_lock(&bitmap->lock);
 163         if (!use_rr) {
 164                 bitmap->last = min(bitmap->last, obj);
 165                 bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)
 166                                 & bitmap->mask;
 167         }
 168         bitmap_clear(bitmap->table, obj, cnt);
 169         bitmap->avail += cnt;
 170         spin_unlock(&bitmap->lock);
 171 }
 172
 173 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,
 174                      u32 reserved_bot, u32 reserved_top)
 175 {
 176         /* num must be a power of 2 */
 177         if (num != roundup_pow_of_two(num))
 178                 return -EINVAL;
 179
 180         bitmap->last = 0;
 181         bitmap->top  = 0;
 182         bitmap->max  = num - reserved_top;
 183         bitmap->mask = mask;
 184         bitmap->reserved_top = reserved_top;
 185         bitmap->avail = num - reserved_top - reserved_bot;
 186         bitmap->effective_len = bitmap->avail;
 187         spin_lock_init(&bitmap->lock);
 188         bitmap->table = kzalloc(BITS_TO_LONGS(bitmap->max) *
 189                                 sizeof (long), GFP_KERNEL);
 190         if (!bitmap->table)
 191                 return -ENOMEM;
 192
 193         bitmap_set(bitmap->table, 0, reserved_bot);
 194
 195         return 0;
 196 }
 197
 198 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)
 199 {
 200         kfree(bitmap->table);
 201 }
 202
 203 struct mlx4_zone_allocator {
 204         struct list_head                entries;
 205         struct list_head                prios;
 206         u32                             last_uid;
 207         u32                             mask;
 208         /* protect the zone_allocator from concurrent accesses */
 209         spinlock_t                      lock;
 210         enum mlx4_zone_alloc_flags      flags;
 211 };
 212
 213 struct mlx4_zone_entry {
 214         struct list_head                list;
 215         struct list_head                prio_list;
 216         u32                             uid;
 217         struct mlx4_zone_allocator      *allocator;
 218         struct mlx4_bitmap              *bitmap;
 219         int                             use_rr;
 220         int                             priority;
 221         int                             offset;
 222         enum mlx4_zone_flags            flags;
 223 };
 224
 225 struct mlx4_zone_allocator *mlx4_zone_allocator_create(enum mlx4_zone_alloc_flags flags)
 226 {
 227         struct mlx4_zone_allocator *zones = kmalloc(sizeof(*zones), GFP_KERNEL);
 228
 229         if (NULL == zones)
 230                 return NULL;
 231
 232         INIT_LIST_HEAD(&zones->entries);
 233         INIT_LIST_HEAD(&zones->prios);
 234         spin_lock_init(&zones->lock);
 235         zones->last_uid = 0;
 236         zones->mask = 0;
 237         zones->flags = flags;
 238
 239         return zones;
 240 }
 241
 242 int mlx4_zone_add_one(struct mlx4_zone_allocator *zone_alloc,
 243                       struct mlx4_bitmap *bitmap,
 244                       u32 flags,
 245                       int priority,
 246                       int offset,
 247                       u32 *puid)
 248 {
 249         u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1);
 250         struct mlx4_zone_entry *it;
 251         struct mlx4_zone_entry *zone = kmalloc(sizeof(*zone), GFP_KERNEL);
 252
 253         if (NULL == zone)
 254                 return -ENOMEM;
 255
 256         zone->flags = flags;
 257         zone->bitmap = bitmap;
 258         zone->use_rr = (flags & MLX4_ZONE_USE_RR) ? MLX4_USE_RR : 0;
 259         zone->priority = priority;
 260         zone->offset = offset;
 261
 262         spin_lock(&zone_alloc->lock);
 263
 264         zone->uid = zone_alloc->last_uid++;
 265         zone->allocator = zone_alloc;
 266
 267         if (zone_alloc->mask < mask)
 268                 zone_alloc->mask = mask;
 269
 270         list_for_each_entry(it, &zone_alloc->prios, prio_list)
 271                 if (it->priority >= priority)
 272                         break;
 273
 274         if (&it->prio_list == &zone_alloc->prios || it->priority > priority)
 275                 list_add_tail(&zone->prio_list, &it->prio_list);
 276         list_add_tail(&zone->list, &it->list);
 277
 278         spin_unlock(&zone_alloc->lock);
 279
 280         *puid = zone->uid;
 281
 282         return 0;
 283 }
 284
 285 /* Should be called under a lock */
 286 static int __mlx4_zone_remove_one_entry(struct mlx4_zone_entry *entry)
 287 {
 288         struct mlx4_zone_allocator *zone_alloc = entry->allocator;
 289
 290         if (!list_empty(&entry->prio_list)) {
 291                 /* Check if we need to add an alternative node to the prio list */
 292                 if (!list_is_last(&entry->list, &zone_alloc->entries)) {
 293                         struct mlx4_zone_entry *next = list_first_entry(&entry->list,
 294                                                                         typeof(*next),
 295                                                                         list);
 296
 297                         if (next->priority == entry->priority)
 298                                 list_add_tail(&next->prio_list, &entry->prio_list);
 299                 }
 300
 301                 list_del(&entry->prio_list);
 302         }
 303
 304         list_del(&entry->list);
 305
 306         if (zone_alloc->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP) {
 307                 u32 mask = 0;
 308                 struct mlx4_zone_entry *it;
 309
 310                 list_for_each_entry(it, &zone_alloc->prios, prio_list) {
 311                         u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1);
 312
 313                         if (mask < cur_mask)
 314                                 mask = cur_mask;
 315                 }
 316                 zone_alloc->mask = mask;
 317         }
 318
 319         return 0;
 320 }
 321
 322 void mlx4_zone_allocator_destroy(struct mlx4_zone_allocator *zone_alloc)
 323 {
 324         struct mlx4_zone_entry *zone, *tmp;
 325
 326         spin_lock(&zone_alloc->lock);
 327
 328         list_for_each_entry_safe(zone, tmp, &zone_alloc->entries, list) {
 329                 list_del(&zone->list);
 330                 list_del(&zone->prio_list);
 331                 kfree(zone);
 332         }
 333
 334         spin_unlock(&zone_alloc->lock);
 335         kfree(zone_alloc);
 336 }
 337
 338 /* Should be called under a lock */
 339 static u32 __mlx4_alloc_from_zone(struct mlx4_zone_entry *zone, int count,
 340                                   int align, u32 skip_mask, u32 *puid)
 341 {
 342         u32 uid;
 343         u32 res;
 344         struct mlx4_zone_allocator *zone_alloc = zone->allocator;
 345         struct mlx4_zone_entry *curr_node;
 346
 347         res = mlx4_bitmap_alloc_range(zone->bitmap, count,
 348                                       align, skip_mask);
 349
 350         if (res != (u32)-1) {
 351                 res += zone->offset;
 352                 uid = zone->uid;
 353                 goto out;
 354         }
 355
 356         list_for_each_entry(curr_node, &zone_alloc->prios, prio_list) {
 357                 if (unlikely(curr_node->priority == zone->priority))
 358                         break;
 359         }
 360
 361         if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO) {
 362                 struct mlx4_zone_entry *it = curr_node;
 363
 364                 list_for_each_entry_continue_reverse(it, &zone_alloc->entries, list) {
 365                         res = mlx4_bitmap_alloc_range(it->bitmap, count,
 366                                                       align, skip_mask);
 367                         if (res != (u32)-1) {
 368                                 res += it->offset;
 369                                 uid = it->uid;
 370                                 goto out;
 371                         }
 372                 }
 373         }
 374
 375         if (zone->flags & MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO) {
 376                 struct mlx4_zone_entry *it = curr_node;
 377
 378                 list_for_each_entry_from(it, &zone_alloc->entries, list) {
 379                         if (unlikely(it == zone))
 380                                 continue;
 381
 382                         if (unlikely(it->priority != curr_node->priority))
 383                                 break;
 384
 385                         res = mlx4_bitmap_alloc_range(it->bitmap, count,
 386                                                       align, skip_mask);
 387                         if (res != (u32)-1) {
 388                                 res += it->offset;
 389                                 uid = it->uid;
 390                                 goto out;
 391                         }
 392                 }
 393         }
 394
 395         if (zone->flags & MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO) {
 396                 if (list_is_last(&curr_node->prio_list, &zone_alloc->prios))
 397                         goto out;
 398
 399                 curr_node = list_first_entry(&curr_node->prio_list,
 400                                              typeof(*curr_node),
 401                                              prio_list);
 402
 403                 list_for_each_entry_from(curr_node, &zone_alloc->entries, list) {
 404                         res = mlx4_bitmap_alloc_range(curr_node->bitmap, count,
 405                                                       align, skip_mask);
 406                         if (res != (u32)-1) {
 407                                 res += curr_node->offset;
 408                                 uid = curr_node->uid;
 409                                 goto out;
 410                         }
 411                 }
 412         }
 413
 414 out:
 415         if (NULL != puid && res != (u32)-1)
 416                 *puid = uid;
 417         return res;
 418 }
 419
 420 /* Should be called under a lock */
 421 static void __mlx4_free_from_zone(struct mlx4_zone_entry *zone, u32 obj,
 422                                   u32 count)
 423 {
 424         mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr);
 425 }
 426
 427 /* Should be called under a lock */
 428 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid(
 429                 struct mlx4_zone_allocator *zones, u32 uid)
 430 {
 431         struct mlx4_zone_entry *zone;
 432
 433         list_for_each_entry(zone, &zones->entries, list) {
 434                 if (zone->uid == uid)
 435                         return zone;
 436         }
 437
 438         return NULL;
 439 }
 440
 441 struct mlx4_bitmap *mlx4_zone_get_bitmap(struct mlx4_zone_allocator *zones, u32 uid)
 442 {
 443         struct mlx4_zone_entry *zone;
 444         struct mlx4_bitmap *bitmap;
 445
 446         spin_lock(&zones->lock);
 447
 448         zone = __mlx4_find_zone_by_uid(zones, uid);
 449
 450         bitmap = zone == NULL ? NULL : zone->bitmap;
 451
 452         spin_unlock(&zones->lock);
 453
 454         return bitmap;
 455 }
 456
 457 int mlx4_zone_remove_one(struct mlx4_zone_allocator *zones, u32 uid)
 458 {
 459         struct mlx4_zone_entry *zone;
 460         int res;
 461
 462         spin_lock(&zones->lock);
 463
 464         zone = __mlx4_find_zone_by_uid(zones, uid);
 465
 466         if (NULL == zone) {
 467                 res = -1;
 468                 goto out;
 469         }
 470
 471         res = __mlx4_zone_remove_one_entry(zone);
 472
 473 out:
 474         spin_unlock(&zones->lock);
 475         kfree(zone);
 476
 477         return res;
 478 }
 479
 480 /* Should be called under a lock */
 481 static struct mlx4_zone_entry *__mlx4_find_zone_by_uid_unique(
 482                 struct mlx4_zone_allocator *zones, u32 obj)
 483 {
 484         struct mlx4_zone_entry *zone, *zone_candidate = NULL;
 485         u32 dist = (u32)-1;
 486
 487         /* Search for the smallest zone that this obj could be
 488          * allocated from. This is done in order to handle
 489          * situations when small bitmaps are allocated from bigger
 490          * bitmaps (and the allocated space is marked as reserved in
 491          * the bigger bitmap.
 492          */
 493         list_for_each_entry(zone, &zones->entries, list) {
 494                 if (obj >= zone->offset) {
 495                         u32 mobj = (obj - zone->offset) & zones->mask;
 496
 497                         if (mobj < zone->bitmap->max) {
 498                                 u32 curr_dist = zone->bitmap->effective_len;
 499
 500                                 if (curr_dist < dist) {
 501                                         dist = curr_dist;
 502                                         zone_candidate = zone;
 503                                 }
 504                         }
 505                 }
 506         }
 507
 508         return zone_candidate;
 509 }
 510
 511 u32 mlx4_zone_alloc_entries(struct mlx4_zone_allocator *zones, u32 uid, int count,
 512                             int align, u32 skip_mask, u32 *puid)
 513 {
 514         struct mlx4_zone_entry *zone;
 515         int res = -1;
 516
 517         spin_lock(&zones->lock);
 518
 519         zone = __mlx4_find_zone_by_uid(zones, uid);
 520
 521         if (NULL == zone)
 522                 goto out;
 523
 524         res = __mlx4_alloc_from_zone(zone, count, align, skip_mask, puid);
 525
 526 out:
 527         spin_unlock(&zones->lock);
 528
 529         return res;
 530 }
 531
 532 u32 mlx4_zone_free_entries(struct mlx4_zone_allocator *zones, u32 uid, u32 obj, u32 count)
 533 {
 534         struct mlx4_zone_entry *zone;
 535         int res = 0;
 536
 537         spin_lock(&zones->lock);
 538
 539         zone = __mlx4_find_zone_by_uid(zones, uid);
 540
 541         if (NULL == zone) {
 542                 res = -1;
 543                 goto out;
 544         }
 545
 546         __mlx4_free_from_zone(zone, obj, count);
 547
 548 out:
 549         spin_unlock(&zones->lock);
 550
 551         return res;
 552 }
 553
 554 u32 mlx4_zone_free_entries_unique(struct mlx4_zone_allocator *zones, u32 obj, u32 count)
 555 {
 556         struct mlx4_zone_entry *zone;
 557         int res;
 558
 559         if (!(zones->flags & MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP))
 560                 return -EFAULT;
 561
 562         spin_lock(&zones->lock);
 563
 564         zone = __mlx4_find_zone_by_uid_unique(zones, obj);
 565
 566         if (NULL == zone) {
 567                 res = -1;
 568                 goto out;
 569         }
 570
 571         __mlx4_free_from_zone(zone, obj, count);
 572         res = 0;
 573
 574 out:
 575         spin_unlock(&zones->lock);
 576
 577         return res;
 578 }
 579 /*
 580  * Handling for queue buffers -- we allocate a bunch of memory and
 581  * register it in a memory region at HCA virtual address 0.  If the
 582  * requested size is > max_direct, we split the allocation into
 583  * multiple pages, so we don't require too much contiguous memory.
 584  */
 585
 586 int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
 587                    struct mlx4_buf *buf, gfp_t gfp)
 588 {
 589         dma_addr_t t;
 590
 591         if (size <= max_direct) {
 592                 buf->nbufs        = 1;
 593                 buf->npages       = 1;
 594                 buf->page_shift   = get_order(size) + PAGE_SHIFT;
 595                 buf->direct.buf   = dma_alloc_coherent(&dev->pdev->dev,
 596                                                        size, &t, gfp);
 597                 if (!buf->direct.buf)
 598                         return -ENOMEM;
 599
 600                 buf->direct.map = t;
 601
 602                 while (t & ((1 << buf->page_shift) - 1)) {
 603                         --buf->page_shift;
 604                         buf->npages *= 2;
 605                 }
 606
 607                 memset(buf->direct.buf, 0, size);
 608         } else {
 609                 int i;
 610
 611                 buf->direct.buf  = NULL;
 612                 buf->nbufs       = (size + PAGE_SIZE - 1) / PAGE_SIZE;
 613                 buf->npages      = buf->nbufs;
 614                 buf->page_shift  = PAGE_SHIFT;
 615                 buf->page_list   = kcalloc(buf->nbufs, sizeof(*buf->page_list),
 616                                            gfp);
 617                 if (!buf->page_list)
 618                         return -ENOMEM;
 619
 620                 for (i = 0; i < buf->nbufs; ++i) {
 621                         buf->page_list[i].buf =
 622                                 dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
 623                                                    &t, gfp);
 624                         if (!buf->page_list[i].buf)
 625                                 goto err_free;
 626
 627                         buf->page_list[i].map = t;
 628
 629                         memset(buf->page_list[i].buf, 0, PAGE_SIZE);
 630                 }
 631
 632                 if (BITS_PER_LONG == 64) {
 633                         struct page **pages;
 634                         pages = kmalloc(sizeof *pages * buf->nbufs, gfp);
 635                         if (!pages)
 636                                 goto err_free;
 637                         for (i = 0; i < buf->nbufs; ++i)
 638                                 pages[i] = virt_to_page(buf->page_list[i].buf);
 639                         buf->direct.buf = vmap(pages, buf->nbufs, VM_MAP, PAGE_KERNEL);
 640                         kfree(pages);
 641                         if (!buf->direct.buf)
 642                                 goto err_free;
 643                 }
 644         }
 645
 646         return 0;
 647
 648 err_free:
 649         mlx4_buf_free(dev, size, buf);
 650
 651         return -ENOMEM;
 652 }
 653 EXPORT_SYMBOL_GPL(mlx4_buf_alloc);
 654
 655 void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf)
 656 {
 657         int i;
 658
 659         if (buf->nbufs == 1)
 660                 dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf,
 661                                   buf->direct.map);
 662         else {
 663                 if (BITS_PER_LONG == 64 && buf->direct.buf)
 664                         vunmap(buf->direct.buf);
 665
 666                 for (i = 0; i < buf->nbufs; ++i)
 667                         if (buf->page_list[i].buf)
 668                                 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
 669                                                   buf->page_list[i].buf,
 670                                                   buf->page_list[i].map);
 671                 kfree(buf->page_list);
 672         }
 673 }
 674 EXPORT_SYMBOL_GPL(mlx4_buf_free);
 675
 676 static struct mlx4_db_pgdir *mlx4_alloc_db_pgdir(struct device *dma_device,
 677                                                  gfp_t gfp)
 678 {
 679         struct mlx4_db_pgdir *pgdir;
 680
 681         pgdir = kzalloc(sizeof *pgdir, gfp);
 682         if (!pgdir)
 683                 return NULL;
 684
 685         bitmap_fill(pgdir->order1, MLX4_DB_PER_PAGE / 2);
 686         pgdir->bits[0] = pgdir->order0;
 687         pgdir->bits[1] = pgdir->order1;
 688         pgdir->db_page = dma_alloc_coherent(dma_device, PAGE_SIZE,
 689                                             &pgdir->db_dma, gfp);
 690         if (!pgdir->db_page) {
 691                 kfree(pgdir);
 692                 return NULL;
 693         }
 694
 695         return pgdir;
 696 }
 697
 698 static int mlx4_alloc_db_from_pgdir(struct mlx4_db_pgdir *pgdir,
 699                                     struct mlx4_db *db, int order)
 700 {
 701         int o;
 702         int i;
 703
 704         for (o = order; o <= 1; ++o) {
 705                 i = find_first_bit(pgdir->bits[o], MLX4_DB_PER_PAGE >> o);
 706                 if (i < MLX4_DB_PER_PAGE >> o)
 707                         goto found;
 708         }
 709
 710         return -ENOMEM;
 711
 712 found:
 713         clear_bit(i, pgdir->bits[o]);
 714
 715         i <<= o;
 716
 717         if (o > order)
 718                 set_bit(i ^ 1, pgdir->bits[order]);
 719
 720         db->u.pgdir = pgdir;
 721         db->index   = i;
 722         db->db      = pgdir->db_page + db->index;
 723         db->dma     = pgdir->db_dma  + db->index * 4;
 724         db->order   = order;
 725
 726         return 0;
 727 }
 728
 729 int mlx4_db_alloc(struct mlx4_dev *dev, struct mlx4_db *db, int order, gfp_t gfp)
 730 {
 731         struct mlx4_priv *priv = mlx4_priv(dev);
 732         struct mlx4_db_pgdir *pgdir;
 733         int ret = 0;
 734
 735         mutex_lock(&priv->pgdir_mutex);
 736
 737         list_for_each_entry(pgdir, &priv->pgdir_list, list)
 738                 if (!mlx4_alloc_db_from_pgdir(pgdir, db, order))
 739                         goto out;
 740
 741         pgdir = mlx4_alloc_db_pgdir(&(dev->pdev->dev), gfp);
 742         if (!pgdir) {
 743                 ret = -ENOMEM;
 744                 goto out;
 745         }
 746
 747         list_add(&pgdir->list, &priv->pgdir_list);
 748
 749         /* This should never fail -- we just allocated an empty page: */
 750         WARN_ON(mlx4_alloc_db_from_pgdir(pgdir, db, order));
 751
 752 out:
 753         mutex_unlock(&priv->pgdir_mutex);
 754
 755         return ret;
 756 }
 757 EXPORT_SYMBOL_GPL(mlx4_db_alloc);
 758
 759 void mlx4_db_free(struct mlx4_dev *dev, struct mlx4_db *db)
 760 {
 761         struct mlx4_priv *priv = mlx4_priv(dev);
 762         int o;
 763         int i;
 764
 765         mutex_lock(&priv->pgdir_mutex);
 766
 767         o = db->order;
 768         i = db->index;
 769
 770         if (db->order == 0 && test_bit(i ^ 1, db->u.pgdir->order0)) {
 771                 clear_bit(i ^ 1, db->u.pgdir->order0);
 772                 ++o;
 773         }
 774         i >>= o;
 775         set_bit(i, db->u.pgdir->bits[o]);
 776
 777         if (bitmap_full(db->u.pgdir->order1, MLX4_DB_PER_PAGE / 2)) {
 778                 dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
 779                                   db->u.pgdir->db_page, db->u.pgdir->db_dma);
 780                 list_del(&db->u.pgdir->list);
 781                 kfree(db->u.pgdir);
 782         }
 783
 784         mutex_unlock(&priv->pgdir_mutex);
 785 }
 786 EXPORT_SYMBOL_GPL(mlx4_db_free);
 787
 788 int mlx4_alloc_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
 789                        int size, int max_direct)
 790 {
 791         int err;
 792
 793         err = mlx4_db_alloc(dev, &wqres->db, 1, GFP_KERNEL);
 794         if (err)
 795                 return err;
 796
 797         *wqres->db.db = 0;
 798
 799         err = mlx4_buf_alloc(dev, size, max_direct, &wqres->buf, GFP_KERNEL);
 800         if (err)
 801                 goto err_db;
 802
 803         err = mlx4_mtt_init(dev, wqres->buf.npages, wqres->buf.page_shift,
 804                             &wqres->mtt);
 805         if (err)
 806                 goto err_buf;
 807
 808         err = mlx4_buf_write_mtt(dev, &wqres->mtt, &wqres->buf, GFP_KERNEL);
 809         if (err)
 810                 goto err_mtt;
 811
 812         return 0;
 813
 814 err_mtt:
 815         mlx4_mtt_cleanup(dev, &wqres->mtt);
 816 err_buf:
 817         mlx4_buf_free(dev, size, &wqres->buf);
 818 err_db:
 819         mlx4_db_free(dev, &wqres->db);
 820
 821         return err;
 822 }
 823 EXPORT_SYMBOL_GPL(mlx4_alloc_hwq_res);
 824
 825 void mlx4_free_hwq_res(struct mlx4_dev *dev, struct mlx4_hwq_resources *wqres,
 826                        int size)
 827 {
 828         mlx4_mtt_cleanup(dev, &wqres->mtt);
 829         mlx4_buf_free(dev, size, &wqres->buf);
 830         mlx4_db_free(dev, &wqres->db);
 831 }
 832 EXPORT_SYMBOL_GPL(mlx4_free_hwq_res);