Merge branch 'for-linus' of git://git.kernel.dk/linux-block

[deliverable/linux.git] / block / blk-mq-tag.c

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index e3d4e4043b496199d9f1eda8d83318cd1f9b0fe9..60c9d4a93fe470ced7471cd8653d8a00fc8922d7 100644 (file)
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -68,9 +68,9 @@ bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx)
 }
 
 /*
- * Wakeup all potentially sleeping on normal (non-reserved) tags
+ * Wakeup all potentially sleeping on tags
  */
-static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags)
+void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve)
 {
        struct blk_mq_bitmap_tags *bt;
        int i, wake_index;
@@ -85,6 +85,12 @@ static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags)
 
                wake_index = bt_index_inc(wake_index);
        }
+
+       if (include_reserve) {
+               bt = &tags->breserved_tags;
+               if (waitqueue_active(&bt->bs[0].wait))
+                       wake_up(&bt->bs[0].wait);
+       }
 }
 
 /*
@@ -100,7 +106,7 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx)
 
        atomic_dec(&tags->active_queues);
 
-       blk_mq_tag_wakeup_all(tags);
+       blk_mq_tag_wakeup_all(tags, false);
 }
 
 /*
@@ -248,8 +254,8 @@ static int bt_get(struct blk_mq_alloc_data *data,
        if (!(data->gfp & __GFP_WAIT))
                return -1;
 
+       bs = bt_wait_ptr(bt, hctx);
        do {
-               bs = bt_wait_ptr(bt, hctx);
                prepare_to_wait(&bs->wait, &wait, TASK_UNINTERRUPTIBLE);
 
                tag = __bt_get(hctx, bt, last_tag);
@@ -285,6 +291,8 @@ static int bt_get(struct blk_mq_alloc_data *data,
                        hctx = data->hctx;
                        bt = &hctx->tags->bitmap_tags;
                }
+               finish_wait(&bs->wait, &wait);
+               bs = bt_wait_ptr(bt, hctx);
        } while (1);
 
        finish_wait(&bs->wait, &wait);
@@ -582,7 +590,7 @@ int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth)
         * static and should never need resizing.
         */
        bt_update_count(&tags->bitmap_tags, tdepth);
-       blk_mq_tag_wakeup_all(tags);
+       blk_mq_tag_wakeup_all(tags, false);
        return 0;
 }