drbd: Reworked the unconfiguring and thread stopping code

* Moved CONFIG_PENDING and DEVICE_DYING from mdev to tconn.
* Renamed drbd_reconfig_start() and drbd_reconfig_done() to
  conn_reconfig_start() and conn_reconfig_done().

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index a346eb87d0fbc9fc214c9eea765e7d72489dd59c..145ae57b3113dc39b648ade47316c8ab65db97a9 100644 (file)
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -787,12 +787,6 @@ enum {
        GO_DISKLESS,            /* Disk is being detached, on io-error or admin request. */
        WAS_IO_ERROR,           /* Local disk failed returned IO error */
        RESYNC_AFTER_NEG,       /* Resync after online grow after the attach&negotiate finished. */
-       CONFIG_PENDING,         /* serialization of (re)configuration requests.
-                                * if set, also prevents the device from dying */
-       DEVICE_DYING,           /* device became unconfigured,
-                                * but worker thread is still handling the cleanup.
-                                * reconfiguring (nl_disk_conf, nl_net_conf) is dissalowed,
-                                * while this is set. */
        RESIZE_PENDING,         /* Size change detected locally, waiting for the response from
                                 * the peer, if it changed there as well. */
        CONN_DRY_RUN,           /* Expect disconnect after resync handshake. */
@@ -921,6 +915,12 @@ enum {
        GOT_PING_ACK,           /* set when we receive a ping_ack packet, ping_wait gets woken */
        CONN_WD_ST_CHG_OKAY,
        CONN_WD_ST_CHG_FAIL,
+       CONFIG_PENDING,         /* serialization of (re)configuration requests.
+                                * if set, also prevents the device from dying */
+       OBJECT_DYING,           /* device became unconfigured,
+                                * but worker thread is still handling the cleanup.
+                                * reconfiguring (nl_disk_conf, nl_net_conf) is dissalowed,
+                                * while this is set. */
 };
 
 struct drbd_tconn {                    /* is a resource from the config file */
@@ -1574,7 +1574,11 @@ extern void _drbd_wait_ee_list_empty(struct drbd_conf *mdev,
                struct list_head *head);
 extern void drbd_set_recv_tcq(struct drbd_conf *mdev, int tcq_enabled);
 extern void _drbd_clear_done_ee(struct drbd_conf *mdev, struct list_head *to_be_freed);
-extern void drbd_flush_workqueue(struct drbd_conf *mdev);
+extern void conn_flush_workqueue(struct drbd_tconn *tconn);
+static inline void drbd_flush_workqueue(struct drbd_conf *mdev)
+{
+       conn_flush_workqueue(mdev->tconn);
+}
 
 /* yes, there is kernel_setsockopt, but only since 2.6.18. we don't need to
  * mess with get_fs/set_fs, we know we are KERNEL_DS always. */

diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 016858741cfd5c1c55d6dfd74f72798f00f6ea01..8cdfb46243e2191605d55907dc93e36b3507a364 100644 (file)
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -871,29 +871,27 @@ void drbd_reconsider_max_bio_size(struct drbd_conf *mdev)
  * or start a new one.  Flush any pending work, there may still be an
  * after_state_change queued.
  */
-static void drbd_reconfig_start(struct drbd_conf *mdev)
+static void conn_reconfig_start(struct drbd_tconn *tconn)
 {
-       wait_event(mdev->state_wait, !test_and_set_bit(CONFIG_PENDING, &mdev->flags));
-       wait_event(mdev->state_wait, !test_bit(DEVICE_DYING, &mdev->flags));
-       drbd_thread_start(&mdev->tconn->worker);
-       drbd_flush_workqueue(mdev);
+       wait_event(tconn->ping_wait, !test_and_set_bit(CONFIG_PENDING, &tconn->flags));
+       wait_event(tconn->ping_wait, !test_bit(OBJECT_DYING, &tconn->flags));
+       drbd_thread_start(&tconn->worker);
+       conn_flush_workqueue(tconn);
 }
 
 /* if still unconfigured, stops worker again.
  * if configured now, clears CONFIG_PENDING.
  * wakes potential waiters */
-static void drbd_reconfig_done(struct drbd_conf *mdev)
+static void conn_reconfig_done(struct drbd_tconn *tconn)
 {
-       spin_lock_irq(&mdev->tconn->req_lock);
-       if (mdev->state.disk == D_DISKLESS &&
-           mdev->state.conn == C_STANDALONE &&
-           mdev->state.role == R_SECONDARY) {
-               set_bit(DEVICE_DYING, &mdev->flags);
-               drbd_thread_stop_nowait(&mdev->tconn->worker);
+       spin_lock_irq(&tconn->req_lock);
+       if (conn_all_vols_unconf(tconn)) {
+               set_bit(OBJECT_DYING, &tconn->flags);
+               drbd_thread_stop_nowait(&tconn->worker);
        } else
-               clear_bit(CONFIG_PENDING, &mdev->flags);
-       spin_unlock_irq(&mdev->tconn->req_lock);
-       wake_up(&mdev->state_wait);
+               clear_bit(CONFIG_PENDING, &tconn->flags);
+       spin_unlock_irq(&tconn->req_lock);
+       wake_up(&tconn->ping_wait);
 }
 
 /* Make sure IO is suspended before calling this function(). */
@@ -933,7 +931,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
        enum drbd_state_rv rv;
        int cp_discovered = 0;
 
-       drbd_reconfig_start(mdev);
+       conn_reconfig_start(mdev->tconn);
 
        /* if you want to reconfigure, please tear down first */
        if (mdev->state.disk > D_DISKLESS) {
@@ -1279,7 +1277,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
        kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
        put_ldev(mdev);
        reply->ret_code = retcode;
-       drbd_reconfig_done(mdev);
+       conn_reconfig_done(mdev->tconn);
        return 0;
 
  force_diskless_dec:
@@ -1300,7 +1298,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
        lc_destroy(resync_lru);
 
        reply->ret_code = retcode;
-       drbd_reconfig_done(mdev);
+       conn_reconfig_done(mdev->tconn);
        return 0;
 }
 
@@ -1344,7 +1342,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
        void *int_dig_vv = NULL;
        struct sockaddr *new_my_addr, *new_peer_addr, *taken_addr;
 
-       drbd_reconfig_start(mdev);
+       conn_reconfig_start(mdev->tconn);
 
        if (mdev->state.conn > C_STANDALONE) {
                retcode = ERR_NET_CONFIGURED;
@@ -1530,7 +1528,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 
        kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
        reply->ret_code = retcode;
-       drbd_reconfig_done(mdev);
+       conn_reconfig_done(mdev->tconn);
        return 0;
 
 fail:
@@ -1543,7 +1541,7 @@ fail:
        kfree(new_conf);
 
        reply->ret_code = retcode;
-       drbd_reconfig_done(mdev);
+       conn_reconfig_done(mdev->tconn);
        return 0;
 }
 

diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 0c3a094f69119d46ad3f62d99df2ccc5fecd0b94..66080e204086b7d020a290e8ed4b8132883d7f5f 100644 (file)
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3932,14 +3932,14 @@ static void drbdd(struct drbd_tconn *tconn)
        }
 }
 
-void drbd_flush_workqueue(struct drbd_conf *mdev)
+void conn_flush_workqueue(struct drbd_tconn *tconn)
 {
        struct drbd_wq_barrier barr;
 
        barr.w.cb = w_prev_work_done;
-       barr.w.mdev = mdev;
+       barr.w.tconn = tconn;
        init_completion(&barr.done);
-       drbd_queue_work(&mdev->tconn->data.work, &barr.w);
+       drbd_queue_work(&tconn->data.work, &barr.w);
        wait_for_completion(&barr.done);
 }
 

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index d3bf8e39fa5675caaf94cd3c446d9d0e03a5c63b..338e1f5c7cd0b597cc9f70e74efdc5079299d01a 100644 (file)
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -41,13 +41,29 @@ extern void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what);
 static int w_after_state_ch(struct drbd_work *w, int unused);
 static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
                           union drbd_state ns, enum chg_state_flags flags);
-static void after_all_state_ch(struct drbd_tconn *tconn, union drbd_state ns);
+static void after_all_state_ch(struct drbd_tconn *tconn);
 static enum drbd_state_rv is_valid_state(struct drbd_conf *, union drbd_state);
 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state);
 static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
 static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
                                       const char **warn_sync_abort);
 
+int conn_all_vols_unconf(struct drbd_tconn *tconn)
+{
+       struct drbd_conf *mdev;
+       int minor, uncfg = 1;
+
+       idr_for_each_entry(&tconn->volumes, mdev, minor) {
+               uncfg &= (mdev->state.disk == D_DISKLESS &&
+                         mdev->state.conn == C_STANDALONE &&
+                         mdev->state.role == R_SECONDARY);
+               if (!uncfg)
+                       break;
+       }
+
+       return uncfg;
+}
+
 /**
  * cl_wide_st_chg() - true if the state change is a cluster wide one
  * @mdev:      DRBD device.
@@ -744,20 +760,6 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 
        print_state_change(mdev, os, ns, flags);
 
-       /* solve the race between becoming unconfigured,
-        * worker doing the cleanup, and
-        * admin reconfiguring us:
-        * on (re)configure, first set CONFIG_PENDING,
-        * then wait for a potentially exiting worker,
-        * start the worker, and schedule one no_op.
-        * then proceed with configuration.
-        */
-       if (ns.disk == D_DISKLESS &&
-           ns.conn == C_STANDALONE &&
-           ns.role == R_SECONDARY &&
-           !test_and_set_bit(CONFIG_PENDING, &mdev->flags))
-               set_bit(DEVICE_DYING, &mdev->flags);
-
        /* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
         * on the ldev here, to be sure the transition -> D_DISKLESS resp.
         * drbd_ldev_destroy() won't happen before our corresponding
@@ -768,6 +770,18 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 
        mdev->state = ns;
 
+       /* solve the race between becoming unconfigured,
+        * worker doing the cleanup, and
+        * admin reconfiguring us:
+        * on (re)configure, first set CONFIG_PENDING,
+        * then wait for a potentially exiting worker,
+        * start the worker, and schedule one no_op.
+        * then proceed with configuration.
+        */
+       if(conn_all_vols_unconf(mdev->tconn) &&
+          !test_and_set_bit(CONFIG_PENDING, &mdev->tconn->flags))
+               set_bit(OBJECT_DYING, &mdev->tconn->flags);
+
        if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING)
                drbd_print_uuids(mdev, "attached to UUIDs");
 
@@ -1236,7 +1250,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
                        resume_next_sg(mdev);
        }
 
-       after_all_state_ch(mdev->tconn, ns);
+       after_all_state_ch(mdev->tconn);
 
        drbd_md_sync(mdev);
 }
@@ -1248,10 +1262,10 @@ struct after_conn_state_chg_work {
        enum chg_state_flags flags;
 };
 
-static void after_all_state_ch(struct drbd_tconn *tconn, union drbd_state ns)
+static void after_all_state_ch(struct drbd_tconn *tconn)
 {
-       if (ns.disk == D_DISKLESS && ns.conn == C_STANDALONE && ns.role == R_SECONDARY) {
-               /* if (test_bit(DEVICE_DYING, &mdev->flags)) TODO: DEVICE_DYING functionality */
+       if (conn_all_vols_unconf(tconn) &&
+           test_bit(OBJECT_DYING, &tconn->flags)) {
                drbd_thread_stop_nowait(&tconn->worker);
        }
 }
@@ -1271,7 +1285,7 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
                drbd_thread_start(&tconn->receiver);
 
        //conn_err(tconn, STATE_FMT, STATE_ARGS("nms", nms));
-       after_all_state_ch(tconn, nms);
+       after_all_state_ch(tconn);
 
        return 1;
 }

diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 5fdbdf0be707d4762d41c952a1e29d93c0b189a9..d9536cd798e53b27c3d21dface69b71f24de5de5 100644 (file)
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -91,6 +91,7 @@ conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_s
                   enum chg_state_flags flags);
 
 extern void drbd_resume_al(struct drbd_conf *mdev);
+extern int conn_all_vols_unconf(struct drbd_tconn *tconn);
 
 /**
  * drbd_request_state() - Reqest a state change

diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 8ee5c4f3d1c553a23bc57b5f3e46733166db8b5b..5cb5ffce097c010d3194f1b0df01fb62d87e8399 100644 (file)
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -1643,29 +1643,13 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
        mutex_unlock(mdev->state_mutex);
 }
 
-static int _worker_dying(int vnr, void *p, void *data)
-{
-       struct drbd_conf *mdev = (struct drbd_conf *)p;
-
-       D_ASSERT(mdev->state.disk == D_DISKLESS && mdev->state.conn == C_STANDALONE);
-       /* _drbd_set_state only uses stop_nowait.
-        * wait here for the exiting receiver. */
-       drbd_thread_stop(&mdev->tconn->receiver);
-       drbd_mdev_cleanup(mdev);
-
-       clear_bit(DEVICE_DYING, &mdev->flags);
-       clear_bit(CONFIG_PENDING, &mdev->flags);
-       wake_up(&mdev->state_wait);
-
-       return 0;
-}
-
 int drbd_worker(struct drbd_thread *thi)
 {
        struct drbd_tconn *tconn = thi->tconn;
        struct drbd_work *w = NULL;
+       struct drbd_conf *mdev;
        LIST_HEAD(work_list);
-       int intr = 0;
+       int minor, intr = 0;
 
        while (get_t_state(thi) == RUNNING) {
                drbd_thread_current_set_cpu(thi);
@@ -1749,7 +1733,16 @@ int drbd_worker(struct drbd_thread *thi)
         */
        spin_unlock_irq(&tconn->data.work.q_lock);
 
-       idr_for_each(&tconn->volumes, _worker_dying, NULL);
+       drbd_thread_stop(&tconn->receiver);
+       idr_for_each_entry(&tconn->volumes, mdev, minor) {
+               D_ASSERT(mdev->state.disk == D_DISKLESS && mdev->state.conn == C_STANDALONE);
+               /* _drbd_set_state only uses stop_nowait.
+                * wait here for the exiting receiver. */
+               drbd_mdev_cleanup(mdev);
+       }
+       clear_bit(OBJECT_DYING, &tconn->flags);
+       clear_bit(CONFIG_PENDING, &tconn->flags);
+       wake_up(&tconn->ping_wait);
 
        return 0;
 }