4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
28 #include <linux/drbd_limits.h>
30 #include "drbd_protocol.h"
33 struct after_state_chg_work
{
37 enum chg_state_flags flags
;
38 struct completion
*done
;
41 enum sanitize_state_warnings
{
43 ABORTED_ONLINE_VERIFY
,
45 CONNECTION_LOST_NEGOTIATING
,
46 IMPLICITLY_UPGRADED_DISK
,
47 IMPLICITLY_UPGRADED_PDSK
,
50 static int w_after_state_ch(struct drbd_work
*w
, int unused
);
51 static void after_state_ch(struct drbd_device
*device
, union drbd_state os
,
52 union drbd_state ns
, enum chg_state_flags flags
);
53 static enum drbd_state_rv
is_valid_state(struct drbd_device
*, union drbd_state
);
54 static enum drbd_state_rv
is_valid_soft_transition(union drbd_state
, union drbd_state
, struct drbd_connection
*);
55 static enum drbd_state_rv
is_valid_transition(union drbd_state os
, union drbd_state ns
);
56 static union drbd_state
sanitize_state(struct drbd_device
*device
, union drbd_state ns
,
57 enum sanitize_state_warnings
*warn
);
59 static inline bool is_susp(union drbd_state s
)
61 return s
.susp
|| s
.susp_nod
|| s
.susp_fen
;
64 bool conn_all_vols_unconf(struct drbd_connection
*connection
)
66 struct drbd_device
*device
;
71 idr_for_each_entry(&connection
->volumes
, device
, vnr
) {
72 if (device
->state
.disk
!= D_DISKLESS
||
73 device
->state
.conn
!= C_STANDALONE
||
74 device
->state
.role
!= R_SECONDARY
) {
84 /* Unfortunately the states where not correctly ordered, when
85 they where defined. therefore can not use max_t() here. */
86 static enum drbd_role
max_role(enum drbd_role role1
, enum drbd_role role2
)
88 if (role1
== R_PRIMARY
|| role2
== R_PRIMARY
)
90 if (role1
== R_SECONDARY
|| role2
== R_SECONDARY
)
94 static enum drbd_role
min_role(enum drbd_role role1
, enum drbd_role role2
)
96 if (role1
== R_UNKNOWN
|| role2
== R_UNKNOWN
)
98 if (role1
== R_SECONDARY
|| role2
== R_SECONDARY
)
103 enum drbd_role
conn_highest_role(struct drbd_connection
*connection
)
105 enum drbd_role role
= R_UNKNOWN
;
106 struct drbd_device
*device
;
110 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
111 role
= max_role(role
, device
->state
.role
);
117 enum drbd_role
conn_highest_peer(struct drbd_connection
*connection
)
119 enum drbd_role peer
= R_UNKNOWN
;
120 struct drbd_device
*device
;
124 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
125 peer
= max_role(peer
, device
->state
.peer
);
131 enum drbd_disk_state
conn_highest_disk(struct drbd_connection
*connection
)
133 enum drbd_disk_state ds
= D_DISKLESS
;
134 struct drbd_device
*device
;
138 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
139 ds
= max_t(enum drbd_disk_state
, ds
, device
->state
.disk
);
145 enum drbd_disk_state
conn_lowest_disk(struct drbd_connection
*connection
)
147 enum drbd_disk_state ds
= D_MASK
;
148 struct drbd_device
*device
;
152 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
153 ds
= min_t(enum drbd_disk_state
, ds
, device
->state
.disk
);
159 enum drbd_disk_state
conn_highest_pdsk(struct drbd_connection
*connection
)
161 enum drbd_disk_state ds
= D_DISKLESS
;
162 struct drbd_device
*device
;
166 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
167 ds
= max_t(enum drbd_disk_state
, ds
, device
->state
.pdsk
);
173 enum drbd_conns
conn_lowest_conn(struct drbd_connection
*connection
)
175 enum drbd_conns conn
= C_MASK
;
176 struct drbd_device
*device
;
180 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
181 conn
= min_t(enum drbd_conns
, conn
, device
->state
.conn
);
187 static bool no_peer_wf_report_params(struct drbd_connection
*connection
)
189 struct drbd_device
*device
;
194 idr_for_each_entry(&connection
->volumes
, device
, vnr
)
195 if (device
->state
.conn
== C_WF_REPORT_PARAMS
) {
206 * cl_wide_st_chg() - true if the state change is a cluster wide one
207 * @device: DRBD device.
208 * @os: old (current) state.
209 * @ns: new (wanted) state.
211 static int cl_wide_st_chg(struct drbd_device
*device
,
212 union drbd_state os
, union drbd_state ns
)
214 return (os
.conn
>= C_CONNECTED
&& ns
.conn
>= C_CONNECTED
&&
215 ((os
.role
!= R_PRIMARY
&& ns
.role
== R_PRIMARY
) ||
216 (os
.conn
!= C_STARTING_SYNC_T
&& ns
.conn
== C_STARTING_SYNC_T
) ||
217 (os
.conn
!= C_STARTING_SYNC_S
&& ns
.conn
== C_STARTING_SYNC_S
) ||
218 (os
.disk
!= D_FAILED
&& ns
.disk
== D_FAILED
))) ||
219 (os
.conn
>= C_CONNECTED
&& ns
.conn
== C_DISCONNECTING
) ||
220 (os
.conn
== C_CONNECTED
&& ns
.conn
== C_VERIFY_S
) ||
221 (os
.conn
== C_CONNECTED
&& ns
.conn
== C_WF_REPORT_PARAMS
);
224 static union drbd_state
225 apply_mask_val(union drbd_state os
, union drbd_state mask
, union drbd_state val
)
228 ns
.i
= (os
.i
& ~mask
.i
) | val
.i
;
233 drbd_change_state(struct drbd_device
*device
, enum chg_state_flags f
,
234 union drbd_state mask
, union drbd_state val
)
238 enum drbd_state_rv rv
;
240 spin_lock_irqsave(&first_peer_device(device
)->connection
->req_lock
, flags
);
241 ns
= apply_mask_val(drbd_read_state(device
), mask
, val
);
242 rv
= _drbd_set_state(device
, ns
, f
, NULL
);
243 spin_unlock_irqrestore(&first_peer_device(device
)->connection
->req_lock
, flags
);
249 * drbd_force_state() - Impose a change which happens outside our control on our state
250 * @device: DRBD device.
251 * @mask: mask of state bits to change.
252 * @val: value of new state bits.
254 void drbd_force_state(struct drbd_device
*device
,
255 union drbd_state mask
, union drbd_state val
)
257 drbd_change_state(device
, CS_HARD
, mask
, val
);
260 static enum drbd_state_rv
261 _req_st_cond(struct drbd_device
*device
, union drbd_state mask
,
262 union drbd_state val
)
264 union drbd_state os
, ns
;
266 enum drbd_state_rv rv
;
268 if (test_and_clear_bit(CL_ST_CHG_SUCCESS
, &device
->flags
))
269 return SS_CW_SUCCESS
;
271 if (test_and_clear_bit(CL_ST_CHG_FAIL
, &device
->flags
))
272 return SS_CW_FAILED_BY_PEER
;
274 spin_lock_irqsave(&first_peer_device(device
)->connection
->req_lock
, flags
);
275 os
= drbd_read_state(device
);
276 ns
= sanitize_state(device
, apply_mask_val(os
, mask
, val
), NULL
);
277 rv
= is_valid_transition(os
, ns
);
278 if (rv
>= SS_SUCCESS
)
279 rv
= SS_UNKNOWN_ERROR
; /* cont waiting, otherwise fail. */
281 if (!cl_wide_st_chg(device
, os
, ns
))
283 if (rv
== SS_UNKNOWN_ERROR
) {
284 rv
= is_valid_state(device
, ns
);
285 if (rv
>= SS_SUCCESS
) {
286 rv
= is_valid_soft_transition(os
, ns
, first_peer_device(device
)->connection
);
287 if (rv
>= SS_SUCCESS
)
288 rv
= SS_UNKNOWN_ERROR
; /* cont waiting, otherwise fail. */
291 spin_unlock_irqrestore(&first_peer_device(device
)->connection
->req_lock
, flags
);
297 * drbd_req_state() - Perform an eventually cluster wide state change
298 * @device: DRBD device.
299 * @mask: mask of state bits to change.
300 * @val: value of new state bits.
303 * Should not be called directly, use drbd_request_state() or
304 * _drbd_request_state().
306 static enum drbd_state_rv
307 drbd_req_state(struct drbd_device
*device
, union drbd_state mask
,
308 union drbd_state val
, enum chg_state_flags f
)
310 struct completion done
;
312 union drbd_state os
, ns
;
313 enum drbd_state_rv rv
;
315 init_completion(&done
);
317 if (f
& CS_SERIALIZE
)
318 mutex_lock(device
->state_mutex
);
320 spin_lock_irqsave(&first_peer_device(device
)->connection
->req_lock
, flags
);
321 os
= drbd_read_state(device
);
322 ns
= sanitize_state(device
, apply_mask_val(os
, mask
, val
), NULL
);
323 rv
= is_valid_transition(os
, ns
);
324 if (rv
< SS_SUCCESS
) {
325 spin_unlock_irqrestore(&first_peer_device(device
)->connection
->req_lock
, flags
);
329 if (cl_wide_st_chg(device
, os
, ns
)) {
330 rv
= is_valid_state(device
, ns
);
331 if (rv
== SS_SUCCESS
)
332 rv
= is_valid_soft_transition(os
, ns
, first_peer_device(device
)->connection
);
333 spin_unlock_irqrestore(&first_peer_device(device
)->connection
->req_lock
, flags
);
335 if (rv
< SS_SUCCESS
) {
337 print_st_err(device
, os
, ns
, rv
);
341 if (drbd_send_state_req(device
, mask
, val
)) {
342 rv
= SS_CW_FAILED_BY_PEER
;
344 print_st_err(device
, os
, ns
, rv
);
348 wait_event(device
->state_wait
,
349 (rv
= _req_st_cond(device
, mask
, val
)));
351 if (rv
< SS_SUCCESS
) {
353 print_st_err(device
, os
, ns
, rv
);
356 spin_lock_irqsave(&first_peer_device(device
)->connection
->req_lock
, flags
);
357 ns
= apply_mask_val(drbd_read_state(device
), mask
, val
);
358 rv
= _drbd_set_state(device
, ns
, f
, &done
);
360 rv
= _drbd_set_state(device
, ns
, f
, &done
);
363 spin_unlock_irqrestore(&first_peer_device(device
)->connection
->req_lock
, flags
);
365 if (f
& CS_WAIT_COMPLETE
&& rv
== SS_SUCCESS
) {
366 D_ASSERT(current
!= first_peer_device(device
)->connection
->worker
.task
);
367 wait_for_completion(&done
);
371 if (f
& CS_SERIALIZE
)
372 mutex_unlock(device
->state_mutex
);
378 * _drbd_request_state() - Request a state change (with flags)
379 * @device: DRBD device.
380 * @mask: mask of state bits to change.
381 * @val: value of new state bits.
384 * Cousin of drbd_request_state(), useful with the CS_WAIT_COMPLETE
385 * flag, or when logging of failed state change requests is not desired.
388 _drbd_request_state(struct drbd_device
*device
, union drbd_state mask
,
389 union drbd_state val
, enum chg_state_flags f
)
391 enum drbd_state_rv rv
;
393 wait_event(device
->state_wait
,
394 (rv
= drbd_req_state(device
, mask
, val
, f
)) != SS_IN_TRANSIENT_STATE
);
399 static void print_st(struct drbd_device
*device
, char *name
, union drbd_state ns
)
401 dev_err(DEV
, " %s = { cs:%s ro:%s/%s ds:%s/%s %c%c%c%c%c%c }\n",
403 drbd_conn_str(ns
.conn
),
404 drbd_role_str(ns
.role
),
405 drbd_role_str(ns
.peer
),
406 drbd_disk_str(ns
.disk
),
407 drbd_disk_str(ns
.pdsk
),
408 is_susp(ns
) ? 's' : 'r',
409 ns
.aftr_isp
? 'a' : '-',
410 ns
.peer_isp
? 'p' : '-',
411 ns
.user_isp
? 'u' : '-',
412 ns
.susp_fen
? 'F' : '-',
413 ns
.susp_nod
? 'N' : '-'
417 void print_st_err(struct drbd_device
*device
, union drbd_state os
,
418 union drbd_state ns
, enum drbd_state_rv err
)
420 if (err
== SS_IN_TRANSIENT_STATE
)
422 dev_err(DEV
, "State change failed: %s\n", drbd_set_st_err_str(err
));
423 print_st(device
, " state", os
);
424 print_st(device
, "wanted", ns
);
427 static long print_state_change(char *pb
, union drbd_state os
, union drbd_state ns
,
428 enum chg_state_flags flags
)
434 if (ns
.role
!= os
.role
&& flags
& CS_DC_ROLE
)
435 pbp
+= sprintf(pbp
, "role( %s -> %s ) ",
436 drbd_role_str(os
.role
),
437 drbd_role_str(ns
.role
));
438 if (ns
.peer
!= os
.peer
&& flags
& CS_DC_PEER
)
439 pbp
+= sprintf(pbp
, "peer( %s -> %s ) ",
440 drbd_role_str(os
.peer
),
441 drbd_role_str(ns
.peer
));
442 if (ns
.conn
!= os
.conn
&& flags
& CS_DC_CONN
)
443 pbp
+= sprintf(pbp
, "conn( %s -> %s ) ",
444 drbd_conn_str(os
.conn
),
445 drbd_conn_str(ns
.conn
));
446 if (ns
.disk
!= os
.disk
&& flags
& CS_DC_DISK
)
447 pbp
+= sprintf(pbp
, "disk( %s -> %s ) ",
448 drbd_disk_str(os
.disk
),
449 drbd_disk_str(ns
.disk
));
450 if (ns
.pdsk
!= os
.pdsk
&& flags
& CS_DC_PDSK
)
451 pbp
+= sprintf(pbp
, "pdsk( %s -> %s ) ",
452 drbd_disk_str(os
.pdsk
),
453 drbd_disk_str(ns
.pdsk
));
458 static void drbd_pr_state_change(struct drbd_device
*device
, union drbd_state os
, union drbd_state ns
,
459 enum chg_state_flags flags
)
464 pbp
+= print_state_change(pbp
, os
, ns
, flags
^ CS_DC_MASK
);
466 if (ns
.aftr_isp
!= os
.aftr_isp
)
467 pbp
+= sprintf(pbp
, "aftr_isp( %d -> %d ) ",
470 if (ns
.peer_isp
!= os
.peer_isp
)
471 pbp
+= sprintf(pbp
, "peer_isp( %d -> %d ) ",
474 if (ns
.user_isp
!= os
.user_isp
)
475 pbp
+= sprintf(pbp
, "user_isp( %d -> %d ) ",
480 dev_info(DEV
, "%s\n", pb
);
483 static void conn_pr_state_change(struct drbd_connection
*connection
, union drbd_state os
, union drbd_state ns
,
484 enum chg_state_flags flags
)
489 pbp
+= print_state_change(pbp
, os
, ns
, flags
);
491 if (is_susp(ns
) != is_susp(os
) && flags
& CS_DC_SUSP
)
492 pbp
+= sprintf(pbp
, "susp( %d -> %d ) ",
497 conn_info(connection
, "%s\n", pb
);
502 * is_valid_state() - Returns an SS_ error code if ns is not valid
503 * @device: DRBD device.
504 * @ns: State to consider.
506 static enum drbd_state_rv
507 is_valid_state(struct drbd_device
*device
, union drbd_state ns
)
509 /* See drbd_state_sw_errors in drbd_strings.c */
511 enum drbd_fencing_p fp
;
512 enum drbd_state_rv rv
= SS_SUCCESS
;
517 if (get_ldev(device
)) {
518 fp
= rcu_dereference(device
->ldev
->disk_conf
)->fencing
;
522 nc
= rcu_dereference(first_peer_device(device
)->connection
->net_conf
);
524 if (!nc
->two_primaries
&& ns
.role
== R_PRIMARY
) {
525 if (ns
.peer
== R_PRIMARY
)
526 rv
= SS_TWO_PRIMARIES
;
527 else if (conn_highest_peer(first_peer_device(device
)->connection
) == R_PRIMARY
)
528 rv
= SS_O_VOL_PEER_PRI
;
533 /* already found a reason to abort */;
534 else if (ns
.role
== R_SECONDARY
&& device
->open_cnt
)
535 rv
= SS_DEVICE_IN_USE
;
537 else if (ns
.role
== R_PRIMARY
&& ns
.conn
< C_CONNECTED
&& ns
.disk
< D_UP_TO_DATE
)
538 rv
= SS_NO_UP_TO_DATE_DISK
;
540 else if (fp
>= FP_RESOURCE
&&
541 ns
.role
== R_PRIMARY
&& ns
.conn
< C_CONNECTED
&& ns
.pdsk
>= D_UNKNOWN
)
544 else if (ns
.role
== R_PRIMARY
&& ns
.disk
<= D_INCONSISTENT
&& ns
.pdsk
<= D_INCONSISTENT
)
545 rv
= SS_NO_UP_TO_DATE_DISK
;
547 else if (ns
.conn
> C_CONNECTED
&& ns
.disk
< D_INCONSISTENT
)
548 rv
= SS_NO_LOCAL_DISK
;
550 else if (ns
.conn
> C_CONNECTED
&& ns
.pdsk
< D_INCONSISTENT
)
551 rv
= SS_NO_REMOTE_DISK
;
553 else if (ns
.conn
> C_CONNECTED
&& ns
.disk
< D_UP_TO_DATE
&& ns
.pdsk
< D_UP_TO_DATE
)
554 rv
= SS_NO_UP_TO_DATE_DISK
;
556 else if ((ns
.conn
== C_CONNECTED
||
557 ns
.conn
== C_WF_BITMAP_S
||
558 ns
.conn
== C_SYNC_SOURCE
||
559 ns
.conn
== C_PAUSED_SYNC_S
) &&
560 ns
.disk
== D_OUTDATED
)
561 rv
= SS_CONNECTED_OUTDATES
;
563 else if ((ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
) &&
564 (nc
->verify_alg
[0] == 0))
565 rv
= SS_NO_VERIFY_ALG
;
567 else if ((ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
) &&
568 first_peer_device(device
)->connection
->agreed_pro_version
< 88)
569 rv
= SS_NOT_SUPPORTED
;
571 else if (ns
.role
== R_PRIMARY
&& ns
.disk
< D_UP_TO_DATE
&& ns
.pdsk
< D_UP_TO_DATE
)
572 rv
= SS_NO_UP_TO_DATE_DISK
;
574 else if ((ns
.conn
== C_STARTING_SYNC_S
|| ns
.conn
== C_STARTING_SYNC_T
) &&
575 ns
.pdsk
== D_UNKNOWN
)
576 rv
= SS_NEED_CONNECTION
;
578 else if (ns
.conn
>= C_CONNECTED
&& ns
.pdsk
== D_UNKNOWN
)
579 rv
= SS_CONNECTED_OUTDATES
;
587 * is_valid_soft_transition() - Returns an SS_ error code if the state transition is not possible
588 * This function limits state transitions that may be declined by DRBD. I.e.
589 * user requests (aka soft transitions).
590 * @device: DRBD device.
594 static enum drbd_state_rv
595 is_valid_soft_transition(union drbd_state os
, union drbd_state ns
, struct drbd_connection
*connection
)
597 enum drbd_state_rv rv
= SS_SUCCESS
;
599 if ((ns
.conn
== C_STARTING_SYNC_T
|| ns
.conn
== C_STARTING_SYNC_S
) &&
600 os
.conn
> C_CONNECTED
)
601 rv
= SS_RESYNC_RUNNING
;
603 if (ns
.conn
== C_DISCONNECTING
&& os
.conn
== C_STANDALONE
)
604 rv
= SS_ALREADY_STANDALONE
;
606 if (ns
.disk
> D_ATTACHING
&& os
.disk
== D_DISKLESS
)
609 if (ns
.conn
== C_WF_CONNECTION
&& os
.conn
< C_UNCONNECTED
)
610 rv
= SS_NO_NET_CONFIG
;
612 if (ns
.disk
== D_OUTDATED
&& os
.disk
< D_OUTDATED
&& os
.disk
!= D_ATTACHING
)
613 rv
= SS_LOWER_THAN_OUTDATED
;
615 if (ns
.conn
== C_DISCONNECTING
&& os
.conn
== C_UNCONNECTED
)
616 rv
= SS_IN_TRANSIENT_STATE
;
618 /* if (ns.conn == os.conn && ns.conn == C_WF_REPORT_PARAMS)
619 rv = SS_IN_TRANSIENT_STATE; */
621 /* While establishing a connection only allow cstate to change.
622 Delay/refuse role changes, detach attach etc... */
623 if (test_bit(STATE_SENT
, &connection
->flags
) &&
624 !(os
.conn
== C_WF_REPORT_PARAMS
||
625 (ns
.conn
== C_WF_REPORT_PARAMS
&& os
.conn
== C_WF_CONNECTION
)))
626 rv
= SS_IN_TRANSIENT_STATE
;
628 if ((ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
) && os
.conn
< C_CONNECTED
)
629 rv
= SS_NEED_CONNECTION
;
631 if ((ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
) &&
632 ns
.conn
!= os
.conn
&& os
.conn
> C_CONNECTED
)
633 rv
= SS_RESYNC_RUNNING
;
635 if ((ns
.conn
== C_STARTING_SYNC_S
|| ns
.conn
== C_STARTING_SYNC_T
) &&
636 os
.conn
< C_CONNECTED
)
637 rv
= SS_NEED_CONNECTION
;
639 if ((ns
.conn
== C_SYNC_TARGET
|| ns
.conn
== C_SYNC_SOURCE
)
640 && os
.conn
< C_WF_REPORT_PARAMS
)
641 rv
= SS_NEED_CONNECTION
; /* No NetworkFailure -> SyncTarget etc... */
643 if (ns
.conn
== C_DISCONNECTING
&& ns
.pdsk
== D_OUTDATED
&&
644 os
.conn
< C_CONNECTED
&& os
.pdsk
> D_OUTDATED
)
645 rv
= SS_OUTDATE_WO_CONN
;
650 static enum drbd_state_rv
651 is_valid_conn_transition(enum drbd_conns oc
, enum drbd_conns nc
)
653 /* no change -> nothing to do, at least for the connection part */
655 return SS_NOTHING_TO_DO
;
657 /* disconnect of an unconfigured connection does not make sense */
658 if (oc
== C_STANDALONE
&& nc
== C_DISCONNECTING
)
659 return SS_ALREADY_STANDALONE
;
661 /* from C_STANDALONE, we start with C_UNCONNECTED */
662 if (oc
== C_STANDALONE
&& nc
!= C_UNCONNECTED
)
663 return SS_NEED_CONNECTION
;
665 /* When establishing a connection we need to go through WF_REPORT_PARAMS!
666 Necessary to do the right thing upon invalidate-remote on a disconnected resource */
667 if (oc
< C_WF_REPORT_PARAMS
&& nc
>= C_CONNECTED
)
668 return SS_NEED_CONNECTION
;
670 /* After a network error only C_UNCONNECTED or C_DISCONNECTING may follow. */
671 if (oc
>= C_TIMEOUT
&& oc
<= C_TEAR_DOWN
&& nc
!= C_UNCONNECTED
&& nc
!= C_DISCONNECTING
)
672 return SS_IN_TRANSIENT_STATE
;
674 /* After C_DISCONNECTING only C_STANDALONE may follow */
675 if (oc
== C_DISCONNECTING
&& nc
!= C_STANDALONE
)
676 return SS_IN_TRANSIENT_STATE
;
683 * is_valid_transition() - Returns an SS_ error code if the state transition is not possible
684 * This limits hard state transitions. Hard state transitions are facts there are
685 * imposed on DRBD by the environment. E.g. disk broke or network broke down.
686 * But those hard state transitions are still not allowed to do everything.
690 static enum drbd_state_rv
691 is_valid_transition(union drbd_state os
, union drbd_state ns
)
693 enum drbd_state_rv rv
;
695 rv
= is_valid_conn_transition(os
.conn
, ns
.conn
);
697 /* we cannot fail (again) if we already detached */
698 if (ns
.disk
== D_FAILED
&& os
.disk
== D_DISKLESS
)
704 static void print_sanitize_warnings(struct drbd_device
*device
, enum sanitize_state_warnings warn
)
706 static const char *msg_table
[] = {
708 [ABORTED_ONLINE_VERIFY
] = "Online-verify aborted.",
709 [ABORTED_RESYNC
] = "Resync aborted.",
710 [CONNECTION_LOST_NEGOTIATING
] = "Connection lost while negotiating, no data!",
711 [IMPLICITLY_UPGRADED_DISK
] = "Implicitly upgraded disk",
712 [IMPLICITLY_UPGRADED_PDSK
] = "Implicitly upgraded pdsk",
715 if (warn
!= NO_WARNING
)
716 dev_warn(DEV
, "%s\n", msg_table
[warn
]);
720 * sanitize_state() - Resolves implicitly necessary additional changes to a state transition
721 * @device: DRBD device.
726 * When we loose connection, we have to set the state of the peers disk (pdsk)
727 * to D_UNKNOWN. This rule and many more along those lines are in this function.
729 static union drbd_state
sanitize_state(struct drbd_device
*device
, union drbd_state ns
,
730 enum sanitize_state_warnings
*warn
)
732 enum drbd_fencing_p fp
;
733 enum drbd_disk_state disk_min
, disk_max
, pdsk_min
, pdsk_max
;
739 if (get_ldev(device
)) {
741 fp
= rcu_dereference(device
->ldev
->disk_conf
)->fencing
;
746 /* Implications from connection to peer and peer_isp */
747 if (ns
.conn
< C_CONNECTED
) {
750 if (ns
.pdsk
> D_UNKNOWN
|| ns
.pdsk
< D_INCONSISTENT
)
754 /* Clear the aftr_isp when becoming unconfigured */
755 if (ns
.conn
== C_STANDALONE
&& ns
.disk
== D_DISKLESS
&& ns
.role
== R_SECONDARY
)
758 /* An implication of the disk states onto the connection state */
759 /* Abort resync if a disk fails/detaches */
760 if (ns
.conn
> C_CONNECTED
&& (ns
.disk
<= D_FAILED
|| ns
.pdsk
<= D_FAILED
)) {
762 *warn
= ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
?
763 ABORTED_ONLINE_VERIFY
: ABORTED_RESYNC
;
764 ns
.conn
= C_CONNECTED
;
767 /* Connection breaks down before we finished "Negotiating" */
768 if (ns
.conn
< C_CONNECTED
&& ns
.disk
== D_NEGOTIATING
&&
769 get_ldev_if_state(device
, D_NEGOTIATING
)) {
770 if (device
->ed_uuid
== device
->ldev
->md
.uuid
[UI_CURRENT
]) {
771 ns
.disk
= device
->new_state_tmp
.disk
;
772 ns
.pdsk
= device
->new_state_tmp
.pdsk
;
775 *warn
= CONNECTION_LOST_NEGOTIATING
;
776 ns
.disk
= D_DISKLESS
;
782 /* D_CONSISTENT and D_OUTDATED vanish when we get connected */
783 if (ns
.conn
>= C_CONNECTED
&& ns
.conn
< C_AHEAD
) {
784 if (ns
.disk
== D_CONSISTENT
|| ns
.disk
== D_OUTDATED
)
785 ns
.disk
= D_UP_TO_DATE
;
786 if (ns
.pdsk
== D_CONSISTENT
|| ns
.pdsk
== D_OUTDATED
)
787 ns
.pdsk
= D_UP_TO_DATE
;
790 /* Implications of the connection stat on the disk states */
791 disk_min
= D_DISKLESS
;
792 disk_max
= D_UP_TO_DATE
;
793 pdsk_min
= D_INCONSISTENT
;
794 pdsk_max
= D_UNKNOWN
;
795 switch ((enum drbd_conns
)ns
.conn
) {
797 case C_PAUSED_SYNC_T
:
798 case C_STARTING_SYNC_T
:
801 disk_min
= D_INCONSISTENT
;
802 disk_max
= D_OUTDATED
;
803 pdsk_min
= D_UP_TO_DATE
;
804 pdsk_max
= D_UP_TO_DATE
;
808 disk_min
= D_UP_TO_DATE
;
809 disk_max
= D_UP_TO_DATE
;
810 pdsk_min
= D_UP_TO_DATE
;
811 pdsk_max
= D_UP_TO_DATE
;
814 disk_min
= D_DISKLESS
;
815 disk_max
= D_UP_TO_DATE
;
816 pdsk_min
= D_DISKLESS
;
817 pdsk_max
= D_UP_TO_DATE
;
820 case C_PAUSED_SYNC_S
:
821 case C_STARTING_SYNC_S
:
823 disk_min
= D_UP_TO_DATE
;
824 disk_max
= D_UP_TO_DATE
;
825 pdsk_min
= D_INCONSISTENT
;
826 pdsk_max
= D_CONSISTENT
; /* D_OUTDATED would be nice. But explicit outdate necessary*/
829 disk_min
= D_INCONSISTENT
;
830 disk_max
= D_INCONSISTENT
;
831 pdsk_min
= D_UP_TO_DATE
;
832 pdsk_max
= D_UP_TO_DATE
;
835 disk_min
= D_UP_TO_DATE
;
836 disk_max
= D_UP_TO_DATE
;
837 pdsk_min
= D_INCONSISTENT
;
838 pdsk_max
= D_INCONSISTENT
;
841 case C_DISCONNECTING
:
845 case C_NETWORK_FAILURE
:
846 case C_PROTOCOL_ERROR
:
848 case C_WF_CONNECTION
:
849 case C_WF_REPORT_PARAMS
:
853 if (ns
.disk
> disk_max
)
856 if (ns
.disk
< disk_min
) {
858 *warn
= IMPLICITLY_UPGRADED_DISK
;
861 if (ns
.pdsk
> pdsk_max
)
864 if (ns
.pdsk
< pdsk_min
) {
866 *warn
= IMPLICITLY_UPGRADED_PDSK
;
870 if (fp
== FP_STONITH
&&
871 (ns
.role
== R_PRIMARY
&& ns
.conn
< C_CONNECTED
&& ns
.pdsk
> D_OUTDATED
))
872 ns
.susp_fen
= 1; /* Suspend IO while fence-peer handler runs (peer lost) */
874 if (first_peer_device(device
)->connection
->res_opts
.on_no_data
== OND_SUSPEND_IO
&&
875 (ns
.role
== R_PRIMARY
&& ns
.disk
< D_UP_TO_DATE
&& ns
.pdsk
< D_UP_TO_DATE
))
876 ns
.susp_nod
= 1; /* Suspend IO while no data available (no accessible data available) */
878 if (ns
.aftr_isp
|| ns
.peer_isp
|| ns
.user_isp
) {
879 if (ns
.conn
== C_SYNC_SOURCE
)
880 ns
.conn
= C_PAUSED_SYNC_S
;
881 if (ns
.conn
== C_SYNC_TARGET
)
882 ns
.conn
= C_PAUSED_SYNC_T
;
884 if (ns
.conn
== C_PAUSED_SYNC_S
)
885 ns
.conn
= C_SYNC_SOURCE
;
886 if (ns
.conn
== C_PAUSED_SYNC_T
)
887 ns
.conn
= C_SYNC_TARGET
;
893 void drbd_resume_al(struct drbd_device
*device
)
895 if (test_and_clear_bit(AL_SUSPENDED
, &device
->flags
))
896 dev_info(DEV
, "Resumed AL updates\n");
899 /* helper for __drbd_set_state */
900 static void set_ov_position(struct drbd_device
*device
, enum drbd_conns cs
)
902 if (first_peer_device(device
)->connection
->agreed_pro_version
< 90)
903 device
->ov_start_sector
= 0;
904 device
->rs_total
= drbd_bm_bits(device
);
905 device
->ov_position
= 0;
906 if (cs
== C_VERIFY_T
) {
907 /* starting online verify from an arbitrary position
908 * does not fit well into the existing protocol.
909 * on C_VERIFY_T, we initialize ov_left and friends
910 * implicitly in receive_DataRequest once the
911 * first P_OV_REQUEST is received */
912 device
->ov_start_sector
= ~(sector_t
)0;
914 unsigned long bit
= BM_SECT_TO_BIT(device
->ov_start_sector
);
915 if (bit
>= device
->rs_total
) {
916 device
->ov_start_sector
=
917 BM_BIT_TO_SECT(device
->rs_total
- 1);
918 device
->rs_total
= 1;
920 device
->rs_total
-= bit
;
921 device
->ov_position
= device
->ov_start_sector
;
923 device
->ov_left
= device
->rs_total
;
927 * __drbd_set_state() - Set a new DRBD state
928 * @device: DRBD device.
931 * @done: Optional completion, that will get completed after the after_state_ch() finished
933 * Caller needs to hold req_lock, and global_state_lock. Do not call directly.
936 __drbd_set_state(struct drbd_device
*device
, union drbd_state ns
,
937 enum chg_state_flags flags
, struct completion
*done
)
940 enum drbd_state_rv rv
= SS_SUCCESS
;
941 enum sanitize_state_warnings ssw
;
942 struct after_state_chg_work
*ascw
;
943 bool did_remote
, should_do_remote
;
945 os
= drbd_read_state(device
);
947 ns
= sanitize_state(device
, ns
, &ssw
);
949 return SS_NOTHING_TO_DO
;
951 rv
= is_valid_transition(os
, ns
);
955 if (!(flags
& CS_HARD
)) {
956 /* pre-state-change checks ; only look at ns */
957 /* See drbd_state_sw_errors in drbd_strings.c */
959 rv
= is_valid_state(device
, ns
);
960 if (rv
< SS_SUCCESS
) {
961 /* If the old state was illegal as well, then let
964 if (is_valid_state(device
, os
) == rv
)
965 rv
= is_valid_soft_transition(os
, ns
, first_peer_device(device
)->connection
);
967 rv
= is_valid_soft_transition(os
, ns
, first_peer_device(device
)->connection
);
970 if (rv
< SS_SUCCESS
) {
971 if (flags
& CS_VERBOSE
)
972 print_st_err(device
, os
, ns
, rv
);
976 print_sanitize_warnings(device
, ssw
);
978 drbd_pr_state_change(device
, os
, ns
, flags
);
980 /* Display changes to the susp* flags that where caused by the call to
981 sanitize_state(). Only display it here if we where not called from
982 _conn_request_state() */
983 if (!(flags
& CS_DC_SUSP
))
984 conn_pr_state_change(first_peer_device(device
)->connection
, os
, ns
,
985 (flags
& ~CS_DC_MASK
) | CS_DC_SUSP
);
987 /* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
988 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
989 * drbd_ldev_destroy() won't happen before our corresponding
990 * after_state_ch works run, where we put_ldev again. */
991 if ((os
.disk
!= D_FAILED
&& ns
.disk
== D_FAILED
) ||
992 (os
.disk
!= D_DISKLESS
&& ns
.disk
== D_DISKLESS
))
993 atomic_inc(&device
->local_cnt
);
995 did_remote
= drbd_should_do_remote(device
->state
);
996 device
->state
.i
= ns
.i
;
997 should_do_remote
= drbd_should_do_remote(device
->state
);
998 first_peer_device(device
)->connection
->susp
= ns
.susp
;
999 first_peer_device(device
)->connection
->susp_nod
= ns
.susp_nod
;
1000 first_peer_device(device
)->connection
->susp_fen
= ns
.susp_fen
;
1002 /* put replicated vs not-replicated requests in seperate epochs */
1003 if (did_remote
!= should_do_remote
)
1004 start_new_tl_epoch(first_peer_device(device
)->connection
);
1006 if (os
.disk
== D_ATTACHING
&& ns
.disk
>= D_NEGOTIATING
)
1007 drbd_print_uuids(device
, "attached to UUIDs");
1009 /* Wake up role changes, that were delayed because of connection establishing */
1010 if (os
.conn
== C_WF_REPORT_PARAMS
&& ns
.conn
!= C_WF_REPORT_PARAMS
&&
1011 no_peer_wf_report_params(first_peer_device(device
)->connection
))
1012 clear_bit(STATE_SENT
, &first_peer_device(device
)->connection
->flags
);
1014 wake_up(&device
->misc_wait
);
1015 wake_up(&device
->state_wait
);
1016 wake_up(&first_peer_device(device
)->connection
->ping_wait
);
1018 /* Aborted verify run, or we reached the stop sector.
1019 * Log the last position, unless end-of-device. */
1020 if ((os
.conn
== C_VERIFY_S
|| os
.conn
== C_VERIFY_T
) &&
1021 ns
.conn
<= C_CONNECTED
) {
1022 device
->ov_start_sector
=
1023 BM_BIT_TO_SECT(drbd_bm_bits(device
) - device
->ov_left
);
1024 if (device
->ov_left
)
1025 dev_info(DEV
, "Online Verify reached sector %llu\n",
1026 (unsigned long long)device
->ov_start_sector
);
1029 if ((os
.conn
== C_PAUSED_SYNC_T
|| os
.conn
== C_PAUSED_SYNC_S
) &&
1030 (ns
.conn
== C_SYNC_TARGET
|| ns
.conn
== C_SYNC_SOURCE
)) {
1031 dev_info(DEV
, "Syncer continues.\n");
1032 device
->rs_paused
+= (long)jiffies
1033 -(long)device
->rs_mark_time
[device
->rs_last_mark
];
1034 if (ns
.conn
== C_SYNC_TARGET
)
1035 mod_timer(&device
->resync_timer
, jiffies
);
1038 if ((os
.conn
== C_SYNC_TARGET
|| os
.conn
== C_SYNC_SOURCE
) &&
1039 (ns
.conn
== C_PAUSED_SYNC_T
|| ns
.conn
== C_PAUSED_SYNC_S
)) {
1040 dev_info(DEV
, "Resync suspended\n");
1041 device
->rs_mark_time
[device
->rs_last_mark
] = jiffies
;
1044 if (os
.conn
== C_CONNECTED
&&
1045 (ns
.conn
== C_VERIFY_S
|| ns
.conn
== C_VERIFY_T
)) {
1046 unsigned long now
= jiffies
;
1049 set_ov_position(device
, ns
.conn
);
1050 device
->rs_start
= now
;
1051 device
->rs_last_events
= 0;
1052 device
->rs_last_sect_ev
= 0;
1053 device
->ov_last_oos_size
= 0;
1054 device
->ov_last_oos_start
= 0;
1056 for (i
= 0; i
< DRBD_SYNC_MARKS
; i
++) {
1057 device
->rs_mark_left
[i
] = device
->ov_left
;
1058 device
->rs_mark_time
[i
] = now
;
1061 drbd_rs_controller_reset(device
);
1063 if (ns
.conn
== C_VERIFY_S
) {
1064 dev_info(DEV
, "Starting Online Verify from sector %llu\n",
1065 (unsigned long long)device
->ov_position
);
1066 mod_timer(&device
->resync_timer
, jiffies
);
1070 if (get_ldev(device
)) {
1071 u32 mdf
= device
->ldev
->md
.flags
& ~(MDF_CONSISTENT
|MDF_PRIMARY_IND
|
1072 MDF_CONNECTED_IND
|MDF_WAS_UP_TO_DATE
|
1073 MDF_PEER_OUT_DATED
|MDF_CRASHED_PRIMARY
);
1075 mdf
&= ~MDF_AL_CLEAN
;
1076 if (test_bit(CRASHED_PRIMARY
, &device
->flags
))
1077 mdf
|= MDF_CRASHED_PRIMARY
;
1078 if (device
->state
.role
== R_PRIMARY
||
1079 (device
->state
.pdsk
< D_INCONSISTENT
&& device
->state
.peer
== R_PRIMARY
))
1080 mdf
|= MDF_PRIMARY_IND
;
1081 if (device
->state
.conn
> C_WF_REPORT_PARAMS
)
1082 mdf
|= MDF_CONNECTED_IND
;
1083 if (device
->state
.disk
> D_INCONSISTENT
)
1084 mdf
|= MDF_CONSISTENT
;
1085 if (device
->state
.disk
> D_OUTDATED
)
1086 mdf
|= MDF_WAS_UP_TO_DATE
;
1087 if (device
->state
.pdsk
<= D_OUTDATED
&& device
->state
.pdsk
>= D_INCONSISTENT
)
1088 mdf
|= MDF_PEER_OUT_DATED
;
1089 if (mdf
!= device
->ldev
->md
.flags
) {
1090 device
->ldev
->md
.flags
= mdf
;
1091 drbd_md_mark_dirty(device
);
1093 if (os
.disk
< D_CONSISTENT
&& ns
.disk
>= D_CONSISTENT
)
1094 drbd_set_ed_uuid(device
, device
->ldev
->md
.uuid
[UI_CURRENT
]);
1098 /* Peer was forced D_UP_TO_DATE & R_PRIMARY, consider to resync */
1099 if (os
.disk
== D_INCONSISTENT
&& os
.pdsk
== D_INCONSISTENT
&&
1100 os
.peer
== R_SECONDARY
&& ns
.peer
== R_PRIMARY
)
1101 set_bit(CONSIDER_RESYNC
, &device
->flags
);
1103 /* Receiver should clean up itself */
1104 if (os
.conn
!= C_DISCONNECTING
&& ns
.conn
== C_DISCONNECTING
)
1105 drbd_thread_stop_nowait(&first_peer_device(device
)->connection
->receiver
);
1107 /* Now the receiver finished cleaning up itself, it should die */
1108 if (os
.conn
!= C_STANDALONE
&& ns
.conn
== C_STANDALONE
)
1109 drbd_thread_stop_nowait(&first_peer_device(device
)->connection
->receiver
);
1111 /* Upon network failure, we need to restart the receiver. */
1112 if (os
.conn
> C_WF_CONNECTION
&&
1113 ns
.conn
<= C_TEAR_DOWN
&& ns
.conn
>= C_TIMEOUT
)
1114 drbd_thread_restart_nowait(&first_peer_device(device
)->connection
->receiver
);
1116 /* Resume AL writing if we get a connection */
1117 if (os
.conn
< C_CONNECTED
&& ns
.conn
>= C_CONNECTED
) {
1118 drbd_resume_al(device
);
1119 first_peer_device(device
)->connection
->connect_cnt
++;
1122 /* remember last attach time so request_timer_fn() won't
1123 * kill newly established sessions while we are still trying to thaw
1124 * previously frozen IO */
1125 if ((os
.disk
== D_ATTACHING
|| os
.disk
== D_NEGOTIATING
) &&
1126 ns
.disk
> D_NEGOTIATING
)
1127 device
->last_reattach_jif
= jiffies
;
1129 ascw
= kmalloc(sizeof(*ascw
), GFP_ATOMIC
);
1133 ascw
->flags
= flags
;
1134 ascw
->w
.cb
= w_after_state_ch
;
1135 ascw
->w
.device
= device
;
1137 drbd_queue_work(&first_peer_device(device
)->connection
->sender_work
, &ascw
->w
);
1139 dev_err(DEV
, "Could not kmalloc an ascw\n");
1145 static int w_after_state_ch(struct drbd_work
*w
, int unused
)
1147 struct after_state_chg_work
*ascw
=
1148 container_of(w
, struct after_state_chg_work
, w
);
1149 struct drbd_device
*device
= w
->device
;
1151 after_state_ch(device
, ascw
->os
, ascw
->ns
, ascw
->flags
);
1152 if (ascw
->flags
& CS_WAIT_COMPLETE
) {
1153 D_ASSERT(ascw
->done
!= NULL
);
1154 complete(ascw
->done
);
1161 static void abw_start_sync(struct drbd_device
*device
, int rv
)
1164 dev_err(DEV
, "Writing the bitmap failed not starting resync.\n");
1165 _drbd_request_state(device
, NS(conn
, C_CONNECTED
), CS_VERBOSE
);
1169 switch (device
->state
.conn
) {
1170 case C_STARTING_SYNC_T
:
1171 _drbd_request_state(device
, NS(conn
, C_WF_SYNC_UUID
), CS_VERBOSE
);
1173 case C_STARTING_SYNC_S
:
1174 drbd_start_resync(device
, C_SYNC_SOURCE
);
1179 int drbd_bitmap_io_from_worker(struct drbd_device
*device
,
1180 int (*io_fn
)(struct drbd_device
*),
1181 char *why
, enum bm_flag flags
)
1185 D_ASSERT(current
== first_peer_device(device
)->connection
->worker
.task
);
1187 /* open coded non-blocking drbd_suspend_io(device); */
1188 set_bit(SUSPEND_IO
, &device
->flags
);
1190 drbd_bm_lock(device
, why
, flags
);
1192 drbd_bm_unlock(device
);
1194 drbd_resume_io(device
);
1200 * after_state_ch() - Perform after state change actions that may sleep
1201 * @device: DRBD device.
1206 static void after_state_ch(struct drbd_device
*device
, union drbd_state os
,
1207 union drbd_state ns
, enum chg_state_flags flags
)
1209 struct sib_info sib
;
1211 sib
.sib_reason
= SIB_STATE_CHANGE
;
1215 if (os
.conn
!= C_CONNECTED
&& ns
.conn
== C_CONNECTED
) {
1216 clear_bit(CRASHED_PRIMARY
, &device
->flags
);
1218 device
->p_uuid
[UI_FLAGS
] &= ~((u64
)2);
1221 /* Inform userspace about the change... */
1222 drbd_bcast_event(device
, &sib
);
1224 if (!(os
.role
== R_PRIMARY
&& os
.disk
< D_UP_TO_DATE
&& os
.pdsk
< D_UP_TO_DATE
) &&
1225 (ns
.role
== R_PRIMARY
&& ns
.disk
< D_UP_TO_DATE
&& ns
.pdsk
< D_UP_TO_DATE
))
1226 drbd_khelper(device
, "pri-on-incon-degr");
1228 /* Here we have the actions that are performed after a
1229 state change. This function might sleep */
1232 struct drbd_connection
*connection
= first_peer_device(device
)->connection
;
1233 enum drbd_req_event what
= NOTHING
;
1235 spin_lock_irq(&connection
->req_lock
);
1236 if (os
.conn
< C_CONNECTED
&& conn_lowest_conn(connection
) >= C_CONNECTED
)
1239 if ((os
.disk
== D_ATTACHING
|| os
.disk
== D_NEGOTIATING
) &&
1240 conn_lowest_disk(connection
) > D_NEGOTIATING
)
1241 what
= RESTART_FROZEN_DISK_IO
;
1243 if (connection
->susp_nod
&& what
!= NOTHING
) {
1244 _tl_restart(connection
, what
);
1245 _conn_request_state(connection
,
1246 (union drbd_state
) { { .susp_nod
= 1 } },
1247 (union drbd_state
) { { .susp_nod
= 0 } },
1250 spin_unlock_irq(&connection
->req_lock
);
1254 struct drbd_connection
*connection
= first_peer_device(device
)->connection
;
1256 spin_lock_irq(&connection
->req_lock
);
1257 if (connection
->susp_fen
&& conn_lowest_conn(connection
) >= C_CONNECTED
) {
1258 /* case2: The connection was established again: */
1259 struct drbd_device
*odev
;
1263 idr_for_each_entry(&connection
->volumes
, odev
, vnr
)
1264 clear_bit(NEW_CUR_UUID
, &odev
->flags
);
1266 _tl_restart(connection
, RESEND
);
1267 _conn_request_state(connection
,
1268 (union drbd_state
) { { .susp_fen
= 1 } },
1269 (union drbd_state
) { { .susp_fen
= 0 } },
1272 spin_unlock_irq(&connection
->req_lock
);
1275 /* Became sync source. With protocol >= 96, we still need to send out
1276 * the sync uuid now. Need to do that before any drbd_send_state, or
1277 * the other side may go "paused sync" before receiving the sync uuids,
1278 * which is unexpected. */
1279 if ((os
.conn
!= C_SYNC_SOURCE
&& os
.conn
!= C_PAUSED_SYNC_S
) &&
1280 (ns
.conn
== C_SYNC_SOURCE
|| ns
.conn
== C_PAUSED_SYNC_S
) &&
1281 first_peer_device(device
)->connection
->agreed_pro_version
>= 96 && get_ldev(device
)) {
1282 drbd_gen_and_send_sync_uuid(device
);
1286 /* Do not change the order of the if above and the two below... */
1287 if (os
.pdsk
== D_DISKLESS
&&
1288 ns
.pdsk
> D_DISKLESS
&& ns
.pdsk
!= D_UNKNOWN
) { /* attach on the peer */
1289 /* we probably will start a resync soon.
1290 * make sure those things are properly reset. */
1291 device
->rs_total
= 0;
1292 device
->rs_failed
= 0;
1293 atomic_set(&device
->rs_pending_cnt
, 0);
1294 drbd_rs_cancel_all(device
);
1296 drbd_send_uuids(device
);
1297 drbd_send_state(device
, ns
);
1299 /* No point in queuing send_bitmap if we don't have a connection
1300 * anymore, so check also the _current_ state, not only the new state
1301 * at the time this work was queued. */
1302 if (os
.conn
!= C_WF_BITMAP_S
&& ns
.conn
== C_WF_BITMAP_S
&&
1303 device
->state
.conn
== C_WF_BITMAP_S
)
1304 drbd_queue_bitmap_io(device
, &drbd_send_bitmap
, NULL
,
1305 "send_bitmap (WFBitMapS)",
1306 BM_LOCKED_TEST_ALLOWED
);
1308 /* Lost contact to peer's copy of the data */
1309 if ((os
.pdsk
>= D_INCONSISTENT
&&
1310 os
.pdsk
!= D_UNKNOWN
&&
1311 os
.pdsk
!= D_OUTDATED
)
1312 && (ns
.pdsk
< D_INCONSISTENT
||
1313 ns
.pdsk
== D_UNKNOWN
||
1314 ns
.pdsk
== D_OUTDATED
)) {
1315 if (get_ldev(device
)) {
1316 if ((ns
.role
== R_PRIMARY
|| ns
.peer
== R_PRIMARY
) &&
1317 device
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && ns
.disk
>= D_UP_TO_DATE
) {
1318 if (drbd_suspended(device
)) {
1319 set_bit(NEW_CUR_UUID
, &device
->flags
);
1321 drbd_uuid_new_current(device
);
1322 drbd_send_uuids(device
);
1329 if (ns
.pdsk
< D_INCONSISTENT
&& get_ldev(device
)) {
1330 if (os
.peer
== R_SECONDARY
&& ns
.peer
== R_PRIMARY
&&
1331 device
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && ns
.disk
>= D_UP_TO_DATE
) {
1332 drbd_uuid_new_current(device
);
1333 drbd_send_uuids(device
);
1335 /* D_DISKLESS Peer becomes secondary */
1336 if (os
.peer
== R_PRIMARY
&& ns
.peer
== R_SECONDARY
)
1337 /* We may still be Primary ourselves.
1338 * No harm done if the bitmap still changes,
1339 * redirtied pages will follow later. */
1340 drbd_bitmap_io_from_worker(device
, &drbd_bm_write
,
1341 "demote diskless peer", BM_LOCKED_SET_ALLOWED
);
1345 /* Write out all changed bits on demote.
1346 * Though, no need to da that just yet
1347 * if there is a resync going on still */
1348 if (os
.role
== R_PRIMARY
&& ns
.role
== R_SECONDARY
&&
1349 device
->state
.conn
<= C_CONNECTED
&& get_ldev(device
)) {
1350 /* No changes to the bitmap expected this time, so assert that,
1351 * even though no harm was done if it did change. */
1352 drbd_bitmap_io_from_worker(device
, &drbd_bm_write
,
1353 "demote", BM_LOCKED_TEST_ALLOWED
);
1357 /* Last part of the attaching process ... */
1358 if (ns
.conn
>= C_CONNECTED
&&
1359 os
.disk
== D_ATTACHING
&& ns
.disk
== D_NEGOTIATING
) {
1360 drbd_send_sizes(device
, 0, 0); /* to start sync... */
1361 drbd_send_uuids(device
);
1362 drbd_send_state(device
, ns
);
1365 /* We want to pause/continue resync, tell peer. */
1366 if (ns
.conn
>= C_CONNECTED
&&
1367 ((os
.aftr_isp
!= ns
.aftr_isp
) ||
1368 (os
.user_isp
!= ns
.user_isp
)))
1369 drbd_send_state(device
, ns
);
1371 /* In case one of the isp bits got set, suspend other devices. */
1372 if ((!os
.aftr_isp
&& !os
.peer_isp
&& !os
.user_isp
) &&
1373 (ns
.aftr_isp
|| ns
.peer_isp
|| ns
.user_isp
))
1374 suspend_other_sg(device
);
1376 /* Make sure the peer gets informed about eventual state
1377 changes (ISP bits) while we were in WFReportParams. */
1378 if (os
.conn
== C_WF_REPORT_PARAMS
&& ns
.conn
>= C_CONNECTED
)
1379 drbd_send_state(device
, ns
);
1381 if (os
.conn
!= C_AHEAD
&& ns
.conn
== C_AHEAD
)
1382 drbd_send_state(device
, ns
);
1384 /* We are in the progress to start a full sync... */
1385 if ((os
.conn
!= C_STARTING_SYNC_T
&& ns
.conn
== C_STARTING_SYNC_T
) ||
1386 (os
.conn
!= C_STARTING_SYNC_S
&& ns
.conn
== C_STARTING_SYNC_S
))
1387 /* no other bitmap changes expected during this phase */
1388 drbd_queue_bitmap_io(device
,
1389 &drbd_bmio_set_n_write
, &abw_start_sync
,
1390 "set_n_write from StartingSync", BM_LOCKED_TEST_ALLOWED
);
1392 /* first half of local IO error, failure to attach,
1393 * or administrative detach */
1394 if (os
.disk
!= D_FAILED
&& ns
.disk
== D_FAILED
) {
1395 enum drbd_io_error_p eh
= EP_PASS_ON
;
1396 int was_io_error
= 0;
1397 /* corresponding get_ldev was in __drbd_set_state, to serialize
1398 * our cleanup here with the transition to D_DISKLESS.
1399 * But is is still not save to dreference ldev here, since
1400 * we might come from an failed Attach before ldev was set. */
1403 eh
= rcu_dereference(device
->ldev
->disk_conf
)->on_io_error
;
1406 was_io_error
= test_and_clear_bit(WAS_IO_ERROR
, &device
->flags
);
1408 if (was_io_error
&& eh
== EP_CALL_HELPER
)
1409 drbd_khelper(device
, "local-io-error");
1411 /* Immediately allow completion of all application IO,
1412 * that waits for completion from the local disk,
1413 * if this was a force-detach due to disk_timeout
1414 * or administrator request (drbdsetup detach --force).
1415 * Do NOT abort otherwise.
1416 * Aborting local requests may cause serious problems,
1417 * if requests are completed to upper layers already,
1418 * and then later the already submitted local bio completes.
1419 * This can cause DMA into former bio pages that meanwhile
1420 * have been re-used for other things.
1421 * So aborting local requests may cause crashes,
1422 * or even worse, silent data corruption.
1424 if (test_and_clear_bit(FORCE_DETACH
, &device
->flags
))
1425 tl_abort_disk_io(device
);
1427 /* current state still has to be D_FAILED,
1428 * there is only one way out: to D_DISKLESS,
1429 * and that may only happen after our put_ldev below. */
1430 if (device
->state
.disk
!= D_FAILED
)
1432 "ASSERT FAILED: disk is %s during detach\n",
1433 drbd_disk_str(device
->state
.disk
));
1435 if (ns
.conn
>= C_CONNECTED
)
1436 drbd_send_state(device
, ns
);
1438 drbd_rs_cancel_all(device
);
1440 /* In case we want to get something to stable storage still,
1441 * this may be the last chance.
1442 * Following put_ldev may transition to D_DISKLESS. */
1443 drbd_md_sync(device
);
1448 /* second half of local IO error, failure to attach,
1449 * or administrative detach,
1450 * after local_cnt references have reached zero again */
1451 if (os
.disk
!= D_DISKLESS
&& ns
.disk
== D_DISKLESS
) {
1452 /* We must still be diskless,
1453 * re-attach has to be serialized with this! */
1454 if (device
->state
.disk
!= D_DISKLESS
)
1456 "ASSERT FAILED: disk is %s while going diskless\n",
1457 drbd_disk_str(device
->state
.disk
));
1459 if (ns
.conn
>= C_CONNECTED
)
1460 drbd_send_state(device
, ns
);
1461 /* corresponding get_ldev in __drbd_set_state
1462 * this may finally trigger drbd_ldev_destroy. */
1466 /* Notify peer that I had a local IO error, and did not detached.. */
1467 if (os
.disk
== D_UP_TO_DATE
&& ns
.disk
== D_INCONSISTENT
&& ns
.conn
>= C_CONNECTED
)
1468 drbd_send_state(device
, ns
);
1470 /* Disks got bigger while they were detached */
1471 if (ns
.disk
> D_NEGOTIATING
&& ns
.pdsk
> D_NEGOTIATING
&&
1472 test_and_clear_bit(RESYNC_AFTER_NEG
, &device
->flags
)) {
1473 if (ns
.conn
== C_CONNECTED
)
1474 resync_after_online_grow(device
);
1477 /* A resync finished or aborted, wake paused devices... */
1478 if ((os
.conn
> C_CONNECTED
&& ns
.conn
<= C_CONNECTED
) ||
1479 (os
.peer_isp
&& !ns
.peer_isp
) ||
1480 (os
.user_isp
&& !ns
.user_isp
))
1481 resume_next_sg(device
);
1483 /* sync target done with resync. Explicitly notify peer, even though
1484 * it should (at least for non-empty resyncs) already know itself. */
1485 if (os
.disk
< D_UP_TO_DATE
&& os
.conn
>= C_SYNC_SOURCE
&& ns
.conn
== C_CONNECTED
)
1486 drbd_send_state(device
, ns
);
1488 /* Verify finished, or reached stop sector. Peer did not know about
1489 * the stop sector, and we may even have changed the stop sector during
1490 * verify to interrupt/stop early. Send the new state. */
1491 if (os
.conn
== C_VERIFY_S
&& ns
.conn
== C_CONNECTED
1492 && verify_can_do_stop_sector(device
))
1493 drbd_send_state(device
, ns
);
1495 /* This triggers bitmap writeout of potentially still unwritten pages
1496 * if the resync finished cleanly, or aborted because of peer disk
1497 * failure, or because of connection loss.
1498 * For resync aborted because of local disk failure, we cannot do
1499 * any bitmap writeout anymore.
1500 * No harm done if some bits change during this phase.
1502 if (os
.conn
> C_CONNECTED
&& ns
.conn
<= C_CONNECTED
&& get_ldev(device
)) {
1503 drbd_queue_bitmap_io(device
, &drbd_bm_write_copy_pages
, NULL
,
1504 "write from resync_finished", BM_LOCKED_CHANGE_ALLOWED
);
1508 if (ns
.disk
== D_DISKLESS
&&
1509 ns
.conn
== C_STANDALONE
&&
1510 ns
.role
== R_SECONDARY
) {
1511 if (os
.aftr_isp
!= ns
.aftr_isp
)
1512 resume_next_sg(device
);
1515 drbd_md_sync(device
);
1518 struct after_conn_state_chg_work
{
1521 union drbd_state ns_min
;
1522 union drbd_state ns_max
; /* new, max state, over all devices */
1523 enum chg_state_flags flags
;
1526 static int w_after_conn_state_ch(struct drbd_work
*w
, int unused
)
1528 struct after_conn_state_chg_work
*acscw
=
1529 container_of(w
, struct after_conn_state_chg_work
, w
);
1530 struct drbd_connection
*connection
= w
->connection
;
1531 enum drbd_conns oc
= acscw
->oc
;
1532 union drbd_state ns_max
= acscw
->ns_max
;
1533 struct drbd_device
*device
;
1538 /* Upon network configuration, we need to start the receiver */
1539 if (oc
== C_STANDALONE
&& ns_max
.conn
== C_UNCONNECTED
)
1540 drbd_thread_start(&connection
->receiver
);
1542 if (oc
== C_DISCONNECTING
&& ns_max
.conn
== C_STANDALONE
) {
1543 struct net_conf
*old_conf
;
1545 mutex_lock(&connection
->conf_update
);
1546 old_conf
= connection
->net_conf
;
1547 connection
->my_addr_len
= 0;
1548 connection
->peer_addr_len
= 0;
1549 rcu_assign_pointer(connection
->net_conf
, NULL
);
1550 conn_free_crypto(connection
);
1551 mutex_unlock(&connection
->conf_update
);
1557 if (ns_max
.susp_fen
) {
1558 /* case1: The outdate peer handler is successful: */
1559 if (ns_max
.pdsk
<= D_OUTDATED
) {
1561 idr_for_each_entry(&connection
->volumes
, device
, vnr
) {
1562 if (test_bit(NEW_CUR_UUID
, &device
->flags
)) {
1563 drbd_uuid_new_current(device
);
1564 clear_bit(NEW_CUR_UUID
, &device
->flags
);
1568 spin_lock_irq(&connection
->req_lock
);
1569 _tl_restart(connection
, CONNECTION_LOST_WHILE_PENDING
);
1570 _conn_request_state(connection
,
1571 (union drbd_state
) { { .susp_fen
= 1 } },
1572 (union drbd_state
) { { .susp_fen
= 0 } },
1574 spin_unlock_irq(&connection
->req_lock
);
1577 kref_put(&connection
->kref
, &conn_destroy
);
1579 conn_md_sync(connection
);
1584 void conn_old_common_state(struct drbd_connection
*connection
, union drbd_state
*pcs
, enum chg_state_flags
*pf
)
1586 enum chg_state_flags flags
= ~0;
1587 struct drbd_device
*device
;
1588 int vnr
, first_vol
= 1;
1589 union drbd_dev_state os
, cs
= {
1590 { .role
= R_SECONDARY
,
1592 .conn
= connection
->cstate
,
1598 idr_for_each_entry(&connection
->volumes
, device
, vnr
) {
1607 if (cs
.role
!= os
.role
)
1608 flags
&= ~CS_DC_ROLE
;
1610 if (cs
.peer
!= os
.peer
)
1611 flags
&= ~CS_DC_PEER
;
1613 if (cs
.conn
!= os
.conn
)
1614 flags
&= ~CS_DC_CONN
;
1616 if (cs
.disk
!= os
.disk
)
1617 flags
&= ~CS_DC_DISK
;
1619 if (cs
.pdsk
!= os
.pdsk
)
1620 flags
&= ~CS_DC_PDSK
;
1629 static enum drbd_state_rv
1630 conn_is_valid_transition(struct drbd_connection
*connection
, union drbd_state mask
, union drbd_state val
,
1631 enum chg_state_flags flags
)
1633 enum drbd_state_rv rv
= SS_SUCCESS
;
1634 union drbd_state ns
, os
;
1635 struct drbd_device
*device
;
1639 idr_for_each_entry(&connection
->volumes
, device
, vnr
) {
1640 os
= drbd_read_state(device
);
1641 ns
= sanitize_state(device
, apply_mask_val(os
, mask
, val
), NULL
);
1643 if (flags
& CS_IGN_OUTD_FAIL
&& ns
.disk
== D_OUTDATED
&& os
.disk
< D_OUTDATED
)
1649 rv
= is_valid_transition(os
, ns
);
1650 if (rv
< SS_SUCCESS
)
1653 if (!(flags
& CS_HARD
)) {
1654 rv
= is_valid_state(device
, ns
);
1655 if (rv
< SS_SUCCESS
) {
1656 if (is_valid_state(device
, os
) == rv
)
1657 rv
= is_valid_soft_transition(os
, ns
, connection
);
1659 rv
= is_valid_soft_transition(os
, ns
, connection
);
1661 if (rv
< SS_SUCCESS
)
1666 if (rv
< SS_SUCCESS
&& flags
& CS_VERBOSE
)
1667 print_st_err(device
, os
, ns
, rv
);
1673 conn_set_state(struct drbd_connection
*connection
, union drbd_state mask
, union drbd_state val
,
1674 union drbd_state
*pns_min
, union drbd_state
*pns_max
, enum chg_state_flags flags
)
1676 union drbd_state ns
, os
, ns_max
= { };
1677 union drbd_state ns_min
= {
1684 struct drbd_device
*device
;
1685 enum drbd_state_rv rv
;
1686 int vnr
, number_of_volumes
= 0;
1688 if (mask
.conn
== C_MASK
) {
1689 /* remember last connect time so request_timer_fn() won't
1690 * kill newly established sessions while we are still trying to thaw
1691 * previously frozen IO */
1692 if (connection
->cstate
!= C_WF_REPORT_PARAMS
&& val
.conn
== C_WF_REPORT_PARAMS
)
1693 connection
->last_reconnect_jif
= jiffies
;
1695 connection
->cstate
= val
.conn
;
1699 idr_for_each_entry(&connection
->volumes
, device
, vnr
) {
1700 number_of_volumes
++;
1701 os
= drbd_read_state(device
);
1702 ns
= apply_mask_val(os
, mask
, val
);
1703 ns
= sanitize_state(device
, ns
, NULL
);
1705 if (flags
& CS_IGN_OUTD_FAIL
&& ns
.disk
== D_OUTDATED
&& os
.disk
< D_OUTDATED
)
1708 rv
= __drbd_set_state(device
, ns
, flags
, NULL
);
1709 if (rv
< SS_SUCCESS
)
1712 ns
.i
= device
->state
.i
;
1713 ns_max
.role
= max_role(ns
.role
, ns_max
.role
);
1714 ns_max
.peer
= max_role(ns
.peer
, ns_max
.peer
);
1715 ns_max
.conn
= max_t(enum drbd_conns
, ns
.conn
, ns_max
.conn
);
1716 ns_max
.disk
= max_t(enum drbd_disk_state
, ns
.disk
, ns_max
.disk
);
1717 ns_max
.pdsk
= max_t(enum drbd_disk_state
, ns
.pdsk
, ns_max
.pdsk
);
1719 ns_min
.role
= min_role(ns
.role
, ns_min
.role
);
1720 ns_min
.peer
= min_role(ns
.peer
, ns_min
.peer
);
1721 ns_min
.conn
= min_t(enum drbd_conns
, ns
.conn
, ns_min
.conn
);
1722 ns_min
.disk
= min_t(enum drbd_disk_state
, ns
.disk
, ns_min
.disk
);
1723 ns_min
.pdsk
= min_t(enum drbd_disk_state
, ns
.pdsk
, ns_min
.pdsk
);
1727 if (number_of_volumes
== 0) {
1728 ns_min
= ns_max
= (union drbd_state
) { {
1729 .role
= R_SECONDARY
,
1737 ns_min
.susp
= ns_max
.susp
= connection
->susp
;
1738 ns_min
.susp_nod
= ns_max
.susp_nod
= connection
->susp_nod
;
1739 ns_min
.susp_fen
= ns_max
.susp_fen
= connection
->susp_fen
;
1745 static enum drbd_state_rv
1746 _conn_rq_cond(struct drbd_connection
*connection
, union drbd_state mask
, union drbd_state val
)
1748 enum drbd_state_rv rv
;
1750 if (test_and_clear_bit(CONN_WD_ST_CHG_OKAY
, &connection
->flags
))
1751 return SS_CW_SUCCESS
;
1753 if (test_and_clear_bit(CONN_WD_ST_CHG_FAIL
, &connection
->flags
))
1754 return SS_CW_FAILED_BY_PEER
;
1756 rv
= conn_is_valid_transition(connection
, mask
, val
, 0);
1757 if (rv
== SS_SUCCESS
&& connection
->cstate
== C_WF_REPORT_PARAMS
)
1758 rv
= SS_UNKNOWN_ERROR
; /* continue waiting */
1764 _conn_request_state(struct drbd_connection
*connection
, union drbd_state mask
, union drbd_state val
,
1765 enum chg_state_flags flags
)
1767 enum drbd_state_rv rv
= SS_SUCCESS
;
1768 struct after_conn_state_chg_work
*acscw
;
1769 enum drbd_conns oc
= connection
->cstate
;
1770 union drbd_state ns_max
, ns_min
, os
;
1771 bool have_mutex
= false;
1774 rv
= is_valid_conn_transition(oc
, val
.conn
);
1775 if (rv
< SS_SUCCESS
)
1779 rv
= conn_is_valid_transition(connection
, mask
, val
, flags
);
1780 if (rv
< SS_SUCCESS
)
1783 if (oc
== C_WF_REPORT_PARAMS
&& val
.conn
== C_DISCONNECTING
&&
1784 !(flags
& (CS_LOCAL_ONLY
| CS_HARD
))) {
1786 /* This will be a cluster-wide state change.
1787 * Need to give up the spinlock, grab the mutex,
1788 * then send the state change request, ... */
1789 spin_unlock_irq(&connection
->req_lock
);
1790 mutex_lock(&connection
->cstate_mutex
);
1793 set_bit(CONN_WD_ST_CHG_REQ
, &connection
->flags
);
1794 if (conn_send_state_req(connection
, mask
, val
)) {
1795 /* sending failed. */
1796 clear_bit(CONN_WD_ST_CHG_REQ
, &connection
->flags
);
1797 rv
= SS_CW_FAILED_BY_PEER
;
1798 /* need to re-aquire the spin lock, though */
1799 goto abort_unlocked
;
1802 if (val
.conn
== C_DISCONNECTING
)
1803 set_bit(DISCONNECT_SENT
, &connection
->flags
);
1805 /* ... and re-aquire the spinlock.
1806 * If _conn_rq_cond() returned >= SS_SUCCESS, we must call
1807 * conn_set_state() within the same spinlock. */
1808 spin_lock_irq(&connection
->req_lock
);
1809 wait_event_lock_irq(connection
->ping_wait
,
1810 (rv
= _conn_rq_cond(connection
, mask
, val
)),
1811 connection
->req_lock
);
1812 clear_bit(CONN_WD_ST_CHG_REQ
, &connection
->flags
);
1813 if (rv
< SS_SUCCESS
)
1817 conn_old_common_state(connection
, &os
, &flags
);
1818 flags
|= CS_DC_SUSP
;
1819 conn_set_state(connection
, mask
, val
, &ns_min
, &ns_max
, flags
);
1820 conn_pr_state_change(connection
, os
, ns_max
, flags
);
1822 acscw
= kmalloc(sizeof(*acscw
), GFP_ATOMIC
);
1824 acscw
->oc
= os
.conn
;
1825 acscw
->ns_min
= ns_min
;
1826 acscw
->ns_max
= ns_max
;
1827 acscw
->flags
= flags
;
1828 acscw
->w
.cb
= w_after_conn_state_ch
;
1829 kref_get(&connection
->kref
);
1830 acscw
->w
.connection
= connection
;
1831 drbd_queue_work(&connection
->sender_work
, &acscw
->w
);
1833 conn_err(connection
, "Could not kmalloc an acscw\n");
1838 /* mutex_unlock() "... must not be used in interrupt context.",
1839 * so give up the spinlock, then re-aquire it */
1840 spin_unlock_irq(&connection
->req_lock
);
1842 mutex_unlock(&connection
->cstate_mutex
);
1843 spin_lock_irq(&connection
->req_lock
);
1845 if (rv
< SS_SUCCESS
&& flags
& CS_VERBOSE
) {
1846 conn_err(connection
, "State change failed: %s\n", drbd_set_st_err_str(rv
));
1847 conn_err(connection
, " mask = 0x%x val = 0x%x\n", mask
.i
, val
.i
);
1848 conn_err(connection
, " old_conn:%s wanted_conn:%s\n", drbd_conn_str(oc
), drbd_conn_str(val
.conn
));
1854 conn_request_state(struct drbd_connection
*connection
, union drbd_state mask
, union drbd_state val
,
1855 enum chg_state_flags flags
)
1857 enum drbd_state_rv rv
;
1859 spin_lock_irq(&connection
->req_lock
);
1860 rv
= _conn_request_state(connection
, mask
, val
, flags
);
1861 spin_unlock_irq(&connection
->req_lock
);