[deliverable/linux.git] / fs / dlm / member.c

/******************************************************************************
*******************************************************************************
**
**  Copyright (C) 2005-2008 Red Hat, Inc.  All rights reserved.
**
**  This copyrighted material is made available to anyone wishing to use,
**  modify, copy, or redistribute it subject to the terms and conditions
**  of the GNU General Public License v.2.
**
*******************************************************************************
******************************************************************************/

#include "dlm_internal.h"
#include "lockspace.h"
#include "member.h"
#include "recoverd.h"
#include "recover.h"
#include "rcom.h"
#include "config.h"

static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new)
{
	struct dlm_member *memb = NULL;
	struct list_head *tmp;
	struct list_head *newlist = &new->list;
	struct list_head *head = &ls->ls_nodes;

	list_for_each(tmp, head) {
		memb = list_entry(tmp, struct dlm_member, list);
		if (new->nodeid < memb->nodeid)
			break;
	}

	if (!memb)
		list_add_tail(newlist, head);
	else {
		/* FIXME: can use list macro here */
		newlist->prev = tmp->prev;
		newlist->next = tmp;
		tmp->prev->next = newlist;
		tmp->prev = newlist;
	}
}

static int dlm_add_member(struct dlm_ls *ls, int nodeid)
{
	struct dlm_member *memb;
	int w;

	memb = kzalloc(sizeof(struct dlm_member), GFP_KERNEL);
	if (!memb)
		return -ENOMEM;

	w = dlm_node_weight(ls->ls_name, nodeid);
	if (w < 0) {
		kfree(memb);
		return w;
	}

	memb->nodeid = nodeid;
	memb->weight = w;
	add_ordered_member(ls, memb);
	ls->ls_num_nodes++;
	return 0;
}

static void dlm_remove_member(struct dlm_ls *ls, struct dlm_member *memb)
{
	list_move(&memb->list, &ls->ls_nodes_gone);
	ls->ls_num_nodes--;
}

int dlm_is_member(struct dlm_ls *ls, int nodeid)
{
	struct dlm_member *memb;

	list_for_each_entry(memb, &ls->ls_nodes, list) {
		if (memb->nodeid == nodeid)
			return 1;
	}
	return 0;
}

int dlm_is_removed(struct dlm_ls *ls, int nodeid)
{
	struct dlm_member *memb;

	list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
		if (memb->nodeid == nodeid)
			return 1;
	}
	return 0;
}

static void clear_memb_list(struct list_head *head)
{
	struct dlm_member *memb;

	while (!list_empty(head)) {
		memb = list_entry(head->next, struct dlm_member, list);
		list_del(&memb->list);
		kfree(memb);
	}
}

void dlm_clear_members(struct dlm_ls *ls)
{
	clear_memb_list(&ls->ls_nodes);
	ls->ls_num_nodes = 0;
}

void dlm_clear_members_gone(struct dlm_ls *ls)
{
	clear_memb_list(&ls->ls_nodes_gone);
}

static void make_member_array(struct dlm_ls *ls)
{
	struct dlm_member *memb;
	int i, w, x = 0, total = 0, all_zero = 0, *array;

	kfree(ls->ls_node_array);
	ls->ls_node_array = NULL;

	list_for_each_entry(memb, &ls->ls_nodes, list) {
		if (memb->weight)
			total += memb->weight;
	}

	/* all nodes revert to weight of 1 if all have weight 0 */

	if (!total) {
		total = ls->ls_num_nodes;
		all_zero = 1;
	}

	ls->ls_total_weight = total;

	array = kmalloc(sizeof(int) * total, GFP_KERNEL);
	if (!array)
		return;

	list_for_each_entry(memb, &ls->ls_nodes, list) {
		if (!all_zero && !memb->weight)
			continue;

		if (all_zero)
			w = 1;
		else
			w = memb->weight;

		DLM_ASSERT(x < total, printk("total %d x %d\n", total, x););

		for (i = 0; i < w; i++)
			array[x++] = memb->nodeid;
	}

	ls->ls_node_array = array;
}

/* send a status request to all members just to establish comms connections */

static int ping_members(struct dlm_ls *ls)
{
	struct dlm_member *memb;
	int error = 0;

	list_for_each_entry(memb, &ls->ls_nodes, list) {
		error = dlm_recovery_stopped(ls);
		if (error)
			break;
		error = dlm_rcom_status(ls, memb->nodeid);
		if (error)
			break;
	}
	if (error)
		log_debug(ls, "ping_members aborted %d last nodeid %d",
			  error, ls->ls_recover_nodeid);
	return error;
}

int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out)
{
	struct dlm_member *memb, *safe;
	int i, error, found, pos = 0, neg = 0, low = -1;

	/* previously removed members that we've not finished removing need to
	   count as a negative change so the "neg" recovery steps will happen */

	list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
		log_debug(ls, "prev removed member %d", memb->nodeid);
		neg++;
	}

	/* move departed members from ls_nodes to ls_nodes_gone */

	list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) {
		found = 0;
		for (i = 0; i < rv->node_count; i++) {
			if (memb->nodeid == rv->nodeids[i]) {
				found = 1;
				break;
			}
		}

		if (!found) {
			neg++;
			dlm_remove_member(ls, memb);
			log_debug(ls, "remove member %d", memb->nodeid);
		}
	}

	/* Add an entry to ls_nodes_gone for members that were removed and
	   then added again, so that previous state for these nodes will be
	   cleared during recovery. */

	for (i = 0; i < rv->new_count; i++) {
		if (!dlm_is_member(ls, rv->new[i]))
			continue;
		log_debug(ls, "new nodeid %d is a re-added member", rv->new[i]);

		memb = kzalloc(sizeof(struct dlm_member), GFP_KERNEL);
		if (!memb)
			return -ENOMEM;
		memb->nodeid = rv->new[i];
		list_add_tail(&memb->list, &ls->ls_nodes_gone);
		neg++;
	}

	/* add new members to ls_nodes */

	for (i = 0; i < rv->node_count; i++) {
		if (dlm_is_member(ls, rv->nodeids[i]))
			continue;
		dlm_add_member(ls, rv->nodeids[i]);
		pos++;
		log_debug(ls, "add member %d", rv->nodeids[i]);
	}

	list_for_each_entry(memb, &ls->ls_nodes, list) {
		if (low == -1 || memb->nodeid < low)
			low = memb->nodeid;
	}
	ls->ls_low_nodeid = low;

	make_member_array(ls);
	dlm_set_recover_status(ls, DLM_RS_NODES);
	*neg_out = neg;

	error = ping_members(ls);
	if (!error || error == -EPROTO) {
		/* new_lockspace() may be waiting to know if the config
		   is good or bad */
		ls->ls_members_result = error;
		complete(&ls->ls_members_done);
	}
	if (error)
		goto out;

	error = dlm_recover_members_wait(ls);
 out:
	log_debug(ls, "total members %d error %d", ls->ls_num_nodes, error);
	return error;
}

/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before
   dlm_ls_start() is called on any of them to start the new recovery. */

int dlm_ls_stop(struct dlm_ls *ls)
{
	int new;

	/*
	 * Prevent dlm_recv from being in the middle of something when we do
	 * the stop.  This includes ensuring dlm_recv isn't processing a
	 * recovery message (rcom), while dlm_recoverd is aborting and
	 * resetting things from an in-progress recovery.  i.e. we want
	 * dlm_recoverd to abort its recovery without worrying about dlm_recv
	 * processing an rcom at the same time.  Stopping dlm_recv also makes
	 * it easy for dlm_receive_message() to check locking stopped and add a
	 * message to the requestqueue without races.
	 */

	down_write(&ls->ls_recv_active);

	/*
	 * Abort any recovery that's in progress (see RECOVERY_STOP,
	 * dlm_recovery_stopped()) and tell any other threads running in the
	 * dlm to quit any processing (see RUNNING, dlm_locking_stopped()).
	 */

	spin_lock(&ls->ls_recover_lock);
	set_bit(LSFL_RECOVERY_STOP, &ls->ls_flags);
	new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags);
	ls->ls_recover_seq++;
	spin_unlock(&ls->ls_recover_lock);

	/*
	 * Let dlm_recv run again, now any normal messages will be saved on the
	 * requestqueue for later.
	 */

	up_write(&ls->ls_recv_active);

	/*
	 * This in_recovery lock does two things:
	 * 1) Keeps this function from returning until all threads are out
	 *    of locking routines and locking is truely stopped.
	 * 2) Keeps any new requests from being processed until it's unlocked
	 *    when recovery is complete.
	 */

	if (new)
		down_write(&ls->ls_in_recovery);

	/*
	 * The recoverd suspend/resume makes sure that dlm_recoverd (if
	 * running) has noticed RECOVERY_STOP above and quit processing the
	 * previous recovery.
	 */

	dlm_recoverd_suspend(ls);
	ls->ls_recover_status = 0;
	dlm_recoverd_resume(ls);

	if (!ls->ls_recover_begin)
		ls->ls_recover_begin = jiffies;
	return 0;
}

int dlm_ls_start(struct dlm_ls *ls)
{
	struct dlm_recover *rv = NULL, *rv_old;
	int *ids = NULL, *new = NULL;
	int error, ids_count = 0, new_count = 0;

	rv = kzalloc(sizeof(struct dlm_recover), GFP_KERNEL);
	if (!rv)
		return -ENOMEM;

	error = dlm_nodeid_list(ls->ls_name, &ids, &ids_count,
				&new, &new_count);
	if (error < 0)
		goto fail;

	spin_lock(&ls->ls_recover_lock);

	/* the lockspace needs to be stopped before it can be started */

	if (!dlm_locking_stopped(ls)) {
		spin_unlock(&ls->ls_recover_lock);
		log_error(ls, "start ignored: lockspace running");
		error = -EINVAL;
		goto fail;
	}

	rv->nodeids = ids;
	rv->node_count = ids_count;
	rv->new = new;
	rv->new_count = new_count;
	rv->seq = ++ls->ls_recover_seq;
	rv_old = ls->ls_recover_args;
	ls->ls_recover_args = rv;
	spin_unlock(&ls->ls_recover_lock);

	if (rv_old) {
		log_error(ls, "unused recovery %llx %d",
			  (unsigned long long)rv_old->seq, rv_old->node_count);
		kfree(rv_old->nodeids);
		kfree(rv_old->new);
		kfree(rv_old);
	}

	dlm_recoverd_kick(ls);
	return 0;

 fail:
	kfree(rv);
	kfree(ids);
	kfree(new);
	return error;
}
Commit	Line	Data
e7fd4179 DT	1	/******************************************************************************
	2	*******************************************************************************
	3	**
46b43eed	4	** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved.
e7fd4179 DT	5	**
	6	** This copyrighted material is made available to anyone wishing to use,
	7	** modify, copy, or redistribute it subject to the terms and conditions
	8	** of the GNU General Public License v.2.
	9	**
	10	*******************************************************************************
	11	******************************************************************************/
	12
	13	#include "dlm_internal.h"
	14	#include "lockspace.h"
	15	#include "member.h"
	16	#include "recoverd.h"
	17	#include "recover.h"
e7fd4179 DT	18	#include "rcom.h"
	19	#include "config.h"
	20
e7fd4179 DT	21	static void add_ordered_member(struct dlm_ls ls, struct dlm_member new)
	22	{
	23	struct dlm_member *memb = NULL;
	24	struct list_head *tmp;
	25	struct list_head *newlist = &new->list;
	26	struct list_head *head = &ls->ls_nodes;
	27
	28	list_for_each(tmp, head) {
	29	memb = list_entry(tmp, struct dlm_member, list);
	30	if (new->nodeid < memb->nodeid)
	31	break;
	32	}
	33
	34	if (!memb)
	35	list_add_tail(newlist, head);
	36	else {
	37	/* FIXME: can use list macro here */
	38	newlist->prev = tmp->prev;
	39	newlist->next = tmp;
	40	tmp->prev->next = newlist;
	41	tmp->prev = newlist;
	42	}
	43	}
	44
	45	static int dlm_add_member(struct dlm_ls *ls, int nodeid)
	46	{
	47	struct dlm_member *memb;
	48	int w;
	49
90135925	50	memb = kzalloc(sizeof(struct dlm_member), GFP_KERNEL);
e7fd4179 DT	51	if (!memb)
	52	return -ENOMEM;
	53
	54	w = dlm_node_weight(ls->ls_name, nodeid);
1a2bf2ee JJ	55	if (w < 0) {
1a2bf2ee JJ	56	kfree(memb);
e7fd4179	57	return w;
1a2bf2ee	58	}
e7fd4179 DT	59
	60	memb->nodeid = nodeid;
	61	memb->weight = w;
	62	add_ordered_member(ls, memb);
	63	ls->ls_num_nodes++;
	64	return 0;
	65	}
	66
	67	static void dlm_remove_member(struct dlm_ls ls, struct dlm_member memb)
	68	{
	69	list_move(&memb->list, &ls->ls_nodes_gone);
	70	ls->ls_num_nodes--;
	71	}
	72
46b43eed	73	int dlm_is_member(struct dlm_ls *ls, int nodeid)
e7fd4179 DT	74	{
	75	struct dlm_member *memb;
	76
	77	list_for_each_entry(memb, &ls->ls_nodes, list) {
	78	if (memb->nodeid == nodeid)
90135925	79	return 1;
e7fd4179	80	}
90135925	81	return 0;
e7fd4179 DT	82	}
	83
	84	int dlm_is_removed(struct dlm_ls *ls, int nodeid)
	85	{
	86	struct dlm_member *memb;
	87
	88	list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
	89	if (memb->nodeid == nodeid)
90135925	90	return 1;
e7fd4179	91	}
90135925	92	return 0;
e7fd4179 DT	93	}
	94
	95	static void clear_memb_list(struct list_head *head)
	96	{
	97	struct dlm_member *memb;
	98
	99	while (!list_empty(head)) {
	100	memb = list_entry(head->next, struct dlm_member, list);
	101	list_del(&memb->list);
	102	kfree(memb);
	103	}
	104	}
	105
	106	void dlm_clear_members(struct dlm_ls *ls)
	107	{
	108	clear_memb_list(&ls->ls_nodes);
	109	ls->ls_num_nodes = 0;
	110	}
	111
	112	void dlm_clear_members_gone(struct dlm_ls *ls)
	113	{
	114	clear_memb_list(&ls->ls_nodes_gone);
	115	}
	116
	117	static void make_member_array(struct dlm_ls *ls)
	118	{
	119	struct dlm_member *memb;
	120	int i, w, x = 0, total = 0, all_zero = 0, *array;
	121
	122	kfree(ls->ls_node_array);
	123	ls->ls_node_array = NULL;
	124
	125	list_for_each_entry(memb, &ls->ls_nodes, list) {
	126	if (memb->weight)
	127	total += memb->weight;
	128	}
	129
	130	/* all nodes revert to weight of 1 if all have weight 0 */
	131
	132	if (!total) {
	133	total = ls->ls_num_nodes;
	134	all_zero = 1;
	135	}
	136
	137	ls->ls_total_weight = total;
	138
	139	array = kmalloc(sizeof(int) * total, GFP_KERNEL);
	140	if (!array)
	141	return;
	142
	143	list_for_each_entry(memb, &ls->ls_nodes, list) {
	144	if (!all_zero && !memb->weight)
	145	continue;
	146
	147	if (all_zero)
	148	w = 1;
	149	else
	150	w = memb->weight;
	151
	152	DLM_ASSERT(x < total, printk("total %d x %d\n", total, x););
	153
	154	for (i = 0; i < w; i++)
	155	array[x++] = memb->nodeid;
	156	}
157
158	ls->ls_node_array = array;
159	}
160
161	/* send a status request to all members just to establish comms connections */
162
f6db1b8e	163	static int ping_members(struct dlm_ls *ls)
e7fd4179 DT	164	{
e7fd4179 DT	165	struct dlm_member *memb;
f6db1b8e DT	166	int error = 0;
	167
	168	list_for_each_entry(memb, &ls->ls_nodes, list) {
	169	error = dlm_recovery_stopped(ls);
	170	if (error)
	171	break;
	172	error = dlm_rcom_status(ls, memb->nodeid);
	173	if (error)
	174	break;
	175	}
	176	if (error)
faa0f267 DT	177	log_debug(ls, "ping_members aborted %d last nodeid %d",
faa0f267 DT	178	error, ls->ls_recover_nodeid);
f6db1b8e	179	return error;
e7fd4179 DT	180	}
	181
	182	int dlm_recover_members(struct dlm_ls ls, struct dlm_recover rv, int *neg_out)
	183	{
	184	struct dlm_member memb, safe;
	185	int i, error, found, pos = 0, neg = 0, low = -1;
	186
91c0dc93 DT	187	/* previously removed members that we've not finished removing need to
	188	count as a negative change so the "neg" recovery steps will happen */
	189
	190	list_for_each_entry(memb, &ls->ls_nodes_gone, list) {
	191	log_debug(ls, "prev removed member %d", memb->nodeid);
	192	neg++;
	193	}
	194
e7fd4179 DT	195	/* move departed members from ls_nodes to ls_nodes_gone */
	196
	197	list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) {
90135925	198	found = 0;
e7fd4179 DT	199	for (i = 0; i < rv->node_count; i++) {
e7fd4179 DT	200	if (memb->nodeid == rv->nodeids[i]) {
90135925	201	found = 1;
e7fd4179 DT	202	break;
	203	}
	204	}
	205
	206	if (!found) {
	207	neg++;
	208	dlm_remove_member(ls, memb);
	209	log_debug(ls, "remove member %d", memb->nodeid);
	210	}
	211	}
	212
d44e0fc7 DT	213	/* Add an entry to ls_nodes_gone for members that were removed and
	214	then added again, so that previous state for these nodes will be
	215	cleared during recovery. */
	216
	217	for (i = 0; i < rv->new_count; i++) {
	218	if (!dlm_is_member(ls, rv->new[i]))
	219	continue;
	220	log_debug(ls, "new nodeid %d is a re-added member", rv->new[i]);
	221
	222	memb = kzalloc(sizeof(struct dlm_member), GFP_KERNEL);
	223	if (!memb)
	224	return -ENOMEM;
	225	memb->nodeid = rv->new[i];
	226	list_add_tail(&memb->list, &ls->ls_nodes_gone);
	227	neg++;
	228	}
	229
e7fd4179 DT	230	/* add new members to ls_nodes */
	231
	232	for (i = 0; i < rv->node_count; i++) {
	233	if (dlm_is_member(ls, rv->nodeids[i]))
	234	continue;
	235	dlm_add_member(ls, rv->nodeids[i]);
	236	pos++;
	237	log_debug(ls, "add member %d", rv->nodeids[i]);
	238	}
	239
	240	list_for_each_entry(memb, &ls->ls_nodes, list) {
	241	if (low == -1 \|\| memb->nodeid < low)
	242	low = memb->nodeid;
	243	}
	244	ls->ls_low_nodeid = low;
	245
	246	make_member_array(ls);
	247	dlm_set_recover_status(ls, DLM_RS_NODES);
	248	*neg_out = neg;
	249
f6db1b8e	250	error = ping_members(ls);
8b0e7b2c DT	251	if (!error \|\| error == -EPROTO) {
	252	/* new_lockspace() may be waiting to know if the config
	253	is good or bad */
	254	ls->ls_members_result = error;
	255	complete(&ls->ls_members_done);
	256	}
f6db1b8e DT	257	if (error)
f6db1b8e DT	258	goto out;
e7fd4179 DT	259
e7fd4179 DT	260	error = dlm_recover_members_wait(ls);
f6db1b8e DT	261	out:
f6db1b8e DT	262	log_debug(ls, "total members %d error %d", ls->ls_num_nodes, error);
e7fd4179 DT	263	return error;
	264	}
	265
c36258b5 DT	266	/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before
c36258b5 DT	267	dlm_ls_start() is called on any of them to start the new recovery. */
e7fd4179 DT	268
	269	int dlm_ls_stop(struct dlm_ls *ls)
	270	{
	271	int new;
	272
	273	/*
c36258b5 DT	274	* Prevent dlm_recv from being in the middle of something when we do
	275	* the stop. This includes ensuring dlm_recv isn't processing a
	276	* recovery message (rcom), while dlm_recoverd is aborting and
	277	* resetting things from an in-progress recovery. i.e. we want
	278	* dlm_recoverd to abort its recovery without worrying about dlm_recv
	279	* processing an rcom at the same time. Stopping dlm_recv also makes
	280	* it easy for dlm_receive_message() to check locking stopped and add a
	281	* message to the requestqueue without races.
	282	*/
	283
	284	down_write(&ls->ls_recv_active);
	285
	286	/*
	287	* Abort any recovery that's in progress (see RECOVERY_STOP,
	288	* dlm_recovery_stopped()) and tell any other threads running in the
	289	* dlm to quit any processing (see RUNNING, dlm_locking_stopped()).
e7fd4179 DT	290	*/
	291
	292	spin_lock(&ls->ls_recover_lock);
	293	set_bit(LSFL_RECOVERY_STOP, &ls->ls_flags);
	294	new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags);
	295	ls->ls_recover_seq++;
	296	spin_unlock(&ls->ls_recover_lock);
	297
c36258b5 DT	298	/*
	299	* Let dlm_recv run again, now any normal messages will be saved on the
	300	* requestqueue for later.
	301	*/
	302
	303	up_write(&ls->ls_recv_active);
	304
e7fd4179 DT	305	/*
e7fd4179 DT	306	* This in_recovery lock does two things:
e7fd4179 DT	307	* 1) Keeps this function from returning until all threads are out
	308	* of locking routines and locking is truely stopped.
	309	* 2) Keeps any new requests from being processed until it's unlocked
	310	* when recovery is complete.
	311	*/
	312
	313	if (new)
	314	down_write(&ls->ls_in_recovery);
	315
	316	/*
	317	* The recoverd suspend/resume makes sure that dlm_recoverd (if
c36258b5 DT	318	* running) has noticed RECOVERY_STOP above and quit processing the
c36258b5 DT	319	* previous recovery.
e7fd4179 DT	320	*/
	321
	322	dlm_recoverd_suspend(ls);
	323	ls->ls_recover_status = 0;
	324	dlm_recoverd_resume(ls);
3ae1acf9 DT	325
	326	if (!ls->ls_recover_begin)
	327	ls->ls_recover_begin = jiffies;
e7fd4179 DT	328	return 0;
	329	}
	330
	331	int dlm_ls_start(struct dlm_ls *ls)
	332	{
	333	struct dlm_recover rv = NULL, rv_old;
d44e0fc7 DT	334	int ids = NULL, new = NULL;
d44e0fc7 DT	335	int error, ids_count = 0, new_count = 0;
e7fd4179	336
90135925	337	rv = kzalloc(sizeof(struct dlm_recover), GFP_KERNEL);
e7fd4179 DT	338	if (!rv)
e7fd4179 DT	339	return -ENOMEM;
e7fd4179	340
d44e0fc7 DT	341	error = dlm_nodeid_list(ls->ls_name, &ids, &ids_count,
	342	&new, &new_count);
	343	if (error < 0)
e7fd4179 DT	344	goto fail;
	345
	346	spin_lock(&ls->ls_recover_lock);
	347
	348	/* the lockspace needs to be stopped before it can be started */
	349
	350	if (!dlm_locking_stopped(ls)) {
	351	spin_unlock(&ls->ls_recover_lock);
	352	log_error(ls, "start ignored: lockspace running");
	353	error = -EINVAL;
	354	goto fail;
	355	}
	356
	357	rv->nodeids = ids;
d44e0fc7 DT	358	rv->node_count = ids_count;
	359	rv->new = new;
	360	rv->new_count = new_count;
e7fd4179 DT	361	rv->seq = ++ls->ls_recover_seq;
	362	rv_old = ls->ls_recover_args;
	363	ls->ls_recover_args = rv;
	364	spin_unlock(&ls->ls_recover_lock);
	365
	366	if (rv_old) {
d44e0fc7 DT	367	log_error(ls, "unused recovery %llx %d",
d44e0fc7 DT	368	(unsigned long long)rv_old->seq, rv_old->node_count);
e7fd4179	369	kfree(rv_old->nodeids);
d44e0fc7	370	kfree(rv_old->new);
e7fd4179 DT	371	kfree(rv_old);
	372	}
	373
	374	dlm_recoverd_kick(ls);
	375	return 0;
	376
	377	fail:
	378	kfree(rv);
	379	kfree(ids);
d44e0fc7	380	kfree(new);
e7fd4179 DT	381	return error;
	382	}
	383