windows-server-2003/base/cluster/service/gs/recovery.c

/*++

Copyright (c) 2000  Microsoft Corporation

Module Name:

    recovery.c

Abstract:

    Handles node down events

Author:

    Ahmed Mohamed (ahmedm) 12, 01, 2000

Revision History:

--*/

#include "gs.h"
#include "gsp.h"
#include <stdio.h>

extern gs_nid_t	GsLocalNodeId;
extern int	GsMaxNodeId;
extern int	GsMinNodeId;
extern gs_group_t	GsGroupTable[];

// Node down event
void
GspRsFree(gs_recovery_state_t *rs)
{
    // free recovery state
    gs_rblk_t	*p;

    while (p = rs->rs_list) {
	rs->rs_list = p->next;
	free((char *)p);
    }

    GsEventFree(rs->rs_event);
    free((char *)rs);
}

void
GspPhase1NodeDown(ULONG	set)
{
    gs_group_t	*gd;
    int i, j;

    for (i = 0; i < GsGroupTableSize; i++) {
	gd = &GsGroupTable[i];
	if (gd->g_state == GS_GROUP_STATE_FREE) {
	    continue;
	}
	GsLockEnter(gd->g_lock);
	if (gd->g_mset & set) {
	    gd->g_mset &= ~set;
	    gd->g_curview++;
	    gd->g_state |= GS_GROUP_FLAGS_RECOVERY;
	    gd->g_sz = 0;
	    for (j = gd->g_mset; j > 0; j = j >> 1) {
		if (j & 0x1)
		    gd->g_sz++;
	    }
	    if (set & (1 << gd->g_mid)) {

		for (j = GsMinNodeId; j != GsMaxNodeId; j++) {
		    if (gd->g_mset & (1 << j)) {
			break;
		    }
		}
		// elect a new master
		gd->g_mid = (gs_memberid_t) j;
		gd->g_state |= GS_GROUP_FLAGS_NEWMASTER;
	    }

	    recovery_log(("Phase1 mask %x gid %d mid %d mset %x sz %d\n",
		      set, gd->g_id, gd->g_mid, gd->g_mset,
		      gd->g_sz));

	    if (gd->g_rs != NULL) {
		set |= gd->g_rs->rs_dset;
		GsEventSignal(gd->g_rs->rs_event);
		GspRsFree(gd->g_rs);
	    }
	    gd->g_rs = (gs_recovery_state_t *) malloc(sizeof(*gd->g_rs));
	    assert(gd->g_rs != NULL);
	    GsManualEventInit(gd->g_rs->rs_event);
	    gd->g_rs->rs_sz = 0;
	    gd->g_rs->rs_list = NULL;
	    gd->g_rs->rs_epoch = gd->g_curview;
	    gd->g_rs->rs_dset = set;
	    gd->g_rs->rs_mset = gd->g_mset;
	    if (gd->g_mid != gd->g_nid) {
		// we are not master, reset our mset to self and master only
		gd->g_rs->rs_mset = (1 << gd->g_nid) | (1 << gd->g_mid);
	    }
	} else if (gd->g_mset == 0 && (set & (1 << gd->g_mid))) {
	    // no one is participating in this group and the sole owner dead
	    // remove the group and free it
	    GsCloseGroup(gd);
	}

	GsLockExit(gd->g_lock);
    }

}

void
GspRsAddSequence(gs_recovery_state_t *rs, gs_sequence_t mseq, int delta)
{
    gs_rblk_t *p, **q;

    for (p = rs->rs_list; p != NULL; p = p->next) {
	if (p->mseq == mseq) {
	    p->have += delta;
	    recovery_log(("Found seq %d cnt %d\n", mseq, p->have));
	    return;
	}
    }
    // if we get here that means the sequence is missing
    p = (gs_rblk_t *) malloc(sizeof(*p));
    if (p == NULL) {
	err_log(("GspRsAddSeq: unable to allocate memory!\n"));
	exit(1);
    }

    p->mseq = mseq;
    p->have = delta;

    recovery_log(("Add seq %d cnt %d\n", mseq, p->have));

    rs->rs_sz++;
    q = &rs->rs_list;
    while (*q != NULL) {
	if ((*q)->mseq > mseq) {
	    p->next = *q;
	    *q = p;
	    return;
	}
	q = &(*q)->next;
    }

    p->next = *q;
    *q = p;
}

void
GspPhase2NodeDown(ULONG set)
{

    gs_group_t	*gd;
    int i, j;

    for (i = 0; i < GsGroupTableSize; i++) {
	gd = &GsGroupTable[i];
	if (!(gd->g_state & GS_GROUP_FLAGS_RECOVERY)) {
	    continue;
	}
	GsLockEnter(gd->g_lock);
	if (gd->g_state & GS_GROUP_FLAGS_RECOVERY) {
	    gs_msg_t *p;
	    extern void GspDumpQueue(gs_group_t*);

	    recovery_log(("Phase2 queue\n"));
	    GspDumpQueue(gd);
	    recovery_log(("Expect gid %d <%d, %d>\n", 
			  gd->g_id, gd->g_recv.r_mseq, gd->g_recv.r_bnum));

	    // walk recv queue and replay messages from dead members
	    for (p = gd->g_recv.r_head; p != NULL; p = p->m_next) {
		if (set & (1 << p->m_hdr.h_sid)) { 
		    // tag message as if we got a reply
		    p->m_hdr.h_flags |= GS_FLAGS_REPLY;
		    if (p->m_hdr.h_type != GS_MSG_TYPE_UCAST){
			p->m_hdr.h_flags |= GS_FLAGS_REPLAY;
			msg_mcast(gd->g_mset, &p->m_hdr, 
				  p->m_buf, p->m_hdr.h_len);
		    }
		    // check of unclosed continued sends
		    if (p->m_hdr.h_flags & GS_FLAGS_CONTINUED) {
			gs_msg_t *q;
			
			q = p->m_next;
			if (q == NULL || 
			    q->m_hdr.h_mseq != p->m_hdr.h_mseq ||
			    q->m_hdr.h_bnum != p->m_hdr.h_bnum+1) {

			    q = msg_alloc(NULL, 0);
			    if (q == NULL) {
				err_log(("Unable to allocate memory!\n"));
				halt(1);
			    }
			    memcpy(&q->m_hdr, &p->m_hdr, sizeof(p->m_hdr));
			    q->m_hdr.h_type = GS_MSG_TYPE_ABORT;
			    q->m_hdr.h_len = 0;
			    q->m_hdr.h_bnum++;
			    q->m_hdr.h_flags = GS_FLAGS_LAST;

			    // insert abort msg
			    q->m_next = p->m_next;
			    p->m_next = q;
			}
		    }
		}
	    }
			    
	    // walk recv queue and build msg of sequences we have
	    for (p = gd->g_recv.r_head; p != NULL; p = p->m_next) {
		if (p->m_hdr.h_mseq != GS_MSG_TYPE_UCAST)
		    GspRsAddSequence(gd->g_rs, p->m_hdr.h_mseq, 1);
	    }

	    // send msg of sequences to master
	    if (gd->g_mid != gd->g_nid) {
		gs_rblk_t *p;
		gs_sequence_t *list;
		int k;
		gs_msg_hdr_t hdr;

		recovery_log(("Sending sequence state to master %d\n", gd->g_mid));

		list = (gs_sequence_t *) malloc(sizeof(*list) * gd->g_rs->rs_sz);
		if (list == NULL) {
		    err_log(("Unable to allocate memory during recovery\n"));
		    exit(1);
		}
		k = 0;
		for (p = gd->g_rs->rs_list; p != NULL; p = p->next) {
		    list[k] = p->mseq;
		    k++;
		}
		assert(k == gd->g_rs->rs_sz);
		k = k * sizeof(*list);

		hdr.h_len = (UINT16) k;
		hdr.h_type = GS_MSG_TYPE_RECOVERY;
		hdr.h_sid = (gs_memberid_t)gd->g_nid;
		hdr.h_mid = (gs_memberid_t) gd->g_mid;
		hdr.h_gid = gd->g_id;
		hdr.h_viewnum = gd->g_curview;
		hdr.h_mseq = gd->g_recv.r_mseq;
		hdr.h_lseq = gd->g_send.s_mseq;
		
		msg_send(gd->g_mid, &hdr, (const char *) list, k);
		free((char *)list);
	    } else {
		// add current sequence to dispatch
		GspRsAddSequence(gd->g_rs, gd->g_recv.r_mseq, 0);
	    }

	    // handle send path
	    for (j = 0; j < gd->g_send.s_wsz; j++) {
		gs_context_t *ctx = &gd->g_send.s_ctxpool[j];
		
		if (ctx->ctx_id != GS_CONTEXT_INVALID_ID && ctx->ctx_msg != NULL){
		    recovery_log(("phase2 gid %d ctx %d mask %x\n",
			      gd->g_id, ctx->ctx_id, ctx->ctx_mask));
		    if (set & ctx->ctx_mask) {
			int k, n;

			recovery_log(("phase2 complete gid %d ctx %d\n",
				  gd->g_id, ctx->ctx_id));
			for (n = 0, k = set; k != 0; k = k >> 1, n++) {
			    if (k & 0x1) {
				GspProcessReply(gd, ctx, n, NULL, 0, 
						STATUS_HOST_UNREACHABLE);
			    }
			}
		    }
		}
	    }
	    
	    // clear this node bit
	    gd->g_rs->rs_mset &= ~(1 << gd->g_nid);
	    if (gd->g_rs->rs_mset == 0) {
		void GspComputeState(gs_group_t *gd);

		GspComputeState(gd);
	    }
		
	}
	GsLockExit(gd->g_lock);
    }
    
}

void GspSyncState(gs_group_t *gd, gs_msg_t *msg, gs_sequence_t *list, int sz);

void
GspComputeState(gs_group_t *gd)
{

    int k;
    gs_sequence_t *list;
    gs_rblk_t *p, *last = NULL;
    gs_msg_t *msg;


    recovery_log(("Compute missing sequences gid %d\n", gd->g_id));

		// compute missing sequences
    list = (gs_sequence_t *) malloc(sizeof(*list) * gd->g_rs->rs_sz);
    if (list == NULL) {
	err_log(("Unable to allocate memory during computestate\n"));
	exit(1);
    }
    k = 0;
    for (p = gd->g_rs->rs_list; p != NULL; p = p->next) {
	recovery_log(("rs list sequence %d\n", p->mseq));
	if (p->have == 0) {
	    recovery_log(("Skip sequence %d\n", p->mseq));
	    list[k] = p->mseq;
	    k++;
	}
	last = p;
    }
    // compute next starting mseq
    gd->g_global_seq = last != NULL ? last->mseq+1 : gd->g_recv.r_mseq;

    k = k * sizeof(*list);

    msg = msg_alloc((char *)list, k);
    assert(msg != NULL);

    msg->m_hdr.h_len = (UINT16) k;
    msg->m_hdr.h_type = GS_MSG_TYPE_SYNC;
    msg->m_hdr.h_flags = GS_FLAGS_LAST;
    msg->m_hdr.h_sid = (gs_memberid_t) gd->g_nid;
    msg->m_hdr.h_mid = (gs_memberid_t) gd->g_mid;
    msg->m_hdr.h_cid = (gs_cookie_t) -1;
    msg->m_hdr.h_gid = gd->g_id;
    msg->m_hdr.h_viewnum = gd->g_curview;
    msg->m_hdr.h_mseq = gd->g_global_seq++;
    msg->m_hdr.h_lseq = gd->g_send.s_lseq;
    msg->m_hdr.h_bnum = 0;
    *((ULONG *)msg->m_hdr.h_tag) = gd->g_rs->rs_dset;

		// send missing sequence list to other nodes
    msg_mcast(gd->g_mset, &msg->m_hdr, (const char *) list, k);

    recovery_log(("Next starting sequence is %d\n", gd->g_global_seq));

		// handle self
    GspSyncState(gd, msg, list, k / sizeof(*list));

    free((char *)list);

}

void
GspRecoveryMsgHandler(gs_msg_t *rmsg)

{
    gs_msg_hdr_t *hdr;
    gs_group_t *gd;
    
    hdr = &rmsg->m_hdr;

    gd = GspLookupGroup(hdr->h_gid);
    // accept messages only if in a valid view
    if (gd && rmsg->m_hdr.h_viewnum == gd->g_curview) {
	gs_sequence_t *list;
	int sz, k;

	list = (gs_sequence_t *) rmsg->m_buf;
	sz = rmsg->m_hdr.h_len / sizeof(*list);

	GsLockEnter(gd->g_lock);

	// make sure group is in recovery mode
	assert(gd->g_state & GS_GROUP_FLAGS_RECOVERY);
	assert(gd->g_mid == gd->g_nid);

	// add current sequence to dispatch
	GspRsAddSequence(gd->g_rs, hdr->h_mseq, 0);
	// insert sequences into have list
	for (k = 0; k < sz; k++) {
	    GspRsAddSequence(gd->g_rs, list[k], 1);
	}

	// clear this node bit
	gd->g_rs->rs_mset &= ~(1 << hdr->h_sid);
	if (gd->g_rs->rs_mset == 0) {
	    GspComputeState(gd);
	}	

	GsLockExit(gd->g_lock);
    }

    msg_free(rmsg);

}

void
GspSyncState(gs_group_t *gd, gs_msg_t *msg, gs_sequence_t *list, int sz)

{
    int  k;

    // make sure group is in recovery mode
    assert(gd->g_state & GS_GROUP_FLAGS_RECOVERY);
    assert(gd->g_mid != gd->g_nid);
    assert(gd->g_mid == hdr->h_sid);

    // mark missing sequences
    for (k = 0; k < sz; k++) {
	gs_msg_t *p;

	recovery_log(("Missing sequence %d\n", list[k]));

	p = msg_alloc(NULL, 0);
	if (p == NULL) {
	    err_log(("Unable to allocate memory during syncstate!\n"));
	    halt(1);
	}

	p->m_hdr.h_sid = gd->g_nid;
	p->m_hdr.h_gid = gd->g_id;
	p->m_hdr.h_cid = (gs_cookie_t) -1;
	p->m_hdr.h_type = GS_MSG_TYPE_SKIP;
	p->m_hdr.h_mseq = list[k];
	p->m_hdr.h_lseq = gd->g_send.s_lseq;
	p->m_hdr.h_bnum = 0;
	p->m_hdr.h_flags = GS_FLAGS_LAST;

	GspOrderInsert(gd, p, p, p->m_hdr.h_mseq, 0);
    }

    // set startview to curview
    gd->g_startview = gd->g_curview;
    // clear recovery state
    gd->g_state &= ~GS_GROUP_FLAGS_RECOVERY;
    // free recovery state
    GsEventSignal(gd->g_rs->rs_event);
    GspRsFree(gd->g_rs);
    gd->g_rs = NULL;

    // insert msg into dispatch queue at proper order  
    GspOrderInsert(gd, msg, msg, msg->m_hdr.h_mseq, 0);
    GspDispatch(gd);
#if 0
    // xxx: need to understand this again
    if (gd->g_recv.r_last != NULL) {
	GspCleanQueue(gd, last_mseq);
    }
#endif
    // restart any pending sends
    if (gd->g_send.s_waitqueue != NULL && (gd->g_state & GS_GROUP_FLAGS_NEWMASTER)) {
	recovery_log(("resend: gs %x s %x\n", gd, gd->g_send.s_waitqueue));
	GspAllocateSequence(gd);
    }
    gd->g_state &= ~GS_GROUP_FLAGS_NEWMASTER;
}


void
GspSyncMsgHandler(gs_msg_t *msg)
{
    gs_msg_hdr_t *hdr;
    gs_group_t *gd;
    
    hdr = &msg->m_hdr;

    gd = GspLookupGroup(hdr->h_gid);
    // accept messages only if in a valid view
    if (gd && msg->m_hdr.h_viewnum == gd->g_curview) {
	gs_sequence_t *list;
	int sz;

	list = (gs_sequence_t *) msg->m_buf;
	sz = msg->m_hdr.h_len / sizeof(*list);

	GsLockEnter(gd->g_lock);

	// clear this node bit
	gd->g_rs->rs_mset &= ~(1 << hdr->h_sid);
	assert(gd->g_rs->rs_mset == 0);

	GspSyncState(gd, msg, list, sz);

	GsLockExit(gd->g_lock);
    } else {
	msg_free(msg);
    }

}