/*++ Copyright (c) 1996 Microsoft Corporation Module Name: send.c Abstract: Routines for sending global updates to the cluster Author: John Vert (jvert) 17-Apr-1996 Revision History: --*/ #include "gump.h" DWORD WINAPI GumSendUpdate( IN GUM_UPDATE_TYPE UpdateType, IN DWORD Context, IN DWORD BufferLength, IN PVOID Buffer ) /*++ Routine Description: Sends an update to all active nodes in the cluster. All registered update handlers for the specified UpdateType are called on each node. Any registered update handlers for the current node will be called on the same thread. This is useful for correct synchronization of the data structures to be updated. Arguments: UpdateType - Supplies the type of update. This determines which update handlers will be called and the sequence number to be used. Context - Supplies a DWORD of context to be passed to the GUM update handlers BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. Return Value: ERROR_SUCCESS if the request is successful. Win32 error code on failure. --*/ { return( GumSendUpdateReturnInfo( UpdateType, Context, NULL, BufferLength, Buffer ) ); } // GumSendUpdate DWORD GumpUpdateRemoteNode( IN PRPC_ASYNC_STATE AsyncState, IN DWORD RemoteNodeId, IN DWORD UpdateType, IN DWORD Context, IN DWORD Sequence, IN DWORD BufferLength, IN UCHAR Buffer[] ) /*++ Routine Description: Issues an update request to a remote node using async RPC. Arguments: AsyncState - A pointer to an RPC async state block. The u.hEvent member field must contain a valid event object handle. RemoteNodeId - Target of the update. Type - Supplies the type of update. This determines which update handlers will be called and the sequence number to be used. Context - Supplies a DWORD of context to be passed to the GUM update handlers ReturnStatusArray - Pointer to an array of structures to be filled in with the return value from the update handler on each node. The array is indexed by node ID. BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. Return Value: ERROR_SUCCESS if the request is successful. Win32 error code on failure. --*/ { DWORD Status; HANDLE hEventHandle; BOOL result; PNM_NODE Node; HANDLE handleArr[2]; CL_ASSERT(AsyncState->u.hEvent != NULL); // // Initialize the async RPC tracking information // hEventHandle = AsyncState->u.hEvent; AsyncState->u.hEvent = NULL; Status = RpcAsyncInitializeHandle(AsyncState, sizeof(RPC_ASYNC_STATE)); AsyncState->u.hEvent = hEventHandle; if (Status != RPC_S_OK) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] UpdateRemoteNode: Failed to initialize async RPC status " "block, status %1!u!\n", Status ); return (Status); } AsyncState->UserInfo = NULL; AsyncState->NotificationType = RpcNotificationTypeEvent; result = ResetEvent(AsyncState->u.hEvent); CL_ASSERT(result != 0); // Now hook onto NM node state down event mechanism to detect node downs, // instead of NmStartRpc()/NmEndRpc(). Node = NmReferenceNodeById(RemoteNodeId); CL_ASSERT(Node != NULL); handleArr[0] = AsyncState->u.hEvent; handleArr[1] = NmGetNodeStateDownEvent(Node); try { Status = GumUpdateNode( AsyncState, GumpRpcBindings[RemoteNodeId], UpdateType, Context, Sequence, BufferLength, Buffer ); if (Status == RPC_S_OK) { DWORD RpcStatus; DWORD WaitStatus; // // The call is pending. Wait for completion. // WaitStatus = WaitForMultipleObjects( 2, handleArr, FALSE, INFINITE ); if (WaitStatus != WAIT_OBJECT_0) { // // Something went wrong. // Either this is a rpc failure or, the target node went down. In either case // the error path is the same, complete the call and evict the target node // (eviction is done by the caller of this function). // ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: WaitforMultipleObjects returned %1!u!\n", WaitStatus ); if (WaitStatus == WAIT_FAILED) { Status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: WaitforMultipleObjects returned WAIT_FAILED, status %1!u!\n", Status); //SS: unexpected error - kill yourself CsInconsistencyHalt(Status); } else if (WaitStatus != (WAIT_OBJECT_0 + 1)) { Status = GetLastError(); //wait objects abandoned - can that happen with events? ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: WaitforMultipleObjects failed, status %1!u!\n", Status); //SS: unexpected error - kill yourself CsInconsistencyHalt(Status); } // SS: we only come here if the remote node is signalled to be down // make sure that a non-zero status is returned to the caller // so that the gum eviction occurs as desirable // // Cancel the call, just to be safe. // RpcStatus = RpcAsyncCancelCall( AsyncState, TRUE // Abortive cancel ); if (RpcStatus != RPC_S_OK) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: RpcAsyncCancelCall()= %1!u!\n", RpcStatus ); Status = RpcStatus; } else { CL_ASSERT(RpcStatus == RPC_S_OK); // // Wait for the call to complete. // WaitStatus = WaitForSingleObject( AsyncState->u.hEvent, INFINITE ); if (WaitStatus != WAIT_OBJECT_0) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: WaitForSingleObject() returns= %1!u!\n", WaitStatus); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: Mapping Status to WAIT_FAILED\n"); //SS: if this call doesnt complete, there is something //strange with RPC - should we kill ourselves or kill the other //node //SS: for now we asssume that the problem is not local Status = WAIT_FAILED; } } } // // The call should now be complete. Get the // completion status. Any RPC error will be // returned in 'RpcStatus'. If there was no // RPC error, then any application error will // be returned in 'Status'. // RpcStatus = RpcAsyncCompleteCall( AsyncState, &Status ); if (RpcStatus != RPC_S_OK) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: Failed to get " "completion status for async RPC call," "status %1!u!\n", RpcStatus ); Status = RpcStatus; } } else { // An error was returned synchronously. ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumUpdateRemoteNode: GumUpdateNode() failed synchronously, status %1!u!\n", Status ); } OmDereferenceObject(Node); } except (I_RpcExceptionFilter(RpcExceptionCode())) { OmDereferenceObject(Node); Status = GetExceptionCode(); } return(Status); } // GumpUpdateRemoteNode DWORD WINAPI GumSendUpdateReturnInfo( IN GUM_UPDATE_TYPE UpdateType, IN DWORD Context, OUT PGUM_NODE_UPDATE_HANDLER_STATUS ReturnStatusArray, IN DWORD BufferLength, IN PVOID Buffer ) /*++ Routine Description: Sends an update to all active nodes in the cluster. All registered update handlers for the specified UpdateType are called on each node. Any registered update handlers for the current node will be called on the same thread. This is useful for correct synchronization of the data structures to be updated. Arguments: UpdateType - Supplies the type of update. This determines which update handlers will be called and the sequence number to be used. Context - Supplies a DWORD of context to be passed to the GUM update handlers ReturnStatusArray - Pointer to an array of structures to be filled in with the return value from the update handler on each node. The array is indexed by node ID. The array must be at least (NmMaxNodeId + 1) entries in length. BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. Return Value: ERROR_SUCCESS if the request is successful. Win32 error code on failure. --*/ { DWORD Sequence; DWORD Status=RPC_S_OK; DWORD i; PGUM_INFO GumInfo; DWORD MyNodeId; DWORD LockerNode; RPC_ASYNC_STATE AsyncState; DWORD GenerationNum; //the generation number wrt to the locker at which the lock is obtained BOOL AssumeLockerWhistler = TRUE; CL_ASSERT(UpdateType < GumUpdateMaximum); // // Prepare for async RPC. We do this here to avoid hitting a failure // after the update is already in progress. // ZeroMemory((PVOID) &AsyncState, sizeof(RPC_ASYNC_STATE)); AsyncState.u.hEvent = CreateEvent( NULL, // no attributes TRUE, // manual reset FALSE, // initial state unsignalled NULL // no object name ); if (AsyncState.u.hEvent == NULL) { Status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumSendUpdate: Failed to allocate event object for async " "RPC call, status %1!u!\n", Status ); return (Status); } // // Initialize the return status array // if (ReturnStatusArray != NULL) { for (i=ClusterMinNodeId; i<=NmMaxNodeId; i++) { ReturnStatusArray[i].UpdateAttempted = FALSE; ReturnStatusArray[i].ReturnStatus = ERROR_NODE_NOT_AVAILABLE; } } GumInfo = &GumTable[UpdateType]; MyNodeId = NmGetNodeId(NmLocalNode); // Grab an RPC handle GumpStartRpc(MyNodeId); retryLock: LockerNode = GumpLockerNode; // // Send locking update to the locker node. // if (LockerNode == MyNodeId) { // // This node is the locker. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n", UpdateType, Context); Status = GumpDoLockingUpdate(UpdateType, MyNodeId, &Sequence, &GenerationNum); if (Status == ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: Locker dispatching seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); Status = GumpDispatchUpdate(UpdateType, Context, TRUE, TRUE, BufferLength, Buffer); if (ReturnStatusArray != NULL) { ReturnStatusArray[MyNodeId].UpdateAttempted = TRUE; ReturnStatusArray[MyNodeId].ReturnStatus = Status; } if (Status != ERROR_SUCCESS) { // // Note we have to use Sequence-1 for the unlock because GumpDispatchUpdate // failed and did not increment the sequence number. // GumpDoUnlockingUpdate(UpdateType, Sequence-1, MyNodeId, GenerationNum); } } } else { // CL_ASSERT(GumpRpcBindings[i] != NULL); ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: queuing update\ttype %1!u! context %2!u!\n", UpdateType, Context); AssumeLockerWhistler = TRUE; RetryLockForRollingUpgrade: try { NmStartRpc(LockerNode); if (AssumeLockerWhistler) { Status = GumQueueLockingUpdate2(GumpRpcBindings[LockerNode], MyNodeId, UpdateType, Context, &Sequence, BufferLength, Buffer, &GenerationNum); } else { //call the win2K version Status = GumQueueLockingUpdate(GumpRpcBindings[LockerNode], MyNodeId, UpdateType, Context, &Sequence, BufferLength, Buffer); } NmEndRpc(LockerNode); } except (I_RpcExceptionFilter(RpcExceptionCode())) { // // An exception from RPC indicates that the other node is either dead // or insane. Kill it and retry with a new locker. // NmEndRpc(LockerNode); Status = GetExceptionCode(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumSendUpdate: GumQueueLocking update to node %1!d! failed with %2!d!\n", LockerNode, Status); if (Status == RPC_S_PROCNUM_OUT_OF_RANGE) { //the locker node is win2K, try the old interface AssumeLockerWhistler = FALSE; goto RetryLockForRollingUpgrade; } else { GumpCommFailure(GumInfo, LockerNode, GetExceptionCode(), TRUE); // // The GUM update handler must have been called to select a new locker // node. // CL_ASSERT(LockerNode != GumpLockerNode); // // Retry the locking update with the new locker node. // goto retryLock; } } if (ReturnStatusArray != NULL) { ReturnStatusArray[LockerNode].UpdateAttempted = TRUE; ReturnStatusArray[LockerNode].ReturnStatus = Status; } if (Status == ERROR_SUCCESS) { CL_ASSERT(Sequence == GumpSequence); } if (Status != RPC_S_OK) { NmDumpRpcExtErrorInfo(Status); } //because there is no synchronization between join and regroups/gumprocessing //the old locker node may die and may come up again and not be the locker //anymore. We have to take care of this case. if (Status == ERROR_CLUSTER_GUM_NOT_LOCKER) { goto retryLock; } } if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[GUM] Queued lock attempt for send type %1!d! failed %2!d!\n", UpdateType, Status); // signal end of RPC handle GumpEndRpc(MyNodeId); if (AsyncState.u.hEvent != NULL) { CloseHandle(AsyncState.u.hEvent); } return(Status); } // // Grap the sendupdate lock to serialize with any replays // EnterCriticalSection(&GumpSendUpdateLock); if (LockerNode != GumpLockerNode) { // // Locker node changed, we need to restart again. // LeaveCriticalSection(&GumpSendUpdateLock); goto retryLock; } // // The update is now committed on the locker node. All remaining nodes // must be updated successfully, or they will be killed. // for (i=LockerNode+1; i != LockerNode; i++) { if (i == (NmMaxNodeId + 1)) { i=ClusterMinNodeId; if (i==LockerNode) { break; } } if (GumInfo->ActiveNode[i]) { // // Dispatch the update to the specified node. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n", Sequence, UpdateType, Context, i); if (i == MyNodeId) { Status = GumpDispatchUpdate(UpdateType, Context, FALSE, TRUE, BufferLength, Buffer); if (ReturnStatusArray != NULL) { ReturnStatusArray[i].UpdateAttempted = TRUE; ReturnStatusArray[i].ReturnStatus = Status; } if (Status != ERROR_SUCCESS){ ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumSendUpdate: Update on non-locker node(self) failed with %1!d! when it must succeed\n", Status); //Commit Suicide CsInconsistencyHalt(Status); } } else { DWORD dwStatus; ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); dwStatus = GumpUpdateRemoteNode( &AsyncState, i, UpdateType, Context, Sequence, BufferLength, Buffer ); if (ReturnStatusArray != NULL) { ReturnStatusArray[i].UpdateAttempted = TRUE; ReturnStatusArray[i].ReturnStatus = dwStatus; } // // If the update on the other node failed, then the // other node must now be out of the cluster since the // update has already completed on the locker node. // if (dwStatus != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumSendUpdate: Update on node %1!d! failed with %2!d! when it must succeed\n", i, dwStatus); NmDumpRpcExtErrorInfo(dwStatus); GumpCommFailure(GumInfo, i, dwStatus, TRUE); } } } } // // Our update is over // LeaveCriticalSection(&GumpSendUpdateLock); // // All nodes have been updated. Send unlocking update. // if (LockerNode == MyNodeId) { GumpDoUnlockingUpdate(UpdateType, Sequence, MyNodeId, GenerationNum); } else { //SS: We will assume that AssumeLockerWhistler is set appropriately when the lock was acquired try { NmStartRpc(LockerNode); if (AssumeLockerWhistler) { //SS: the sequence number will protect if the locker has gone down //and come back up since we got the lock and tried to release it Status = GumUnlockUpdate2( GumpRpcBindings[LockerNode], UpdateType, Sequence, MyNodeId, GenerationNum ); } else { Status = GumUnlockUpdate( GumpRpcBindings[LockerNode], UpdateType, Sequence); } NmEndRpc(LockerNode); } except (I_RpcExceptionFilter(RpcExceptionCode())) { // // The locker node has crashed. Notify the NM, it will call our // notification routine to select a new locker node. Then retry // the unlock on the new locker node. // SS: changed to not retry unlocks..the new locker node will // unlock after propagating this change in any case. // NmEndRpc(LockerNode); Status = GetExceptionCode(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumSendUpdate: Unlocking update to node %1!d! failed with %2!d!\n", LockerNode, Status); GumpCommFailure(GumInfo, LockerNode, Status, TRUE); CL_ASSERT(LockerNode != GumpLockerNode); } if(Status != RPC_S_OK) { NmDumpRpcExtErrorInfo(Status); } } ClRtlLogPrint(LOG_NOISE, "[GUM] GumSendUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); // signal end of RPC handle GumpEndRpc(MyNodeId); if (AsyncState.u.hEvent != NULL) { CloseHandle(AsyncState.u.hEvent); } return(ERROR_SUCCESS); } // GumSendUpdateReturnInfo #ifdef GUM_POST_SUPPORT John Vert (jvert) 11/18/1996 POST is disabled for now since nobody uses it. N.B. The below code does not handle locker node failures DWORD WINAPI GumPostUpdate( IN GUM_UPDATE_TYPE UpdateType, IN DWORD Context, IN DWORD BufferLength, IN PVOID Buffer // THIS WILL BE FREED ) /*++ Routine Description: Posts an update to all active nodes in the cluster. All registered update handlers for the specified UpdateType are called on each node. The update will not be reported on the current node. The update will not necessarily have completed when this function returns, but will complete eventually if the current node does not fail. Arguments: UpdateType - Supplies the type of update. This determines which update handlers will be called and the sequence number to be used. Context - Supplies a DWORD of context to be passed to the GUM update handlers BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. THIS BUFFER WILL BE FREED ONCE THE POST HAS COMPLETED. Return Value: ERROR_SUCCESS if the request is successful. Win32 error code on failure. --*/ { DWORD Sequence; DWORD Status; DWORD i; BOOL IsLocker = TRUE; PGUM_INFO GumInfo; DWORD MyNodeId; DWORD LockerNode=(DWORD)-1; CL_ASSERT(UpdateType < GumUpdateMaximum); GumInfo = &GumTable[UpdateType]; MyNodeId = NmGetNodeId(NmLocalNode); // // Find the lowest active node in the cluster. This is the // locker. for (i=ClusterMinNodeId; i <= NmMaxNodeId; i++) { if (GumInfo->ActiveNode[i]) { LockerNode = i; break; } } CL_ASSERT(i <= NmMaxNodeId); // // Post a locking update to the locker node. If this succeeds // immediately, we can go do the work directly. If it pends, // the locker node will call us back when it is our turn to // make the updates. // if (i == MyNodeId) { // // This node is the locker. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumPostUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n", UpdateType, Context); Status = GumpDoLockingPost(UpdateType, MyNodeId, &Sequence, Context, BufferLength, (DWORD)Buffer, Buffer); if (Status == ERROR_SUCCESS) { // // Update our sequence number so we stay in sync, even though // we aren't dispatching the update. // GumpSequence += 1; } } else { CL_ASSERT(GumpRpcBindings[i] != NULL); ClRtlLogPrint(LOG_NOISE, "[GUM] GumPostUpdate: queuing update\ttype %1!u! context %2!u!\n", UpdateType, Context); Status = GumQueueLockingPost(GumpRpcBindings[i], MyNodeId, UpdateType, Context, &Sequence, BufferLength, Buffer, (DWORD)Buffer); if (Status == ERROR_SUCCESS) { CL_ASSERT(Sequence == GumpSequence); } } if (Status == ERROR_SUCCESS) { // // The lock was immediately acquired, go ahead and post directly // here. // GumpDeliverPosts(LockerNode+1, UpdateType, Sequence, Context, BufferLength, Buffer); // // All nodes have been updated. Send unlocking update. // if (LockerNode == MyNodeId) { GumpDoUnlockingUpdate(UpdateType, Sequence); } else { GumUnlockUpdate( GumpRpcBindings[LockerNode], UpdateType, Sequence ); } ClRtlLogPrint(LOG_NOISE, "[GUM] GumPostUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); return(ERROR_SUCCESS); } else { // // The lock is currently held. We will get called back when it is released // ClRtlLogPrint(LOG_NOISE, "[GUM] GumPostUpdate: pending update type %1!u! context %2!u!\n", UpdateType, Context); return(ERROR_IO_PENDING); } } VOID GumpDeliverPosts( IN DWORD FirstNodeId, IN GUM_UPDATE_TYPE UpdateType, IN DWORD Sequence, IN DWORD Context, IN DWORD BufferLength, IN PVOID Buffer // THIS WILL BE FREED ) /*++ Routine Description: Actually delivers the update post to the specified nodes. The GUM lock is assumed to be held. Arguments: FirstNodeId - Supplies the node ID where the posts should start. This is generally the LockerNode+1. UpdateType - Supplies the type of update. This determines which update handlers will be called and the sequence number to be used. Context - Supplies a DWORD of context to be passed to the GUM update handlers BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. THIS BUFFER WILL BE FREED ONCE THE POST HAS COMPLETED. Return Value: None. --*/ { DWORD i; PGUM_INFO GumInfo; DWORD MyNodeId; GumInfo = &GumTable[UpdateType]; MyNodeId = NmGetNodeId(NmLocalNode); for (i=FirstNodeId; i<=NmMaxNodeId; i++) { if (GumInfo->ActiveNode[i]) { // // Dispatch the update to the specified node. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumpDeliverPosts: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n", Sequence, UpdateType, Context, i); if (i == MyNodeId) { // // Update our sequence number so we stay in sync, even though // we aren't dispatching the update. // GumpSequence += 1; } else { CL_ASSERT(GumpRpcBindings[i] != NULL); ClRtlLogPrint(LOG_NOISE, "[GUM] GumpDeliverPosts: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); GumUpdateNode(GumpRpcBindings[i], UpdateType, Context, Sequence, BufferLength, Buffer); } } } LocalFree(Buffer); } #endif DWORD WINAPI GumAttemptUpdate( IN DWORD Sequence, IN GUM_UPDATE_TYPE UpdateType, IN DWORD Context, IN DWORD BufferLength, IN PVOID Buffer ) /*++ Routine Description: Conditionally sends an update to all active nodes in the cluster. If the clusterwise sequence number matches the supplied sequence number, all registered update handlers for the specified UpdateType are called on each node. Any registered update handlers for the current node will be called on the same thread. This is useful for correct synchronization of the data structures to be updated. The normal usage of this routine is as follows: · obtain current sequence number from GumGetCurrentSequence · make modification to cluster state · conditionally update cluster state with GumAttemptUpdate · If update fails, undo modification, release any locks, try again later Arguments: Sequence - Supplies the sequence number obtained from GumGetCurrentSequence. UpdateType - Supplies the type of update. This determines which update handlers will be called Context - Supplies a DWORD of context to be passed to the GUM update handlers BufferLength - Supplies the length of the update buffer to be passed to the update handlers Buffer - Supplies a pointer to the update buffer to be passed to the update handlers. Return Value: ERROR_SUCCESS if the request is successful. Win32 error code on failure. --*/ { DWORD Status=RPC_S_OK; DWORD i; PGUM_INFO GumInfo; DWORD MyNodeId; DWORD LockerNode=(DWORD)-1; RPC_ASYNC_STATE AsyncState; DWORD dwGenerationNum; //the generation id of the node at which the lock is acquired CL_ASSERT(UpdateType < GumUpdateMaximum); ZeroMemory((PVOID) &AsyncState, sizeof(RPC_ASYNC_STATE)); AsyncState.u.hEvent = CreateEvent( NULL, // no attributes TRUE, // manual reset FALSE, // initial state unsignalled NULL // no object name ); if (AsyncState.u.hEvent == NULL) { Status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumAttemptUpdate: Failed to allocate event object for " "async RPC call, status %1!u!\n", Status ); return (Status); } GumInfo = &GumTable[UpdateType]; MyNodeId = NmGetNodeId(NmLocalNode); retryLock: LockerNode = GumpLockerNode; // // Send locking update to the locker node. // if (LockerNode == MyNodeId) { // // This node is the locker. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n", UpdateType, Context); if (GumpTryLockingUpdate(UpdateType, MyNodeId, Sequence, &dwGenerationNum)) { ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: Locker dispatching seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); Status = GumpDispatchUpdate(UpdateType, Context, TRUE, TRUE, BufferLength, Buffer); if (Status != ERROR_SUCCESS) { // // Note we have to use Sequence-1 for the unlock because GumpDispatchUpdate // failed and did not increment the sequence number. // GumpDoUnlockingUpdate(UpdateType, Sequence-1, MyNodeId, dwGenerationNum); } } else { Status = ERROR_CLUSTER_DATABASE_SEQMISMATCH; } } else { // //send the locking update to the locker node ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: queuing update\ttype %1!u! context %2!u!\n", UpdateType, Context); try { NmStartRpc(LockerNode); Status = GumAttemptLockingUpdate(GumpRpcBindings[LockerNode], MyNodeId, UpdateType, Context, Sequence, BufferLength, Buffer); NmEndRpc(LockerNode); } except (I_RpcExceptionFilter(RpcExceptionCode())) { // // An exception from RPC indicates that the other node is either dead // or insane. Kill it and retry with a new locker. // NmEndRpc(LockerNode); GumpCommFailure(GumInfo, LockerNode, GetExceptionCode(), TRUE); // // The GUM update handler must have been called to select a new locker // node. // CL_ASSERT(LockerNode != GumpLockerNode); // // Retry the locking update with the new locker node. // goto retryLock; } if (Status == ERROR_SUCCESS) { CL_ASSERT(Sequence == GumpSequence); } if(Status != RPC_S_OK) { NmDumpRpcExtErrorInfo(Status); } } if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[GUM] GumAttemptUpdate: Queued lock attempt for send type %1!d! failed %2!d!\n", UpdateType, Status); return(Status); } // // Grap the sendupdate lock to serialize with any replays // EnterCriticalSection(&GumpSendUpdateLock); if (LockerNode != GumpLockerNode) { // // Locker node changed, we need to restart again. // LeaveCriticalSection(&GumpSendUpdateLock); goto retryLock; } // The update is now committed on the locker node. All remaining nodes // must be updated successfully, or they will be killed. // for (i=LockerNode+1; i != LockerNode; i++) { if (i == (NmMaxNodeId + 1)) { i=ClusterMinNodeId; if (i==LockerNode) { break; } } if (GumInfo->ActiveNode[i]) { // // Dispatch the update to the specified node. // ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n", Sequence, UpdateType, Context, i); if (i == MyNodeId) { Status = GumpDispatchUpdate(UpdateType, Context, FALSE, TRUE, BufferLength, Buffer); if (Status != ERROR_SUCCESS){ ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumAttemptUpdate: Update on non-locker node(self) failed with %1!d! when it must succeed\n", Status); //Commit Suicide CsInconsistencyHalt(Status); } } else { DWORD dwStatus; ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); dwStatus = GumpUpdateRemoteNode( &AsyncState, i, UpdateType, Context, Sequence, BufferLength, Buffer ); // // If the update on the other node failed, then the // other node must now be out of the cluster since the // update has already completed on the locker node. // if (dwStatus != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumAttemptUpdate: Update on node %1!d! failed with %2!d! when it must succeed\n", i, dwStatus); NmDumpRpcExtErrorInfo(dwStatus); GumpCommFailure(GumInfo, i, dwStatus, TRUE); } } } } // // Our update is over // LeaveCriticalSection(&GumpSendUpdateLock); // // All nodes have been updated. Send unlocking update. // if (LockerNode == MyNodeId) { GumpDoUnlockingUpdate(UpdateType, Sequence, MyNodeId, dwGenerationNum); } else { try { NmStartRpc(LockerNode); Status = GumUnlockUpdate( GumpRpcBindings[LockerNode], UpdateType, Sequence ); NmEndRpc(LockerNode); } except (I_RpcExceptionFilter(RpcExceptionCode())) { // // The locker node has crashed. Notify the NM, it will call our // notification routine to select a new locker node. The new // locker node will release the gum lock after propagating // the current update. // NmEndRpc(LockerNode); Status = GetExceptionCode(); ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumAttemptUpdate: Unlocking update to node %1!d! failed with %2!d!\n", LockerNode, Status); GumpCommFailure(GumInfo, LockerNode, Status, TRUE); CL_ASSERT(LockerNode != GumpLockerNode); } if(Status != RPC_S_OK) { NmDumpRpcExtErrorInfo(Status); } } ClRtlLogPrint(LOG_NOISE, "[GUM] GumAttemptUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n", Sequence, UpdateType, Context); if (AsyncState.u.hEvent != NULL) { CloseHandle(AsyncState.u.hEvent); } return(ERROR_SUCCESS); } DWORD WINAPI GumGetCurrentSequence( IN GUM_UPDATE_TYPE UpdateType ) /*++ Routine Description: Obtains the current clusterwise global update sequence number Arguments: UpdateType - Supplies the type of update. Each update type may have an independent sequence number. Return Value: Current global update sequence number for the specified update type. --*/ { CL_ASSERT(UpdateType < GumUpdateMaximum); return(GumpSequence); } VOID GumSetCurrentSequence( IN GUM_UPDATE_TYPE UpdateType, DWORD Sequence ) /*++ Routine Description: Sets the current sequence for the specified global update. Arguments: UpdateType - Supplies the update type whose sequence is to be updated. Sequence - Supplies the new sequence number. Return Value: None. --*/ { CL_ASSERT(UpdateType < GumUpdateMaximum); GumpSequence = Sequence; } VOID GumCommFailure( IN GUM_UPDATE_TYPE UpdateType, IN DWORD NodeId, IN DWORD ErrorCode, IN BOOL Wait ) /*++ Routine Description: Informs the NM that a fatal communication error has occurred trying to talk to another node. Arguments: GumInfo - Supplies the update type where the communication failure occurred. NodeId - Supplies the node id of the other node. ErrorCode - Supplies the error that was returned from RPC Wait - if TRUE, this function blocks until the GUM event handler has processed the NodeDown notification for the specified node. if FALSE, this function returns immediately after notifying NM Return Value: None. --*/ { PGUM_INFO GumInfo = &GumTable[UpdateType]; ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumCommFailure %1!d! communicating with node %2!d!\n", ErrorCode, NodeId); GumpCommFailure(GumInfo, NodeId, ErrorCode, Wait); } VOID GumpCommFailure( IN PGUM_INFO GumInfo, IN DWORD NodeId, IN DWORD ErrorCode, IN BOOL Wait ) /*++ Routine Description: Informs the NM that a fatal communication error has occurred trying to talk to another node. Arguments: GumInfo - Supplies the update type where the communication failure occurred. NodeId - Supplies the node id of the other node. ErrorCode - Supplies the error that was returned from RPC Wait - if TRUE, this function blocks until the GUM event handler has processed the NodeDown notification for the specified node. if FALSE, this function returns immediately after notifying NM Return Value: None. --*/ { DWORD dwCur; ClRtlLogPrint(LOG_CRITICAL, "[GUM] GumpCommFailure %1!d! communicating with node %2!d!\n", ErrorCode, NodeId); // This is the general GUM RPC failure path, let's dump the extended error info. // NOTE: The dumping routine is benign, so calling this from a non RPC failure path would just return. NmDumpRpcExtErrorInfo(ErrorCode); // This is a hack to check if we are shutting down. See bug 88411 if (ErrorCode == ERROR_SHUTDOWN_IN_PROGRESS) { // if we are shutting down, just kill self // set to our node id NodeId = NmGetNodeId(NmLocalNode); } // // Get current generation number // if (Wait) { dwCur = GumpGetNodeGenNum(GumInfo, NodeId); } NmAdviseNodeFailure(NodeId, ErrorCode); if (Wait) { // // Wait for this node to be declared down and // GumpEventHandler to mark it as inactive. // GumpWaitNodeDown(NodeId, dwCur); } }