You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1525 lines
47 KiB
1525 lines
47 KiB
/*++
|
|
|
|
Copyright (c) 1996 Microsoft Corporation
|
|
|
|
Module Name:
|
|
|
|
send.c
|
|
|
|
Abstract:
|
|
|
|
Routines for sending global updates to the cluster
|
|
|
|
Author:
|
|
|
|
John Vert (jvert) 17-Apr-1996
|
|
|
|
Revision History:
|
|
|
|
--*/
|
|
|
|
#include "gump.h"
|
|
|
|
|
|
DWORD
|
|
WINAPI
|
|
GumSendUpdate(
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD Context,
|
|
IN DWORD BufferLength,
|
|
IN PVOID Buffer
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Sends an update to all active nodes in the cluster. All
|
|
registered update handlers for the specified UpdateType
|
|
are called on each node. Any registered update handlers
|
|
for the current node will be called on the same thread.
|
|
This is useful for correct synchronization of the data
|
|
structures to be updated.
|
|
|
|
Arguments:
|
|
|
|
UpdateType - Supplies the type of update. This determines
|
|
which update handlers will be called and the sequence
|
|
number to be used.
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
BufferLength - Supplies the length of the update buffer to
|
|
be passed to the update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed
|
|
to the update handlers.
|
|
|
|
Return Value:
|
|
|
|
ERROR_SUCCESS if the request is successful.
|
|
|
|
Win32 error code on failure.
|
|
|
|
|
|
--*/
|
|
|
|
{
|
|
return(
|
|
GumSendUpdateReturnInfo(
|
|
UpdateType,
|
|
Context,
|
|
NULL,
|
|
BufferLength,
|
|
Buffer
|
|
)
|
|
);
|
|
|
|
} // GumSendUpdate
|
|
|
|
|
|
DWORD
|
|
GumpUpdateRemoteNode(
|
|
IN PRPC_ASYNC_STATE AsyncState,
|
|
IN DWORD RemoteNodeId,
|
|
IN DWORD UpdateType,
|
|
IN DWORD Context,
|
|
IN DWORD Sequence,
|
|
IN DWORD BufferLength,
|
|
IN UCHAR Buffer[]
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Issues an update request to a remote node using async RPC.
|
|
|
|
Arguments:
|
|
|
|
AsyncState - A pointer to an RPC async state block. The u.hEvent
|
|
member field must contain a valid event object handle.
|
|
|
|
RemoteNodeId - Target of the update.
|
|
|
|
Type - Supplies the type of update. This determines
|
|
which update handlers will be called and the sequence
|
|
number to be used.
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
ReturnStatusArray - Pointer to an array of structures to be filled in
|
|
with the return value from the update handler on each node. The
|
|
array is indexed by node ID.
|
|
|
|
BufferLength - Supplies the length of the update buffer to
|
|
be passed to the update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed
|
|
to the update handlers.
|
|
|
|
Return Value:
|
|
|
|
ERROR_SUCCESS if the request is successful.
|
|
|
|
Win32 error code on failure.
|
|
|
|
--*/
|
|
{
|
|
DWORD Status;
|
|
HANDLE hEventHandle;
|
|
BOOL result;
|
|
PNM_NODE Node;
|
|
HANDLE handleArr[2];
|
|
|
|
|
|
CL_ASSERT(AsyncState->u.hEvent != NULL);
|
|
|
|
//
|
|
// Initialize the async RPC tracking information
|
|
//
|
|
hEventHandle = AsyncState->u.hEvent;
|
|
AsyncState->u.hEvent = NULL;
|
|
|
|
|
|
Status = RpcAsyncInitializeHandle(AsyncState, sizeof(RPC_ASYNC_STATE));
|
|
AsyncState->u.hEvent = hEventHandle;
|
|
|
|
if (Status != RPC_S_OK) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] UpdateRemoteNode: Failed to initialize async RPC status "
|
|
"block, status %1!u!\n",
|
|
Status
|
|
);
|
|
|
|
return (Status);
|
|
}
|
|
|
|
AsyncState->UserInfo = NULL;
|
|
AsyncState->NotificationType = RpcNotificationTypeEvent;
|
|
|
|
|
|
result = ResetEvent(AsyncState->u.hEvent);
|
|
CL_ASSERT(result != 0);
|
|
|
|
// Now hook onto NM node state down event mechanism to detect node downs,
|
|
// instead of NmStartRpc()/NmEndRpc().
|
|
Node = NmReferenceNodeById(RemoteNodeId);
|
|
CL_ASSERT(Node != NULL);
|
|
handleArr[0] = AsyncState->u.hEvent;
|
|
handleArr[1] = NmGetNodeStateDownEvent(Node);
|
|
|
|
try {
|
|
|
|
Status = GumUpdateNode(
|
|
AsyncState,
|
|
GumpRpcBindings[RemoteNodeId],
|
|
UpdateType,
|
|
Context,
|
|
Sequence,
|
|
BufferLength,
|
|
Buffer
|
|
);
|
|
|
|
if (Status == RPC_S_OK) {
|
|
DWORD RpcStatus;
|
|
DWORD WaitStatus;
|
|
//
|
|
// The call is pending. Wait for completion.
|
|
//
|
|
WaitStatus = WaitForMultipleObjects(
|
|
2,
|
|
handleArr,
|
|
FALSE,
|
|
INFINITE
|
|
);
|
|
|
|
if (WaitStatus != WAIT_OBJECT_0) {
|
|
//
|
|
// Something went wrong.
|
|
// Either this is a rpc failure or, the target node went down. In either case
|
|
// the error path is the same, complete the call and evict the target node
|
|
// (eviction is done by the caller of this function).
|
|
//
|
|
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: WaitforMultipleObjects returned %1!u!\n",
|
|
WaitStatus
|
|
);
|
|
if (WaitStatus == WAIT_FAILED) {
|
|
Status = GetLastError();
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: WaitforMultipleObjects returned WAIT_FAILED, status %1!u!\n",
|
|
Status);
|
|
//SS: unexpected error - kill yourself
|
|
CsInconsistencyHalt(Status);
|
|
}
|
|
else if (WaitStatus != (WAIT_OBJECT_0 + 1)) {
|
|
Status = GetLastError();
|
|
//wait objects abandoned - can that happen with events?
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: WaitforMultipleObjects failed, status %1!u!\n",
|
|
Status);
|
|
//SS: unexpected error - kill yourself
|
|
CsInconsistencyHalt(Status);
|
|
|
|
}
|
|
// SS: we only come here if the remote node is signalled to be down
|
|
// make sure that a non-zero status is returned to the caller
|
|
// so that the gum eviction occurs as desirable
|
|
//
|
|
// Cancel the call, just to be safe.
|
|
//
|
|
RpcStatus = RpcAsyncCancelCall(
|
|
AsyncState,
|
|
TRUE // Abortive cancel
|
|
);
|
|
if (RpcStatus != RPC_S_OK) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: RpcAsyncCancelCall()= %1!u!\n",
|
|
RpcStatus
|
|
);
|
|
Status = RpcStatus;
|
|
}
|
|
else {
|
|
CL_ASSERT(RpcStatus == RPC_S_OK);
|
|
|
|
//
|
|
// Wait for the call to complete.
|
|
//
|
|
WaitStatus = WaitForSingleObject(
|
|
AsyncState->u.hEvent,
|
|
INFINITE
|
|
);
|
|
if (WaitStatus != WAIT_OBJECT_0) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: WaitForSingleObject() returns= %1!u!\n",
|
|
WaitStatus);
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: Mapping Status to WAIT_FAILED\n");
|
|
|
|
//SS: if this call doesnt complete, there is something
|
|
//strange with RPC - should we kill ourselves or kill the other
|
|
//node
|
|
//SS: for now we asssume that the problem is not local
|
|
Status = WAIT_FAILED;
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// The call should now be complete. Get the
|
|
// completion status. Any RPC error will be
|
|
// returned in 'RpcStatus'. If there was no
|
|
// RPC error, then any application error will
|
|
// be returned in 'Status'.
|
|
//
|
|
RpcStatus = RpcAsyncCompleteCall(
|
|
AsyncState,
|
|
&Status
|
|
);
|
|
|
|
if (RpcStatus != RPC_S_OK) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: Failed to get "
|
|
"completion status for async RPC call,"
|
|
"status %1!u!\n",
|
|
RpcStatus
|
|
);
|
|
Status = RpcStatus;
|
|
}
|
|
}
|
|
else {
|
|
// An error was returned synchronously.
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumUpdateRemoteNode: GumUpdateNode() failed synchronously, status %1!u!\n",
|
|
Status
|
|
);
|
|
}
|
|
|
|
OmDereferenceObject(Node);
|
|
|
|
} except (I_RpcExceptionFilter(RpcExceptionCode())) {
|
|
OmDereferenceObject(Node);
|
|
Status = GetExceptionCode();
|
|
}
|
|
|
|
return(Status);
|
|
|
|
} // GumpUpdateRemoteNode
|
|
|
|
|
|
DWORD
|
|
WINAPI
|
|
GumSendUpdateReturnInfo(
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD Context,
|
|
OUT PGUM_NODE_UPDATE_HANDLER_STATUS ReturnStatusArray,
|
|
IN DWORD BufferLength,
|
|
IN PVOID Buffer
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Sends an update to all active nodes in the cluster. All
|
|
registered update handlers for the specified UpdateType
|
|
are called on each node. Any registered update handlers
|
|
for the current node will be called on the same thread.
|
|
This is useful for correct synchronization of the data
|
|
structures to be updated.
|
|
|
|
Arguments:
|
|
|
|
UpdateType - Supplies the type of update. This determines
|
|
which update handlers will be called and the sequence
|
|
number to be used.
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
ReturnStatusArray - Pointer to an array of structures to be filled in
|
|
with the return value from the update handler on each node. The
|
|
array is indexed by node ID. The array must be at least
|
|
(NmMaxNodeId + 1) entries in length.
|
|
|
|
BufferLength - Supplies the length of the update buffer to
|
|
be passed to the update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed
|
|
to the update handlers.
|
|
|
|
Return Value:
|
|
|
|
ERROR_SUCCESS if the request is successful.
|
|
|
|
Win32 error code on failure.
|
|
|
|
--*/
|
|
{
|
|
DWORD Sequence;
|
|
DWORD Status=RPC_S_OK;
|
|
DWORD i;
|
|
PGUM_INFO GumInfo;
|
|
DWORD MyNodeId;
|
|
DWORD LockerNode;
|
|
RPC_ASYNC_STATE AsyncState;
|
|
DWORD GenerationNum; //the generation number wrt to the locker at which the lock is obtained
|
|
BOOL AssumeLockerWhistler = TRUE;
|
|
|
|
CL_ASSERT(UpdateType < GumUpdateMaximum);
|
|
|
|
//
|
|
// Prepare for async RPC. We do this here to avoid hitting a failure
|
|
// after the update is already in progress.
|
|
//
|
|
ZeroMemory((PVOID) &AsyncState, sizeof(RPC_ASYNC_STATE));
|
|
|
|
AsyncState.u.hEvent = CreateEvent(
|
|
NULL, // no attributes
|
|
TRUE, // manual reset
|
|
FALSE, // initial state unsignalled
|
|
NULL // no object name
|
|
);
|
|
|
|
if (AsyncState.u.hEvent == NULL) {
|
|
Status = GetLastError();
|
|
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumSendUpdate: Failed to allocate event object for async "
|
|
"RPC call, status %1!u!\n",
|
|
Status
|
|
);
|
|
|
|
return (Status);
|
|
}
|
|
|
|
//
|
|
// Initialize the return status array
|
|
//
|
|
if (ReturnStatusArray != NULL) {
|
|
for (i=ClusterMinNodeId; i<=NmMaxNodeId; i++) {
|
|
ReturnStatusArray[i].UpdateAttempted = FALSE;
|
|
ReturnStatusArray[i].ReturnStatus = ERROR_NODE_NOT_AVAILABLE;
|
|
}
|
|
}
|
|
|
|
GumInfo = &GumTable[UpdateType];
|
|
MyNodeId = NmGetNodeId(NmLocalNode);
|
|
|
|
// Grab an RPC handle
|
|
GumpStartRpc(MyNodeId);
|
|
|
|
retryLock:
|
|
LockerNode = GumpLockerNode;
|
|
//
|
|
// Send locking update to the locker node.
|
|
//
|
|
if (LockerNode == MyNodeId) {
|
|
//
|
|
// This node is the locker.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
Status = GumpDoLockingUpdate(UpdateType, MyNodeId, &Sequence, &GenerationNum);
|
|
if (Status == ERROR_SUCCESS) {
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: Locker dispatching seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
Status = GumpDispatchUpdate(UpdateType,
|
|
Context,
|
|
TRUE,
|
|
TRUE,
|
|
BufferLength,
|
|
Buffer);
|
|
|
|
if (ReturnStatusArray != NULL) {
|
|
ReturnStatusArray[MyNodeId].UpdateAttempted = TRUE;
|
|
ReturnStatusArray[MyNodeId].ReturnStatus = Status;
|
|
}
|
|
|
|
if (Status != ERROR_SUCCESS) {
|
|
//
|
|
// Note we have to use Sequence-1 for the unlock because GumpDispatchUpdate
|
|
// failed and did not increment the sequence number.
|
|
//
|
|
GumpDoUnlockingUpdate(UpdateType, Sequence-1, MyNodeId, GenerationNum);
|
|
}
|
|
}
|
|
} else {
|
|
// CL_ASSERT(GumpRpcBindings[i] != NULL);
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: queuing update\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
AssumeLockerWhistler = TRUE;
|
|
|
|
RetryLockForRollingUpgrade:
|
|
try {
|
|
NmStartRpc(LockerNode);
|
|
if (AssumeLockerWhistler)
|
|
{
|
|
Status = GumQueueLockingUpdate2(GumpRpcBindings[LockerNode],
|
|
MyNodeId,
|
|
UpdateType,
|
|
Context,
|
|
&Sequence,
|
|
BufferLength,
|
|
Buffer,
|
|
&GenerationNum);
|
|
}
|
|
else
|
|
{
|
|
//call the win2K version
|
|
Status = GumQueueLockingUpdate(GumpRpcBindings[LockerNode],
|
|
MyNodeId,
|
|
UpdateType,
|
|
Context,
|
|
&Sequence,
|
|
BufferLength,
|
|
Buffer);
|
|
}
|
|
NmEndRpc(LockerNode);
|
|
} except (I_RpcExceptionFilter(RpcExceptionCode())) {
|
|
//
|
|
// An exception from RPC indicates that the other node is either dead
|
|
// or insane. Kill it and retry with a new locker.
|
|
//
|
|
|
|
NmEndRpc(LockerNode);
|
|
|
|
Status = GetExceptionCode();
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumSendUpdate: GumQueueLocking update to node %1!d! failed with %2!d!\n",
|
|
LockerNode,
|
|
Status);
|
|
if (Status == RPC_S_PROCNUM_OUT_OF_RANGE)
|
|
{
|
|
//the locker node is win2K, try the old interface
|
|
AssumeLockerWhistler = FALSE;
|
|
goto RetryLockForRollingUpgrade;
|
|
}
|
|
else
|
|
{
|
|
GumpCommFailure(GumInfo,
|
|
LockerNode,
|
|
GetExceptionCode(),
|
|
TRUE);
|
|
//
|
|
// The GUM update handler must have been called to select a new locker
|
|
// node.
|
|
//
|
|
CL_ASSERT(LockerNode != GumpLockerNode);
|
|
|
|
//
|
|
// Retry the locking update with the new locker node.
|
|
//
|
|
goto retryLock;
|
|
}
|
|
}
|
|
|
|
if (ReturnStatusArray != NULL) {
|
|
ReturnStatusArray[LockerNode].UpdateAttempted = TRUE;
|
|
ReturnStatusArray[LockerNode].ReturnStatus = Status;
|
|
}
|
|
|
|
if (Status == ERROR_SUCCESS) {
|
|
CL_ASSERT(Sequence == GumpSequence);
|
|
}
|
|
|
|
if (Status != RPC_S_OK) {
|
|
NmDumpRpcExtErrorInfo(Status);
|
|
}
|
|
|
|
//because there is no synchronization between join and regroups/gumprocessing
|
|
//the old locker node may die and may come up again and not be the locker
|
|
//anymore. We have to take care of this case.
|
|
if (Status == ERROR_CLUSTER_GUM_NOT_LOCKER)
|
|
{
|
|
goto retryLock;
|
|
}
|
|
}
|
|
if (Status != ERROR_SUCCESS) {
|
|
ClRtlLogPrint(LOG_UNUSUAL,
|
|
"[GUM] Queued lock attempt for send type %1!d! failed %2!d!\n",
|
|
UpdateType,
|
|
Status);
|
|
// signal end of RPC handle
|
|
GumpEndRpc(MyNodeId);
|
|
if (AsyncState.u.hEvent != NULL) {
|
|
CloseHandle(AsyncState.u.hEvent);
|
|
}
|
|
return(Status);
|
|
}
|
|
|
|
//
|
|
// Grap the sendupdate lock to serialize with any replays
|
|
//
|
|
EnterCriticalSection(&GumpSendUpdateLock);
|
|
if (LockerNode != GumpLockerNode) {
|
|
//
|
|
// Locker node changed, we need to restart again.
|
|
//
|
|
LeaveCriticalSection(&GumpSendUpdateLock);
|
|
goto retryLock;
|
|
}
|
|
|
|
//
|
|
// The update is now committed on the locker node. All remaining nodes
|
|
// must be updated successfully, or they will be killed.
|
|
//
|
|
for (i=LockerNode+1; i != LockerNode; i++) {
|
|
if (i == (NmMaxNodeId + 1)) {
|
|
i=ClusterMinNodeId;
|
|
if (i==LockerNode) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (GumInfo->ActiveNode[i]) {
|
|
//
|
|
// Dispatch the update to the specified node.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context,
|
|
i);
|
|
if (i == MyNodeId) {
|
|
Status = GumpDispatchUpdate(UpdateType,
|
|
Context,
|
|
FALSE,
|
|
TRUE,
|
|
BufferLength,
|
|
Buffer);
|
|
|
|
|
|
if (ReturnStatusArray != NULL) {
|
|
ReturnStatusArray[i].UpdateAttempted = TRUE;
|
|
ReturnStatusArray[i].ReturnStatus = Status;
|
|
}
|
|
|
|
if (Status != ERROR_SUCCESS){
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumSendUpdate: Update on non-locker node(self) failed with %1!d! when it must succeed\n",
|
|
Status);
|
|
//Commit Suicide
|
|
CsInconsistencyHalt(Status);
|
|
}
|
|
|
|
} else {
|
|
DWORD dwStatus;
|
|
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
dwStatus = GumpUpdateRemoteNode(
|
|
&AsyncState,
|
|
i,
|
|
UpdateType,
|
|
Context,
|
|
Sequence,
|
|
BufferLength,
|
|
Buffer
|
|
);
|
|
|
|
if (ReturnStatusArray != NULL) {
|
|
ReturnStatusArray[i].UpdateAttempted = TRUE;
|
|
ReturnStatusArray[i].ReturnStatus = dwStatus;
|
|
}
|
|
|
|
//
|
|
// If the update on the other node failed, then the
|
|
// other node must now be out of the cluster since the
|
|
// update has already completed on the locker node.
|
|
//
|
|
if (dwStatus != ERROR_SUCCESS) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumSendUpdate: Update on node %1!d! failed with %2!d! when it must succeed\n",
|
|
i,
|
|
dwStatus);
|
|
|
|
NmDumpRpcExtErrorInfo(dwStatus);
|
|
|
|
GumpCommFailure(GumInfo,
|
|
i,
|
|
dwStatus,
|
|
TRUE);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//
|
|
// Our update is over
|
|
//
|
|
LeaveCriticalSection(&GumpSendUpdateLock);
|
|
|
|
//
|
|
// All nodes have been updated. Send unlocking update.
|
|
//
|
|
if (LockerNode == MyNodeId) {
|
|
GumpDoUnlockingUpdate(UpdateType, Sequence, MyNodeId, GenerationNum);
|
|
} else {
|
|
//SS: We will assume that AssumeLockerWhistler is set appropriately when the lock was acquired
|
|
try {
|
|
NmStartRpc(LockerNode);
|
|
if (AssumeLockerWhistler)
|
|
{
|
|
//SS: the sequence number will protect if the locker has gone down
|
|
//and come back up since we got the lock and tried to release it
|
|
Status = GumUnlockUpdate2(
|
|
GumpRpcBindings[LockerNode],
|
|
UpdateType,
|
|
Sequence,
|
|
MyNodeId,
|
|
GenerationNum
|
|
);
|
|
}
|
|
else
|
|
{
|
|
Status = GumUnlockUpdate(
|
|
GumpRpcBindings[LockerNode],
|
|
UpdateType,
|
|
Sequence);
|
|
}
|
|
NmEndRpc(LockerNode);
|
|
} except (I_RpcExceptionFilter(RpcExceptionCode())) {
|
|
//
|
|
// The locker node has crashed. Notify the NM, it will call our
|
|
// notification routine to select a new locker node. Then retry
|
|
// the unlock on the new locker node.
|
|
// SS: changed to not retry unlocks..the new locker node will
|
|
// unlock after propagating this change in any case.
|
|
//
|
|
NmEndRpc(LockerNode);
|
|
Status = GetExceptionCode();
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumSendUpdate: Unlocking update to node %1!d! failed with %2!d!\n",
|
|
LockerNode,
|
|
Status);
|
|
GumpCommFailure(GumInfo,
|
|
LockerNode,
|
|
Status,
|
|
TRUE);
|
|
CL_ASSERT(LockerNode != GumpLockerNode);
|
|
}
|
|
|
|
if(Status != RPC_S_OK) {
|
|
NmDumpRpcExtErrorInfo(Status);
|
|
}
|
|
}
|
|
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumSendUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
// signal end of RPC handle
|
|
GumpEndRpc(MyNodeId);
|
|
|
|
if (AsyncState.u.hEvent != NULL) {
|
|
CloseHandle(AsyncState.u.hEvent);
|
|
}
|
|
|
|
return(ERROR_SUCCESS);
|
|
|
|
} // GumSendUpdateReturnInfo
|
|
|
|
|
|
#ifdef GUM_POST_SUPPORT
|
|
|
|
John Vert (jvert) 11/18/1996
|
|
POST is disabled for now since nobody uses it.
|
|
N.B. The below code does not handle locker node failures
|
|
|
|
|
|
DWORD
|
|
WINAPI
|
|
GumPostUpdate(
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD Context,
|
|
IN DWORD BufferLength,
|
|
IN PVOID Buffer // THIS WILL BE FREED
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Posts an update to all active nodes in the cluster. All
|
|
registered update handlers for the specified UpdateType
|
|
are called on each node. The update will not be reported
|
|
on the current node. The update will not necessarily have
|
|
completed when this function returns, but will complete
|
|
eventually if the current node does not fail.
|
|
|
|
Arguments:
|
|
|
|
UpdateType - Supplies the type of update. This determines
|
|
which update handlers will be called and the sequence
|
|
number to be used.
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
BufferLength - Supplies the length of the update buffer to
|
|
be passed to the update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed
|
|
to the update handlers. THIS BUFFER WILL BE FREED ONCE THE
|
|
POST HAS COMPLETED.
|
|
|
|
Return Value:
|
|
|
|
ERROR_SUCCESS if the request is successful.
|
|
|
|
Win32 error code on failure.
|
|
|
|
|
|
--*/
|
|
{
|
|
DWORD Sequence;
|
|
DWORD Status;
|
|
DWORD i;
|
|
BOOL IsLocker = TRUE;
|
|
PGUM_INFO GumInfo;
|
|
DWORD MyNodeId;
|
|
DWORD LockerNode=(DWORD)-1;
|
|
|
|
CL_ASSERT(UpdateType < GumUpdateMaximum);
|
|
|
|
GumInfo = &GumTable[UpdateType];
|
|
MyNodeId = NmGetNodeId(NmLocalNode);
|
|
|
|
//
|
|
// Find the lowest active node in the cluster. This is the
|
|
// locker.
|
|
for (i=ClusterMinNodeId; i <= NmMaxNodeId; i++) {
|
|
if (GumInfo->ActiveNode[i]) {
|
|
LockerNode = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
CL_ASSERT(i <= NmMaxNodeId);
|
|
|
|
//
|
|
// Post a locking update to the locker node. If this succeeds
|
|
// immediately, we can go do the work directly. If it pends,
|
|
// the locker node will call us back when it is our turn to
|
|
// make the updates.
|
|
//
|
|
if (i == MyNodeId) {
|
|
//
|
|
// This node is the locker.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumPostUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
Status = GumpDoLockingPost(UpdateType,
|
|
MyNodeId,
|
|
&Sequence,
|
|
Context,
|
|
BufferLength,
|
|
(DWORD)Buffer,
|
|
Buffer);
|
|
if (Status == ERROR_SUCCESS) {
|
|
//
|
|
// Update our sequence number so we stay in sync, even though
|
|
// we aren't dispatching the update.
|
|
//
|
|
GumpSequence += 1;
|
|
}
|
|
} else {
|
|
CL_ASSERT(GumpRpcBindings[i] != NULL);
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumPostUpdate: queuing update\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
Status = GumQueueLockingPost(GumpRpcBindings[i],
|
|
MyNodeId,
|
|
UpdateType,
|
|
Context,
|
|
&Sequence,
|
|
BufferLength,
|
|
Buffer,
|
|
(DWORD)Buffer);
|
|
if (Status == ERROR_SUCCESS) {
|
|
CL_ASSERT(Sequence == GumpSequence);
|
|
}
|
|
}
|
|
|
|
if (Status == ERROR_SUCCESS) {
|
|
//
|
|
// The lock was immediately acquired, go ahead and post directly
|
|
// here.
|
|
//
|
|
GumpDeliverPosts(LockerNode+1,
|
|
UpdateType,
|
|
Sequence,
|
|
Context,
|
|
BufferLength,
|
|
Buffer);
|
|
|
|
//
|
|
// All nodes have been updated. Send unlocking update.
|
|
//
|
|
if (LockerNode == MyNodeId) {
|
|
GumpDoUnlockingUpdate(UpdateType, Sequence);
|
|
} else {
|
|
GumUnlockUpdate(
|
|
GumpRpcBindings[LockerNode],
|
|
UpdateType,
|
|
Sequence
|
|
);
|
|
}
|
|
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumPostUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
return(ERROR_SUCCESS);
|
|
} else {
|
|
//
|
|
// The lock is currently held. We will get called back when it is released
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumPostUpdate: pending update type %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
return(ERROR_IO_PENDING);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
VOID
|
|
GumpDeliverPosts(
|
|
IN DWORD FirstNodeId,
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD Sequence,
|
|
IN DWORD Context,
|
|
IN DWORD BufferLength,
|
|
IN PVOID Buffer // THIS WILL BE FREED
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Actually delivers the update post to the specified nodes.
|
|
The GUM lock is assumed to be held.
|
|
|
|
Arguments:
|
|
|
|
FirstNodeId - Supplies the node ID where the posts should start.
|
|
This is generally the LockerNode+1.
|
|
|
|
UpdateType - Supplies the type of update. This determines
|
|
which update handlers will be called and the sequence
|
|
number to be used.
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
BufferLength - Supplies the length of the update buffer to
|
|
be passed to the update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed
|
|
to the update handlers. THIS BUFFER WILL BE FREED ONCE THE
|
|
POST HAS COMPLETED.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
DWORD i;
|
|
PGUM_INFO GumInfo;
|
|
DWORD MyNodeId;
|
|
|
|
|
|
GumInfo = &GumTable[UpdateType];
|
|
MyNodeId = NmGetNodeId(NmLocalNode);
|
|
|
|
for (i=FirstNodeId; i<=NmMaxNodeId; i++) {
|
|
if (GumInfo->ActiveNode[i]) {
|
|
//
|
|
// Dispatch the update to the specified node.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumpDeliverPosts: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context,
|
|
i);
|
|
if (i == MyNodeId) {
|
|
//
|
|
// Update our sequence number so we stay in sync, even though
|
|
// we aren't dispatching the update.
|
|
//
|
|
GumpSequence += 1;
|
|
} else {
|
|
CL_ASSERT(GumpRpcBindings[i] != NULL);
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumpDeliverPosts: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
|
|
|
|
GumUpdateNode(GumpRpcBindings[i],
|
|
UpdateType,
|
|
Context,
|
|
Sequence,
|
|
BufferLength,
|
|
Buffer);
|
|
}
|
|
}
|
|
}
|
|
|
|
LocalFree(Buffer);
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
DWORD
|
|
WINAPI
|
|
GumAttemptUpdate(
|
|
IN DWORD Sequence,
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD Context,
|
|
IN DWORD BufferLength,
|
|
IN PVOID Buffer
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Conditionally sends an update to all active nodes in the
|
|
cluster. If the clusterwise sequence number matches the supplied
|
|
sequence number, all registered update handlers for the specified
|
|
UpdateType are called on each node. Any registered update handlers
|
|
for the current node will be called on the same thread. This is
|
|
useful for correct synchronization of the data structures to be updated.
|
|
|
|
The normal usage of this routine is as follows:
|
|
· obtain current sequence number from GumGetCurrentSequence
|
|
· make modification to cluster state
|
|
· conditionally update cluster state with GumAttemptUpdate
|
|
· If update fails, undo modification, release any locks, try again later
|
|
|
|
Arguments:
|
|
|
|
Sequence - Supplies the sequence number obtained from GumGetCurrentSequence.
|
|
|
|
UpdateType - Supplies the type of update. This determines which update handlers
|
|
will be called
|
|
|
|
Context - Supplies a DWORD of context to be passed to the
|
|
GUM update handlers
|
|
|
|
BufferLength - Supplies the length of the update buffer to be passed to the
|
|
update handlers
|
|
|
|
Buffer - Supplies a pointer to the update buffer to be passed to the update
|
|
handlers.
|
|
|
|
Return Value:
|
|
|
|
ERROR_SUCCESS if the request is successful.
|
|
|
|
Win32 error code on failure.
|
|
|
|
|
|
--*/
|
|
{
|
|
DWORD Status=RPC_S_OK;
|
|
DWORD i;
|
|
PGUM_INFO GumInfo;
|
|
DWORD MyNodeId;
|
|
DWORD LockerNode=(DWORD)-1;
|
|
RPC_ASYNC_STATE AsyncState;
|
|
DWORD dwGenerationNum; //the generation id of the node at which the lock is acquired
|
|
|
|
CL_ASSERT(UpdateType < GumUpdateMaximum);
|
|
|
|
ZeroMemory((PVOID) &AsyncState, sizeof(RPC_ASYNC_STATE));
|
|
|
|
AsyncState.u.hEvent = CreateEvent(
|
|
NULL, // no attributes
|
|
TRUE, // manual reset
|
|
FALSE, // initial state unsignalled
|
|
NULL // no object name
|
|
);
|
|
|
|
if (AsyncState.u.hEvent == NULL) {
|
|
Status = GetLastError();
|
|
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumAttemptUpdate: Failed to allocate event object for "
|
|
"async RPC call, status %1!u!\n",
|
|
Status
|
|
);
|
|
|
|
return (Status);
|
|
}
|
|
|
|
|
|
GumInfo = &GumTable[UpdateType];
|
|
MyNodeId = NmGetNodeId(NmLocalNode);
|
|
|
|
retryLock:
|
|
LockerNode = GumpLockerNode;
|
|
|
|
//
|
|
// Send locking update to the locker node.
|
|
//
|
|
if (LockerNode == MyNodeId)
|
|
{
|
|
//
|
|
// This node is the locker.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: Locker waiting\t\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
|
|
if (GumpTryLockingUpdate(UpdateType, MyNodeId, Sequence, &dwGenerationNum))
|
|
{
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: Locker dispatching seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
Status = GumpDispatchUpdate(UpdateType,
|
|
Context,
|
|
TRUE,
|
|
TRUE,
|
|
BufferLength,
|
|
Buffer);
|
|
if (Status != ERROR_SUCCESS) {
|
|
//
|
|
// Note we have to use Sequence-1 for the unlock because GumpDispatchUpdate
|
|
// failed and did not increment the sequence number.
|
|
//
|
|
GumpDoUnlockingUpdate(UpdateType, Sequence-1, MyNodeId, dwGenerationNum);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Status = ERROR_CLUSTER_DATABASE_SEQMISMATCH;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//
|
|
//send the locking update to the locker node
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: queuing update\ttype %1!u! context %2!u!\n",
|
|
UpdateType,
|
|
Context);
|
|
try {
|
|
NmStartRpc(LockerNode);
|
|
Status = GumAttemptLockingUpdate(GumpRpcBindings[LockerNode],
|
|
MyNodeId,
|
|
UpdateType,
|
|
Context,
|
|
Sequence,
|
|
BufferLength,
|
|
Buffer);
|
|
NmEndRpc(LockerNode);
|
|
} except (I_RpcExceptionFilter(RpcExceptionCode())) {
|
|
//
|
|
// An exception from RPC indicates that the other node is either dead
|
|
// or insane. Kill it and retry with a new locker.
|
|
//
|
|
NmEndRpc(LockerNode);
|
|
GumpCommFailure(GumInfo,
|
|
LockerNode,
|
|
GetExceptionCode(),
|
|
TRUE);
|
|
|
|
//
|
|
// The GUM update handler must have been called to select a new locker
|
|
// node.
|
|
//
|
|
CL_ASSERT(LockerNode != GumpLockerNode);
|
|
|
|
//
|
|
// Retry the locking update with the new locker node.
|
|
//
|
|
goto retryLock;
|
|
}
|
|
if (Status == ERROR_SUCCESS)
|
|
{
|
|
CL_ASSERT(Sequence == GumpSequence);
|
|
}
|
|
|
|
if(Status != RPC_S_OK) {
|
|
NmDumpRpcExtErrorInfo(Status);
|
|
}
|
|
|
|
}
|
|
|
|
if (Status != ERROR_SUCCESS) {
|
|
ClRtlLogPrint(LOG_UNUSUAL,
|
|
"[GUM] GumAttemptUpdate: Queued lock attempt for send type %1!d! failed %2!d!\n",
|
|
UpdateType,
|
|
Status);
|
|
return(Status);
|
|
}
|
|
|
|
//
|
|
// Grap the sendupdate lock to serialize with any replays
|
|
//
|
|
EnterCriticalSection(&GumpSendUpdateLock);
|
|
if (LockerNode != GumpLockerNode) {
|
|
//
|
|
// Locker node changed, we need to restart again.
|
|
//
|
|
LeaveCriticalSection(&GumpSendUpdateLock);
|
|
goto retryLock;
|
|
}
|
|
|
|
|
|
// The update is now committed on the locker node. All remaining nodes
|
|
// must be updated successfully, or they will be killed.
|
|
//
|
|
for (i=LockerNode+1; i != LockerNode; i++)
|
|
{
|
|
if (i == (NmMaxNodeId + 1))
|
|
{
|
|
i=ClusterMinNodeId;
|
|
if (i==LockerNode)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (GumInfo->ActiveNode[i])
|
|
{
|
|
//
|
|
// Dispatch the update to the specified node.
|
|
//
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: Dispatching seq %1!u!\ttype %2!u! context %3!u! to node %4!d!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context,
|
|
i);
|
|
if (i == MyNodeId) {
|
|
Status = GumpDispatchUpdate(UpdateType,
|
|
Context,
|
|
FALSE,
|
|
TRUE,
|
|
BufferLength,
|
|
Buffer);
|
|
if (Status != ERROR_SUCCESS){
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumAttemptUpdate: Update on non-locker node(self) failed with %1!d! when it must succeed\n",
|
|
Status);
|
|
//Commit Suicide
|
|
CsInconsistencyHalt(Status);
|
|
}
|
|
|
|
} else {
|
|
DWORD dwStatus;
|
|
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: Locker updating seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
dwStatus = GumpUpdateRemoteNode(
|
|
&AsyncState,
|
|
i,
|
|
UpdateType,
|
|
Context,
|
|
Sequence,
|
|
BufferLength,
|
|
Buffer
|
|
);
|
|
|
|
//
|
|
// If the update on the other node failed, then the
|
|
// other node must now be out of the cluster since the
|
|
// update has already completed on the locker node.
|
|
//
|
|
if (dwStatus != ERROR_SUCCESS) {
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumAttemptUpdate: Update on node %1!d! failed with %2!d! when it must succeed\n",
|
|
i,
|
|
dwStatus);
|
|
|
|
NmDumpRpcExtErrorInfo(dwStatus);
|
|
|
|
GumpCommFailure(GumInfo,
|
|
i,
|
|
dwStatus,
|
|
TRUE);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
//
|
|
// Our update is over
|
|
//
|
|
LeaveCriticalSection(&GumpSendUpdateLock);
|
|
|
|
//
|
|
// All nodes have been updated. Send unlocking update.
|
|
//
|
|
if (LockerNode == MyNodeId) {
|
|
GumpDoUnlockingUpdate(UpdateType, Sequence, MyNodeId, dwGenerationNum);
|
|
} else {
|
|
try {
|
|
NmStartRpc(LockerNode);
|
|
Status = GumUnlockUpdate(
|
|
GumpRpcBindings[LockerNode],
|
|
UpdateType,
|
|
Sequence
|
|
);
|
|
NmEndRpc(LockerNode);
|
|
} except (I_RpcExceptionFilter(RpcExceptionCode())) {
|
|
//
|
|
// The locker node has crashed. Notify the NM, it will call our
|
|
// notification routine to select a new locker node. The new
|
|
// locker node will release the gum lock after propagating
|
|
// the current update.
|
|
//
|
|
NmEndRpc(LockerNode);
|
|
Status = GetExceptionCode();
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumAttemptUpdate: Unlocking update to node %1!d! failed with %2!d!\n",
|
|
LockerNode,
|
|
Status);
|
|
GumpCommFailure(GumInfo,
|
|
LockerNode,
|
|
Status,
|
|
TRUE);
|
|
CL_ASSERT(LockerNode != GumpLockerNode);
|
|
}
|
|
|
|
if(Status != RPC_S_OK) {
|
|
NmDumpRpcExtErrorInfo(Status);
|
|
}
|
|
}
|
|
|
|
ClRtlLogPrint(LOG_NOISE,
|
|
"[GUM] GumAttemptUpdate: completed update seq %1!u!\ttype %2!u! context %3!u!\n",
|
|
Sequence,
|
|
UpdateType,
|
|
Context);
|
|
|
|
if (AsyncState.u.hEvent != NULL) {
|
|
CloseHandle(AsyncState.u.hEvent);
|
|
}
|
|
|
|
return(ERROR_SUCCESS);
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
DWORD
|
|
WINAPI
|
|
GumGetCurrentSequence(
|
|
IN GUM_UPDATE_TYPE UpdateType
|
|
)
|
|
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Obtains the current clusterwise global update sequence number
|
|
|
|
Arguments:
|
|
|
|
UpdateType - Supplies the type of update. Each update type may
|
|
have an independent sequence number.
|
|
|
|
Return Value:
|
|
|
|
Current global update sequence number for the specified update type.
|
|
|
|
--*/
|
|
|
|
{
|
|
CL_ASSERT(UpdateType < GumUpdateMaximum);
|
|
|
|
return(GumpSequence);
|
|
}
|
|
|
|
|
|
VOID
|
|
GumSetCurrentSequence(
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
DWORD Sequence
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Sets the current sequence for the specified global update.
|
|
|
|
Arguments:
|
|
|
|
UpdateType - Supplies the update type whose sequence is to be updated.
|
|
|
|
Sequence - Supplies the new sequence number.
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
CL_ASSERT(UpdateType < GumUpdateMaximum);
|
|
|
|
GumpSequence = Sequence;
|
|
|
|
}
|
|
|
|
|
|
VOID
|
|
GumCommFailure(
|
|
IN GUM_UPDATE_TYPE UpdateType,
|
|
IN DWORD NodeId,
|
|
IN DWORD ErrorCode,
|
|
IN BOOL Wait
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Informs the NM that a fatal communication error has occurred trying
|
|
to talk to another node.
|
|
|
|
Arguments:
|
|
|
|
GumInfo - Supplies the update type where the communication failure occurred.
|
|
|
|
NodeId - Supplies the node id of the other node.
|
|
|
|
ErrorCode - Supplies the error that was returned from RPC
|
|
|
|
Wait - if TRUE, this function blocks until the GUM event handler has
|
|
processed the NodeDown notification for the specified node.
|
|
|
|
if FALSE, this function returns immediately after notifying NM
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
PGUM_INFO GumInfo = &GumTable[UpdateType];
|
|
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumCommFailure %1!d! communicating with node %2!d!\n",
|
|
ErrorCode,
|
|
NodeId);
|
|
|
|
|
|
GumpCommFailure(GumInfo, NodeId, ErrorCode, Wait);
|
|
}
|
|
|
|
|
|
VOID
|
|
GumpCommFailure(
|
|
IN PGUM_INFO GumInfo,
|
|
IN DWORD NodeId,
|
|
IN DWORD ErrorCode,
|
|
IN BOOL Wait
|
|
)
|
|
/*++
|
|
|
|
Routine Description:
|
|
|
|
Informs the NM that a fatal communication error has occurred trying
|
|
to talk to another node.
|
|
|
|
Arguments:
|
|
|
|
GumInfo - Supplies the update type where the communication failure occurred.
|
|
|
|
NodeId - Supplies the node id of the other node.
|
|
|
|
ErrorCode - Supplies the error that was returned from RPC
|
|
|
|
Wait - if TRUE, this function blocks until the GUM event handler has
|
|
processed the NodeDown notification for the specified node.
|
|
|
|
if FALSE, this function returns immediately after notifying NM
|
|
|
|
Return Value:
|
|
|
|
None.
|
|
|
|
--*/
|
|
|
|
{
|
|
DWORD dwCur;
|
|
|
|
ClRtlLogPrint(LOG_CRITICAL,
|
|
"[GUM] GumpCommFailure %1!d! communicating with node %2!d!\n",
|
|
ErrorCode,
|
|
NodeId);
|
|
|
|
// This is the general GUM RPC failure path, let's dump the extended error info.
|
|
// NOTE: The dumping routine is benign, so calling this from a non RPC failure path would just return.
|
|
NmDumpRpcExtErrorInfo(ErrorCode);
|
|
|
|
|
|
// This is a hack to check if we are shutting down. See bug 88411
|
|
if (ErrorCode == ERROR_SHUTDOWN_IN_PROGRESS) {
|
|
// if we are shutting down, just kill self
|
|
// set to our node id
|
|
NodeId = NmGetNodeId(NmLocalNode);
|
|
}
|
|
|
|
|
|
//
|
|
// Get current generation number
|
|
//
|
|
if (Wait) {
|
|
dwCur = GumpGetNodeGenNum(GumInfo, NodeId);
|
|
}
|
|
|
|
NmAdviseNodeFailure(NodeId, ErrorCode);
|
|
|
|
if (Wait) {
|
|
//
|
|
// Wait for this node to be declared down and
|
|
// GumpEventHandler to mark it as inactive.
|
|
//
|
|
|
|
GumpWaitNodeDown(NodeId, dwCur);
|
|
}
|
|
}
|
|
|