|
|
/*++
Copyright (c) 1996-1999 Microsoft Corporation
Module Name:
nminit.c
Abstract:
Initialization, cluster join, and cluster form routines for the Node Manager.
Author:
Mike Massa (mikemas)
Revision History:
6/03/96 Created.
--*/
/*
General Implementation Notes:
The functions DmBeginLocalUpdate, DmCommitLocalUpdate, and DmAbortLocalUpdate cannot be called while holding the NM lock, or a deadlock with the NmTimer thread may result during regroup when disk writes are stalled. These functions attempt to write to the quorum disk.
*/
#include "nmp.h"
#include <ntmsv1_0.h>
#define CLUSREG_NAME_DISABLE_TRY_CACHE_FIRST L"DisableLsaTryCacheFirst"
#define NM_TIMER_THREAD_PRIORITY 15 // same as MM timer thread
//
// External Data
//
extern BOOL CsNoQuorum;
//
// Public Data
//
HANDLE NmClusnetHandle = NULL;HCRYPTPROV NmCryptServiceProvider = 0;
//
// Private Data
//
CRITICAL_SECTION NmpLock;NM_STATE NmpState = NmStateOffline;DWORD NmpActiveThreadCount = 0;HANDLE NmpShutdownEvent = NULL;CL_NODE_ID NmpJoinerNodeId = ClusterInvalidNodeId;CL_NODE_ID NmpSponsorNodeId = ClusterInvalidNodeId;DWORD NmpJoinTimer = 0;BOOLEAN NmpJoinAbortPending = FALSE;DWORD NmpJoinSequence = 0;BOOLEAN NmpJoinerUp = FALSE;BOOLEAN NmpJoinBeginInProgress = FALSE;BOOLEAN NmpJoinerOutOfSynch = FALSE;LPWSTR NmpClusnetEndpoint = NULL;WCHAR NmpInvalidJoinerIdString[] = L"0";CL_NODE_ID NmpLeaderNodeId = ClusterInvalidNodeId;BOOL NmpCleanupIfJoinAborted = FALSE;BOOL NmpSuccessfulMMJoin = FALSE;DWORD NmpAddNodeId = ClusterInvalidNodeId;LPWSTR NmpClusterInstanceId = NULL;BOOLEAN NmpGumUpdateHandlerRegistered = FALSE;HANDLE NmpNetworkTimerThreadHandle = NULL;HANDLE NmpNetworkTimerThreadStopEvent = NULL;
//externs
extern DWORD CsMyHighestVersion;extern DWORD CsMyLowestVersion;extern DWORD CsClusterHighestVersion;extern DWORD CsClusterLowestVersion;
GUM_DISPATCH_ENTRY NmGumDispatchTable[] = { {1, NmpUpdateCreateNode}, {1, NmpUpdatePauseNode}, {1, NmpUpdateResumeNode}, {1, NmpUpdateEvictNode}, {4, (PGUM_DISPATCH_ROUTINE1) NmpUpdateCreateNetwork}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetNetworkName}, {1, NmpUpdateSetNetworkPriorityOrder}, {3, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetNetworkCommonProperties}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdateCreateInterface}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetInterfaceInfo}, {3, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetInterfaceCommonProperties}, {1, NmpUpdateDeleteInterface}, {3, (PGUM_DISPATCH_ROUTINE1) NmpUpdateJoinBegin}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdateJoinAbort}, //
// Version 2 (NT 5.0) extensions that are understood by NT4 SP4
//
{5, (PGUM_DISPATCH_ROUTINE1) NmpUpdateJoinBegin2}, {4, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetNetworkAndInterfaceStates}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdatePerformFixups}, {5, (PGUM_DISPATCH_ROUTINE1) NmpUpdatePerformFixups2}, //
// Version 2 (NT 5.0) extensions that are not understood by NT4 SP4
// These may not be called in a mixed NT4/NT5 cluster.
//
{5, (PGUM_DISPATCH_ROUTINE1) NmpUpdateAddNode}, {2, (PGUM_DISPATCH_ROUTINE1) NmpUpdateExtendedNodeState}, //
// NT 5.1 extensions that are not understood by NT5 and
// earlier. NT5 nodes will ignore these updates without
// error.
//
{4, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetNetworkMulticastConfiguration}, {8, (PGUM_DISPATCH_ROUTINE1) NmpUpdateSetServiceAccountPassword}, };
//
// Local prototypes
//
DWORDNmpCreateRpcBindings( IN PNM_NODE Node );
DWORDNmpCreateClusterInstanceId( VOID );
DWORDNmpStartNetworkTimerThread( VOID );
VOIDNmpStopNetworkTimerThread( VOID );
//
// Component initialization routines.
//
DWORDNmInitialize( VOID )/*++
Routine Description:
Initializes the Node Manager component.
Arguments:
None
Return Value:
A Win32 status code.
Notes:
The local node object is created by this routine.
--*/{ DWORD status; OM_OBJECT_TYPE_INITIALIZE nodeTypeInitializer; HDMKEY nodeKey = NULL; DWORD nameSize = CS_MAX_NODE_NAME_LENGTH + 1; HKEY serviceKey; DWORD nodeIdSize = (CS_MAX_NODE_ID_LENGTH + 1) * sizeof(WCHAR); LPWSTR nodeIdString = NULL; WSADATA wsaData; WORD versionRequested; int err; ULONG ndx; DWORD valueType; NM_NODE_INFO2 nodeInfo; WCHAR errorString[12]; DWORD eventCode = 0; LPWSTR string; PVOID wTimer;
ULONG lsaProcessOptions; HDMKEY clusParamKey = NULL;
CL_ASSERT(NmpState == NmStateOffline);
ClRtlLogPrint(LOG_NOISE,"[NM] Initializing...\n");
//
// Initialize globals.
//
InitializeCriticalSection(&NmpLock);
InitializeListHead(&NmpNodeList); InitializeListHead(&NmpNetworkList); InitializeListHead(&NmpInternalNetworkList); InitializeListHead(&NmpDeletedNetworkList); InitializeListHead(&NmpInterfaceList); InitializeListHead(&NmpDeletedInterfaceList);
NmMaxNodes = ClusterDefaultMaxNodes; NmMaxNodeId = ClusterMinNodeId + NmMaxNodes - 1;
//
// Initializing the RPC Recording/cancelling mechanism
// NOTE - This should move if NmMaxNodeId Definition above moves.
//
NmpIntraClusterRpcArr = LocalAlloc(LMEM_FIXED, sizeof(NM_INTRACLUSTER_RPC_THREAD) * (NmMaxNodeId +1));
if(NmpIntraClusterRpcArr == NULL) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to allocate memory for RPC monitoring.\n" ); status = ERROR_NOT_ENOUGH_MEMORY; eventCode = CS_EVENT_ALLOCATION_FAILURE; goto error_exit; } else { ZeroMemory(NmpIntraClusterRpcArr, sizeof(NM_INTRACLUSTER_RPC_THREAD) * (NmMaxNodeId + 1)); for(ndx = 0;ndx <= NmMaxNodeId;ndx++) InitializeListHead(&NmpIntraClusterRpcArr[ndx]);
InitializeCriticalSection(&NmpRPCLock); }
//
// Initialize the network configuration package.
//
ClNetInitialize( ClNetPrint, ClNetLogEvent, ClNetLogEvent1, ClNetLogEvent2, ClNetLogEvent3 );
//
// Initialize WinSock
//
versionRequested = MAKEWORD(2,0);
err = WSAStartup(versionRequested, &wsaData);
if (err != 0) { status = WSAGetLastError(); wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, NM_EVENT_WSASTARTUP_FAILED, errorString); ClRtlLogPrint(LOG_NOISE,"[NM] Failed to initialize Winsock, status %1!u!\n", status); return(status); }
if ( (LOBYTE(wsaData.wVersion) != 2) || (HIBYTE(wsaData.wVersion) != 0)) { status = WSAVERNOTSUPPORTED; wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, NM_EVENT_WSASTARTUP_FAILED, errorString); ClRtlLogPrint(LOG_CRITICAL, "[NM] Found unexpected Windows Sockets version %1!u!\n", wsaData.wVersion ); WSACleanup(); return(status); }
NmpShutdownEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
if (NmpShutdownEvent == NULL) { status = GetLastError(); wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, CS_EVENT_ALLOCATION_FAILURE, errorString); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create shutdown event, status %1!u!\n", status ); WSACleanup(); return(status); }
NmpState = NmStateOnlinePending;
//
// Get the name of this node.
//
if (!GetComputerName(&(NmLocalNodeName[0]), &nameSize)) { status = GetLastError(); eventCode = NM_EVENT_GETCOMPUTERNAME_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to get local computername, status %1!u!\n", status ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE, "[NM] Local node name = %1!ws!.\n", NmLocalNodeName );
//
// Open a control channel to the Cluster Network driver
//
NmClusnetHandle = ClusnetOpenControlChannel(0);
if (NmClusnetHandle == NULL) { status = GetLastError(); eventCode = NM_EVENT_CLUSNET_UNAVAILABLE; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to open a handle to the Cluster Network driver, status %1!u!\n", status ); goto error_exit; }
//
// Tell the Cluster Network driver to shutdown when our handle is closed
// in case the Cluster Service crashes.
//
status = ClusnetEnableShutdownOnClose(NmClusnetHandle);
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CLUSNET_ENABLE_SHUTDOWN_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to register Cluster Network shutdown trigger, status %1!u!\n", status );
goto error_exit; }
//
// Allocate the node ID array.
//
CL_ASSERT(NmpIdArray == NULL);
NmpIdArray = LocalAlloc( LMEM_FIXED, (sizeof(PNM_NODE) * (NmMaxNodeId + 1)) );
if (NmpIdArray == NULL) { status = ERROR_NOT_ENOUGH_MEMORY; eventCode = CS_EVENT_ALLOCATION_FAILURE; goto error_exit; }
ZeroMemory(NmpIdArray, (sizeof(PNM_NODE) * (NmMaxNodeId + 1)));
//
// Create the node object type
//
ZeroMemory(&nodeTypeInitializer, sizeof(OM_OBJECT_TYPE_INITIALIZE)); nodeTypeInitializer.ObjectSize = sizeof(NM_NODE); nodeTypeInitializer.Signature = NM_NODE_SIG; nodeTypeInitializer.Name = L"Node"; nodeTypeInitializer.DeleteObjectMethod = NmpDestroyNodeObject;
status = OmCreateType(ObjectTypeNode, &nodeTypeInitializer);
if (status != ERROR_SUCCESS) { eventCode = CS_EVENT_ALLOCATION_FAILURE; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to create node object type, status %1!u!\n", status ); goto error_exit; }
//
// Get the local node ID from the local registry.
//
status = RegCreateKeyW( HKEY_LOCAL_MACHINE, CLUSREG_KEYNAME_CLUSSVC_PARAMETERS, &serviceKey );
if (status != ERROR_SUCCESS) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, CS_EVENT_REG_OPEN_FAILED, CLUSREG_KEYNAME_CLUSSVC_PARAMETERS, errorString ); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to open cluster service parameters key, status %1!u!\n", status ); goto error_exit; }
string = L"NodeId"; status = RegQueryValueExW( serviceKey, string, 0, &valueType, (LPBYTE) &(NmLocalNodeIdString[0]), &nodeIdSize );
RegCloseKey(serviceKey);
if (status != ERROR_SUCCESS) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, CS_EVENT_REG_QUERY_FAILED, string, errorString ); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to read local node ID from registry, status %1!u!\n", status ); goto error_exit; }
if (valueType != REG_SZ) { status = ERROR_INVALID_PARAMETER; wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, CS_EVENT_REG_QUERY_FAILED, string, errorString ); ClRtlLogPrint(LOG_CRITICAL, "[NM] Local Node ID registry value is not of type REG_SZ.\n" ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE, "[NM] Local node ID = %1!ws!.\n", NmLocalNodeIdString );
NmLocalNodeId = wcstoul(NmLocalNodeIdString, NULL, 10);
//
// Get information about the local node.
//
wcscpy(&(nodeInfo.NodeId[0]), NmLocalNodeIdString);
status = NmpGetNodeDefinition(&nodeInfo);
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Create the local node object. We must do this here because GUM
// requires the local node object to initialize.
//
status = NmpCreateLocalNodeObject(&nodeInfo);
ClNetFreeNodeInfo(&nodeInfo);
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Initialize the network and interface object types
//
status = NmpInitializeNetworks();
if (status != ERROR_SUCCESS) { goto error_exit; }
status = NmpInitializeInterfaces();
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Initialize net PnP handling
//
status = NmpInitializePnp();
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// init the advise sink that tells when a connection object has been
// renamed
//
// Wrapping this call with watchdog timeout of 3 mins. 411333
wTimer = ClRtlSetWatchdogTimer(180000, L"Calling Initialize Connectoid Advise Sink"); status = NmpInitializeConnectoidAdviseSink(); ClRtlCancelWatchdogTimer(wTimer);
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Set the LSA options for this process.
//
// When authenticating, try cached credentials first.
lsaProcessOptions = MSV1_0_OPTION_TRY_CACHE_FIRST;
// Check if a registry parameter commands us to disable the
// try cache first option.
clusParamKey = DmOpenKey( DmClusterParametersKey, CLUSREG_KEYNAME_PARAMETERS, KEY_READ ); if (clusParamKey != NULL) {
DWORD type = REG_DWORD; DWORD disabled = 0; DWORD len = sizeof(disabled);
status = DmQueryValue( clusParamKey, CLUSREG_NAME_DISABLE_TRY_CACHE_FIRST, &type, (LPBYTE) &disabled, &len ); if (status == ERROR_SUCCESS) { // The value was found. Make sure it is a DWORD.
if (type == REG_DWORD) { // If disabled, turn off the cache first option.
if (disabled != 0) { lsaProcessOptions &= ~MSV1_0_OPTION_TRY_CACHE_FIRST; ClRtlLogPrint(LOG_NOISE, "[NM] The LSA cache-first process option will " "not be enabled.\n" ); } } else { // Warn in the cluster log that the type
// is incorrect.
ClRtlLogPrint(LOG_UNUSUAL, "[NM] Ignoring cluster parameter %1!ws! " "because it is not of type REG_DWORD (%2!u!).\n", CLUSREG_NAME_DISABLE_TRY_CACHE_FIRST, type ); } }
} else { status = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[NM] Failed to open cluster parameters key, " "status %1!u!.\n", status ); // Non-fatal error. Stick with default.
}
// Set password policy
if ( CsRunningAsService) {
// Set password policy to allow RPC authentication to ignore a WRONG_PASSWORD
// message from a domain controller and try authentication against the current
// in-memory credentials.
//
// The password change operation is only valid when running as a service.
lsaProcessOptions |= MSV1_0_OPTION_ALLOW_OLD_PASSWORD; }
// Call into LSA to set the options.
if (lsaProcessOptions != 0) { status = NmpSetLsaProcessOptions(lsaProcessOptions); if (status != ERROR_SUCCESS) {
if (!CsRunningAsService && // Not a fatal error if running in debug mode and only trying
// to set cache-first option.
lsaProcessOptions == MSV1_0_OPTION_TRY_CACHE_FIRST) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Failed to enable LSA cache-first process " "option, status %1!u!.\n", status ); status = ERROR_SUCCESS; } else { // Fatal error.
ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to set LSA process options to %1!x!, " "status %2!u!.\n", lsaProcessOptions, status ); goto error_exit; } } }
//
// Initialize NmCryptServiceProvider
//
status = NmpCreateCSPHandle(&NmCryptServiceProvider); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to initialize NmCryptServiceProvider, " "status %1!u!.\n", status ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE,"[NM] Initialization complete.\n");
return(ERROR_SUCCESS);
error_exit:
if (clusParamKey != NULL) { DmCloseKey(clusParamKey); clusParamKey = NULL; }
if (eventCode != 0) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, eventCode, errorString); }
wsprintfW( &(errorString[0]), L"%u", status ); CsLogEvent1(LOG_CRITICAL, NM_INIT_FAILED, errorString);
ClRtlLogPrint(LOG_CRITICAL,"[NM] Initialization failed %1!d!\n",status);
NmShutdown();
return(status);
} // NmInitialize
VOIDNmShutdown( VOID )/*++
Routine Description:
Terminates all processing - shuts down all sources of work for worker threads.
Arguments:
Return Value:
--*/{ DWORD status;
if (NmpState == NmStateOffline) { return; }
NmCloseConnectoidAdviseSink();
NmpShutdownPnp();
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE,"[NM] Shutdown starting...\n");
//
// Release NmCryptServiceProvider.
//
if(NmCryptServiceProvider) { if (!CryptReleaseContext(NmCryptServiceProvider,0)) { ClRtlLogPrint( LOG_UNUSUAL, "NM] Failed to release " "crypto provider, status %1!u!\n", GetLastError() ); } }
NmpState = NmStateOfflinePending;
if (NmpActiveThreadCount > 0) { ClRtlLogPrint(LOG_NOISE, "[NM] Waiting for %1!u! active threads to terminate...\n", NmpActiveThreadCount );
NmpReleaseLock();
status = WaitForSingleObject(NmpShutdownEvent, INFINITE);
CL_ASSERT(status == WAIT_OBJECT_0);
ClRtlLogPrint(LOG_NOISE, "[NM] All active threads have completed. Continuing shutdown...\n" );
} else { NmpReleaseLock(); }
NmLeaveCluster();
NmpCleanupPnp();
if (NmLocalNode != NULL) { NmpDeleteNodeObject(NmLocalNode, FALSE); NmLocalNode = NULL; }
if (NmpIdArray != NULL) { LocalFree(NmpIdArray); NmpIdArray = NULL; }
NmpFreeClusterKey();
if (NmpClusterInstanceId != NULL) { MIDL_user_free(NmpClusterInstanceId); NmpClusterInstanceId = NULL; }
if (NmClusnetHandle != NULL) { ClusnetCloseControlChannel(NmClusnetHandle); NmClusnetHandle = NULL; }
CloseHandle(NmpShutdownEvent); NmpShutdownEvent = NULL;
WSACleanup();
//
// As long as the GUM and Clusapi RPC interfaces cannot be
// shutdown, it is not safe to delete this critical section.
//
// DeleteCriticalSection(&NmpLock);
NmpState = NmStateOffline;
ClRtlLogPrint(LOG_NOISE,"[NM] Shutdown complete.\n");
return;
} // NmShutdown
VOIDNmLeaveCluster( VOID )/*++
Routine Description:
Arguments:
Return Value:
--*/{ DWORD status;
if (NmLocalNode != NULL) { if ( (NmLocalNode->State == ClusterNodeUp) || (NmLocalNode->State == ClusterNodePaused) || (NmLocalNode->State == ClusterNodeJoining) ) { //
// Leave the cluster.
//
ClRtlLogPrint(LOG_NOISE,"[NM] Leaving cluster.\n");
MMLeave();
#ifdef MM_IN_CLUSNET
status = ClusnetLeaveCluster(NmClusnetHandle); CL_ASSERT(status == ERROR_SUCCESS);
#endif // MM_IN_CLUSNET
} }
NmpStopNetworkTimerThread();
NmpMembershipShutdown();
NmpCleanupInterfaces();
NmpCleanupNetworks();
NmpCleanupNodes();
//
// Shutdown the Cluster Network driver.
//
if (NmClusnetHandle != NULL) { status = ClusnetShutdown(NmClusnetHandle);
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Shutdown of the Cluster Network driver failed, status %1!u!\n", status ); } }
if (NmpClusnetEndpoint != NULL) { MIDL_user_free(NmpClusnetEndpoint); NmpClusnetEndpoint = NULL; }
return;
} // NmLeaveCluster
DWORDNmpCreateClusterObjects( IN RPC_BINDING_HANDLE JoinSponsorBinding )/*++
Routine Description:
Creates objects to represent the cluster's nodes, networks, and interfaces.
Arguments:
JoinSponsorBinding - A pointer to an RPC binding handle for the sponsor node if this node is joining a cluster. NULL if this node is forming a cluster.
Return Value:
ERROR_SUCCESS if the routine is successful. A Win32 error code otherwise.
Notes:
This routine MUST NOT be called with the NM lock held.
--*/{ DWORD status; PNM_NODE_ENUM2 nodeEnum = NULL; PNM_NETWORK_ENUM networkEnum = NULL; PNM_INTERFACE_ENUM2 interfaceEnum = NULL; PNM_NODE node = NULL; DWORD matchedNetworkCount = 0; DWORD newNetworkCount = 0; DWORD InitRetry = 2; WCHAR errorString[12]; DWORD eventCode = 0; BOOL renameConnectoids;
do { //
// Initialize the Cluster Network driver. This will clean up
// any old state that was left around from the last run of the
// Cluster Service. Note that the local node object is registered in
// this call.
//
status = ClusnetInitialize( NmClusnetHandle, NmLocalNodeId, NmMaxNodes, NULL, NULL, NULL, NULL, NULL, NULL );
if (status == ERROR_SUCCESS) { break; } else { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Shutting down Cluster Network driver before retrying Initialization, status %1!u!\n", status);
ClusnetShutdown( NmClusnetHandle ); } } while ( InitRetry-- );
if ( status != ERROR_SUCCESS ) { eventCode = NM_EVENT_CLUSNET_INITIALIZE_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Initialization of the Cluster Network driver failed, status %1!u!\n", status ); goto error_exit; }
//
// Tell the Cluster Network driver to reserve the Cluster Network
// endpoint on this node.
//
status = ClusnetReserveEndpoint( NmClusnetHandle, NmpClusnetEndpoint );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to reserve Clusnet Network endpoint %1!ws!, " "status %2!u!\n", NmpClusnetEndpoint, status ); wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, NM_EVENT_CLUSNET_RESERVE_ENDPOINT_FAILED, NmpClusnetEndpoint, errorString ); goto error_exit; }
//
// Obtain the node portion of the cluster database.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Synchronizing node information.\n" );
if (JoinSponsorBinding == NULL) { status = NmpEnumNodeDefinitions(&nodeEnum); } else { status = NmRpcEnumNodeDefinitions2( JoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &nodeEnum ); }
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CONFIG_SYNCH_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to synchronize node information, status %1!u!.\n", status ); goto error_exit; }
//
// Create the node objects.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Creating node objects.\n" );
status = NmpCreateNodeObjects(nodeEnum);
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Obtain the networks portion of the cluster database.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Synchronizing network information.\n" );
if (JoinSponsorBinding == NULL) { status = NmpEnumNetworkDefinitions(&networkEnum); } else { status = NmRpcEnumNetworkDefinitions( JoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &networkEnum ); }
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CONFIG_SYNCH_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to synchronize network information, status %1!u!.\n", status ); goto error_exit; }
//
// Obtain the interfaces portion of the cluster database.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Synchronizing interface information.\n" );
if (JoinSponsorBinding == NULL) { status = NmpEnumInterfaceDefinitions(&interfaceEnum); } else { status = NmRpcEnumInterfaceDefinitions2( JoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &interfaceEnum ); }
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CONFIG_SYNCH_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to synchronize interface information, status %1!u!.\n", status ); goto error_exit; }
if ( CsUpgrade ) { //
// If this is an upgrade from NT4 to Whistler, then fix up the
// connectoid names so they align with the cluster network
// names.
//
// REMOVE THIS PORTION AFTER WHISTLER HAS SHIPPED.
//
if ( CLUSTER_GET_MAJOR_VERSION( NmLocalNode->HighestVersion ) <= NT4SP4_MAJOR_VERSION ) { renameConnectoids = TRUE; } else { //
// upgrade from W2K to Whistler. Nothing should have changed but
// if it did, connectoids should have precedence
//
renameConnectoids = FALSE; } } else { //
// THIS SECTION MUST ALWAYS BE HERE
//
// if forming, cluster network objects are renamed to its
// corresponding connectoid name. During a join, the opposite is true.
//
if ( JoinSponsorBinding ) { renameConnectoids = TRUE; } else { renameConnectoids = FALSE; } }
//
// Post a PnP notification ioctl. If we receive a PnP notification
// before we finish initializing, we must restart the process.
//
NmpWatchForPnpEvents();
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Run the network configuration engine. This will update the
// cluster database.
//
status = NmpConfigureNetworks( JoinSponsorBinding, NmLocalNodeIdString, NmLocalNodeName, &networkEnum, &interfaceEnum, NmpClusnetEndpoint, &matchedNetworkCount, &newNetworkCount, renameConnectoids );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to configure networks & interfaces, status %1!u!.\n", status ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE, "[NM] Matched %1!u! networks, created %2!u! new networks.\n", matchedNetworkCount, newNetworkCount );
//
// Get the updated network information from the database.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Resynchronizing network information.\n" );
if (JoinSponsorBinding == NULL) { status = NmpEnumNetworkDefinitions(&networkEnum); } else { status = NmRpcEnumNetworkDefinitions( JoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &networkEnum ); }
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CONFIG_SYNCH_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to resynchronize network information, " "status %1!u!.\n", status ); goto error_exit; }
//
// Get the updated interface information from the database.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Resynchronizing interface information.\n" );
if (JoinSponsorBinding == NULL) { status = NmpEnumInterfaceDefinitions(&interfaceEnum); } else { status = NmRpcEnumInterfaceDefinitions2( JoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &interfaceEnum ); }
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_CONFIG_SYNCH_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NM] Unable to resynchronize interface information, status %1!u!.\n", status ); goto error_exit; }
//
// Create the network objects.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Creating network objects.\n" );
status = NmpCreateNetworkObjects(networkEnum);
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create network objects, status %1!u!.\n", status ); goto error_exit; }
//
// Fixup the priorities of the internal networks if we are forming
// a cluster.
//
if (JoinSponsorBinding == NULL) { DWORD networkCount; PNM_NETWORK * networkList;
status = NmpEnumInternalNetworks(&networkCount, &networkList);
if ((status == ERROR_SUCCESS) && (networkCount > 0)) { DWORD i; HLOCALXSACTION xaction;
//
// Begin a transaction - this must not be done while holding
// the NM lock.
//
xaction = DmBeginLocalUpdate();
if (xaction == NULL) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to start a transaction, status %1!u!\n", status ); goto error_exit; }
status = NmpSetNetworkPriorityOrder( networkCount, networkList, xaction );
if (status == ERROR_SUCCESS) { DmCommitLocalUpdate(xaction); } else { DmAbortLocalUpdate(xaction); goto error_exit; }
for (i=0; i<networkCount; i++) { if (networkList[i] != NULL) { OmDereferenceObject(networkList[i]); } }
LocalFree(networkList); } }
//
// Create the interface objects.
//
ClRtlLogPrint(LOG_NOISE, "[NM] Creating interface objects.\n" );
status = NmpCreateInterfaceObjects(interfaceEnum);
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create interface objects, status %1!u!.\n", status ); goto error_exit; }
if (JoinSponsorBinding != NULL) { //
// The node must have connectivity to all active cluster nodes
// in order to join a cluster.
//
PNM_NODE unreachableNode;
if (!NmpVerifyJoinerConnectivity(NmLocalNode, &unreachableNode)) { status = ERROR_CLUSTER_NETWORK_NOT_FOUND; CsLogEvent1( LOG_CRITICAL, NM_EVENT_NODE_UNREACHABLE, OmObjectName(unreachableNode) ); ClRtlLogPrint(LOG_CRITICAL, "[NM] Joining node cannot communicate with all other " "active nodes.\n" ); goto error_exit; } }
status = NmpMembershipInit();
if (status != ERROR_SUCCESS) { goto error_exit; }
status = NmpStartNetworkTimerThread();
if (status != ERROR_SUCCESS) { goto error_exit; }
error_exit:
if (eventCode != 0) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, eventCode, errorString); }
if (nodeEnum != NULL) { ClNetFreeNodeEnum(nodeEnum); }
if (networkEnum != NULL) { ClNetFreeNetworkEnum(networkEnum); }
if (interfaceEnum != NULL) { ClNetFreeInterfaceEnum(interfaceEnum); }
return(status);
} // NmpCreateClusterObjects
//
// Routines common to joining and forming.
//
DWORDNmpCreateClusterInstanceId( VOID )/*++
Routine Description:
Checks the cluster database for the cluster instance id. Creates if not present.
--*/{ DWORD status; LPWSTR clusterInstanceId = NULL; DWORD clusterInstanceIdBufSize = 0; DWORD clusterInstanceIdSize = 0; BOOLEAN found = FALSE; UUID guid;
do {
status = NmpQueryString( DmClusterParametersKey, L"ClusterInstanceID", REG_SZ, &clusterInstanceId, &clusterInstanceIdBufSize, &clusterInstanceIdSize );
if (status == ERROR_SUCCESS) { found = TRUE; } else {
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Cluster Instance ID not found in " "cluster database, status %1!u!.\n", status );
status = UuidCreate(&guid); if (status == RPC_S_OK) {
status = UuidToString(&guid, &clusterInstanceId); if (status == RPC_S_OK) {
status = DmSetValue( DmClusterParametersKey, L"ClusterInstanceID", REG_SZ, (PBYTE) clusterInstanceId, NM_WCSLEN(clusterInstanceId) ); if (status != ERROR_SUCCESS) {
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to store Cluster Instance ID " "in cluster database, status %1!u!.\n", status ); }
if (clusterInstanceId != NULL) { RpcStringFree(&clusterInstanceId); clusterInstanceId = NULL; } } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to convert Cluster Instance ID " "GUID into string, status %1!u!.\n", status ); }
} else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to create Cluster Instance ID GUID, " "status %1!u!.\n", status ); } } } while ( !found && (status == ERROR_SUCCESS) );
if (status == ERROR_SUCCESS) {
CL_ASSERT(clusterInstanceId != NULL);
NmpAcquireLock();
if (NmpClusterInstanceId == NULL) { NmpClusterInstanceId = clusterInstanceId; clusterInstanceId = NULL; }
NmpReleaseLock(); }
if (clusterInstanceId != NULL) { midl_user_free(clusterInstanceId); clusterInstanceId = NULL; }
return(status);
} // NmpCreateClusterInstanceId
//
// Routines for forming a new cluster.
//
DWORDNmFormNewCluster( VOID ){ DWORD status; DWORD isPaused = FALSE; DWORD pausedDefault = FALSE; HDMKEY nodeKey; DWORD valueLength, valueSize; WCHAR errorString[12], errorString2[12]; DWORD eventCode = 0; PLIST_ENTRY entry; PNM_NETWORK network;
ClRtlLogPrint(LOG_NOISE, "[NM] Beginning cluster form process.\n" );
//
// Since this node is forming the cluster, it is the leader.
//
NmpLeaderNodeId = NmLocalNodeId;
//
// Read the clusnet endpoint override value from the registry, if it
// exists.
//
if (NmpClusnetEndpoint != NULL) { MIDL_user_free(NmpClusnetEndpoint); NmpClusnetEndpoint = NULL; }
valueLength = 0;
status = NmpQueryString( DmClusterParametersKey, L"ClusnetEndpoint", REG_SZ, &NmpClusnetEndpoint, &valueLength, &valueSize );
if (status == ERROR_SUCCESS) { USHORT endpoint;
//
// Validate the value
//
status = ClRtlTcpipStringToEndpoint( NmpClusnetEndpoint, &endpoint );
if (status != ERROR_SUCCESS) { CsLogEvent2( LOG_UNUSUAL, NM_EVENT_INVALID_CLUSNET_ENDPOINT, NmpClusnetEndpoint, CLUSNET_DEFAULT_ENDPOINT_STRING ); ClRtlLogPrint( LOG_CRITICAL, "[NM] '%1!ws!' is not valid endpoint value. Using default value %2!ws!.\n", NmpClusnetEndpoint, CLUSNET_DEFAULT_ENDPOINT_STRING ); MIDL_user_free(NmpClusnetEndpoint); NmpClusnetEndpoint = NULL; } }
if (status != ERROR_SUCCESS) { NmpClusnetEndpoint = MIDL_user_allocate( NM_WCSLEN(CLUSNET_DEFAULT_ENDPOINT_STRING) );
if (NmpClusnetEndpoint == NULL) { status = ERROR_NOT_ENOUGH_MEMORY; wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, CS_EVENT_ALLOCATION_FAILURE, errorString); return(status); }
lstrcpyW(NmpClusnetEndpoint, CLUSNET_DEFAULT_ENDPOINT_STRING); }
//
// Create the node, network, and interface objects
//
status = NmpCreateClusterObjects(NULL);
if (status != ERROR_SUCCESS) { goto error_exit; }
//
// Perform version checking - check if we are compatible with the rest of the cluster
//
status = NmpIsNodeVersionAllowed(NmLocalNodeId, CsMyHighestVersion, CsMyLowestVersion, FALSE); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Version of Node %1!ws! is no longer compatible with other members of the cluster.\n", NmLocalNodeIdString); goto error_exit;
}
//If the forming node's version has changed, fix it up
status = NmpValidateNodeVersion( NmLocalNodeIdString, CsMyHighestVersion, CsMyLowestVersion );
if (status == ERROR_REVISION_MISMATCH) { //there was a version change, try and fix it up
status = NmpFormFixupNodeVersion( NmLocalNodeIdString, CsMyHighestVersion, CsMyLowestVersion ); NmLocalNodeVersionChanged = TRUE; } if (status != ERROR_SUCCESS) { goto error_exit; }
//
//at this point we ready to calculate the cluster version
//all the node versions are in the registry, the fixups have
//been made if neccessary
//
NmpResetClusterVersion(FALSE);
NmpMulticastInitialize();
ClRtlLogPrint(LOG_NOISE, "[NM] Forming cluster membership.\n" );
status = MMJoin( NmLocalNodeId, NM_CLOCK_PERIOD, NM_SEND_HB_RATE, NM_RECV_HB_RATE, NM_MM_JOIN_TIMEOUT );
if (status != MM_OK) { status = MMMapStatusToDosError(status); eventCode = NM_EVENT_MM_FORM_FAILED; ClRtlLogPrint( LOG_CRITICAL, "[NM] Membership form failed, status %1!u!. Unable to form a cluster.\n", status ); goto error_exit; }
#ifdef MM_IN_CLUSNET
status = ClusnetFormCluster( NmClusnetHandle, NM_CLOCK_PERIOD, NM_SEND_HB_RATE, NM_RECV_HB_RATE );
if (status != ERROR_SUCCESS) { ClRtlLogPrint( LOG_CRITICAL, "[NM] Failed to form a cluster, status %1!u!.\n", status ); goto error_exit; }
#endif // MM_IN_CLUSNET
//
// Check to see if we should come up in the paused state.
//
nodeKey = DmOpenKey( DmNodesKey, NmLocalNodeIdString, KEY_READ );
if (nodeKey != NULL) { status = DmQueryDword( nodeKey, CLUSREG_NAME_NODE_PAUSED, &isPaused, &pausedDefault );
if (status != ERROR_SUCCESS) { ClRtlLogPrint( LOG_UNUSUAL, "[NM] Unable to query Paused value for local node, status %1!u!.\n", status ); }
DmCloseKey(nodeKey); } else { ClRtlLogPrint( LOG_UNUSUAL, "[NM] Unable to open database key to local node, status %1!u!. Unable to determine Pause state.\n", status ); }
NmpAcquireLock();
if (isPaused) { NmLocalNode->State = ClusterNodePaused; } else { NmLocalNode->State = ClusterNodeUp; } NmLocalNode->ExtendedState = ClusterNodeJoining;
NmpState = NmStateOnline;
NmpReleaseLock();
//
// If the cluster instance ID does not exist, create it now. The cluster
// instance ID should be in the database unless this is the first uplevel
// node.
//
NmpCreateClusterInstanceId();
//
// Derive the cluster key.
//
status = NmpRederiveClusterKey(); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to derive cluster key, status %1!u!. " "Aborting ...\n", status ); goto error_exit; }
//
// Enable communication for the local node.
//
status = ClusnetOnlineNodeComm(NmClusnetHandle, NmLocalNodeId);
if (status != ERROR_SUCCESS) {
wsprintfW(&(errorString[0]), L"%u", NmLocalNodeId); wsprintfW(&(errorString2[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, NM_EVENT_CLUSNET_ONLINE_COMM_FAILED, errorString, errorString2 );
ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to enable communication for local node, status %1!u!\n", status ); goto error_exit; }
GumReceiveUpdates(FALSE, GumUpdateMembership, NmpGumUpdateHandler, NULL, sizeof(NmGumDispatchTable)/sizeof(GUM_DISPATCH_ENTRY), NmGumDispatchTable, NULL );
// DavidDio 8/16/2001
// Bug 456951: There is a race condition between the network and
// interface state update and setting the NM state to online. If the
// state is online but no GUM handler is registered, the update will
// not be invoked. Hence, use the following flag to indicate that
// a handler is ready.
NmpGumUpdateHandlerRegistered = TRUE;
//
// Enable network PnP event handling.
//
// If a PnP event occured during the form process, an error code will
// be returned, which will abort startup of the service.
//
status = NmpEnablePnpEvents();
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] A network PnP event occurred during form - abort.\n"); goto error_exit; }
//
// Check if we formed without any viable networks. The form is still
// allowed, but we record an entry in the system event log.
//
if (!NmpCheckForNetwork()) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Formed cluster with no viable networks.\n" ); CsLogEvent(LOG_UNUSUAL, NM_EVENT_FORM_WITH_NO_NETWORKS); }
//
// Force a reconfiguration of multicast parameters and plumb
// the results in clusnet.
//
NmpAcquireLock();
status = NmpStartMulticast(NULL, NmStartMulticastForm); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Failed to start multicast " "on cluster networks, status %1!u!.\n", status ); //
// Not a de facto fatal error.
//
status = ERROR_SUCCESS; }
NmpReleaseLock();
error_exit:
if (eventCode != 0) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, eventCode, errorString); }
return(status);
} // NmFormNewCluster
//
//
// Client-side routines for joining a cluster.
//
//
DWORDNmJoinCluster( IN RPC_BINDING_HANDLE SponsorBinding ){ DWORD status; DWORD sponsorNodeId; PNM_INTERFACE netInterface; PNM_NETWORK network; PNM_NODE node; PLIST_ENTRY nodeEntry, ifEntry; WCHAR errorString[12], errorString2[12]; DWORD eventCode = 0; DWORD versionFlags = 0; extern BOOLEAN bFormCluster; DWORD retry; BOOLEAN joinBegin3 = TRUE; LPWSTR clusterInstanceId = NULL;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Beginning cluster join process.\n" );
// GN: If a node tries to restart immediately after a clean shutdown,
// NmRpcJoinBegin2 can fail with ERROR_CLUSTER_NODE_UP. Since the regroup
// incident caused by this node might not be finished.
//
// If we are getting error CLUSTER_NODE_UP, we will keep retrying for
// 12 seconds, hoping that regroup will finish.
retry = 120 / 3; // We sleep for 3 seconds. Need to wait 2 minutes //
for (;;) { //
// Get the join sequence number so we can tell if the cluster
// configuration changes during the join process. We overload the
// use of the NmpJoinSequence variable since it isn't used in the
// sponsor capacity until the node joins.
//
//
// Try NmRpcJoinBegin3. If it fails with an RPC procnum out of
// range error, the sponsor is a downlevel node. Revert to
// NmRpcJoinBegin2.
//
if (joinBegin3) {
// Only read the cluster instance ID from the registry on
// the first try.
if (clusterInstanceId == NULL) {
DWORD clusterInstanceIdBufSize = 0; DWORD clusterInstanceIdSize = 0;
status = NmpQueryString( DmClusterParametersKey, L"ClusterInstanceID", REG_SZ, &clusterInstanceId, &clusterInstanceIdBufSize, &clusterInstanceIdSize ); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to read cluster instance ID from database, status %1!u!.\n", status ); // Try to join with the downlevel interface. It is
// possible that this node was just upgraded and the
// last time it was in the cluster there was no
// cluster instance ID.
joinBegin3 = FALSE; continue; } }
status = NmRpcJoinBegin3( SponsorBinding, clusterInstanceId, NmLocalNodeIdString, NmLocalNodeName, CsMyHighestVersion, CsMyLowestVersion, 0, // joiner's major node version
0, // joiner's minor node version
L"", // joiner's CsdVersion
0, // joiner's product suite
&sponsorNodeId, &NmpJoinSequence, &NmpClusnetEndpoint ); if (status == RPC_S_PROCNUM_OUT_OF_RANGE) { // retry immediately with JoinBegin2
joinBegin3 = FALSE; continue; } } else {
status = NmRpcJoinBegin2( SponsorBinding, NmLocalNodeIdString, NmLocalNodeName, CsMyHighestVersion, CsMyLowestVersion, &sponsorNodeId, &NmpJoinSequence, &NmpClusnetEndpoint ); }
if ( ((status != ERROR_CLUSTER_NODE_UP && status != ERROR_CLUSTER_JOIN_IN_PROGRESS) ) || retry == 0 ) { break; } ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Unable to begin join, status %1!u!. Retrying ...\n", status ); CsServiceStatus.dwCheckPoint++; CsAnnounceServiceStatus(); Sleep(3000); --retry; }
// Free the cluster instance ID string, if necessary.
if (clusterInstanceId != NULL) { midl_user_free(clusterInstanceId); }
// [GORN Jan/7/2000]
// If we are here, then we have already successfully talked to the sponsor
// via JoinVersion interface.
//
// We shouldn't try to form the cluster if NmRpcJoinBegin2 fails.
// Otherwise we may steal the quorum on the move [452108]
//
// Past this point we will not try to form a cluster
//
bFormCluster = FALSE;
if (status != ERROR_SUCCESS) { eventCode = NM_EVENT_BEGIN_JOIN_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Unable to begin join, status %1!u!.\n", status ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Sponsor node ID = %1!u!. Join sequence number = %2!u!, endpoint = %3!ws!.\n", sponsorNodeId, NmpJoinSequence, NmpClusnetEndpoint );
//
// Create all of the cluster objects for which we are responsible.
//
status = NmpCreateClusterObjects(SponsorBinding);
if (status != ERROR_SUCCESS) { goto error_exit; }
// The local node version might have changed, fix it
// The sponsorer fixes it in the registry and tells other
// nodes about it, however the joining node is not a part
// of the cluster membership as yet.
// The local node structure is created early on in NmInitialize()
// hence it must get fixed up
if ((NmLocalNode->HighestVersion != CsMyHighestVersion) || (NmLocalNode->LowestVersion != CsMyLowestVersion)) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Local Node version changed probably due to upgrade/deinstall\n"); NmLocalNode->HighestVersion = CsMyHighestVersion; NmLocalNode->LowestVersion = CsMyLowestVersion; NmLocalNodeVersionChanged = TRUE; }
//at this point we ready to calculate the cluster version
//all the node objects contain the correct node versions
NmpResetClusterVersion(FALSE);
NmpMulticastInitialize();
//
// Enable communication for the local node.
//
status = ClusnetOnlineNodeComm(NmClusnetHandle, NmLocalNodeId);
if (status != ERROR_SUCCESS) { wsprintfW(&(errorString[0]), L"%u", NmLocalNodeId); wsprintfW(&(errorString2[0]), L"%u", status); CsLogEvent2( LOG_CRITICAL, NM_EVENT_CLUSNET_ONLINE_COMM_FAILED, errorString, errorString2 );
ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Unable to enable communication for local node, status %1!u!.\n", status ); goto error_exit; }
//
// Fire up the intracluster RPC server so we can perform the membership
// join.
//
status = ClusterRegisterIntraclusterRpcInterface();
if ( status != ERROR_SUCCESS ) { eventCode = CS_EVENT_RPC_INIT_FAILED; ClRtlLogPrint(LOG_CRITICAL, "ClusSvc: Error starting intracluster RPC server, Status = %1!u!\n", status); goto error_exit; }
//
// Cycle through the list of cluster nodes and create mutual RPC bindings
// for the intracluster interface with each.
//
for (nodeEntry = NmpNodeList.Flink; nodeEntry != &NmpNodeList; nodeEntry = nodeEntry->Flink ) { node = CONTAINING_RECORD(nodeEntry, NM_NODE, Linkage);
if ( (node != NmLocalNode) && ( (node->State == ClusterNodeUp) || (node->State == ClusterNodePaused) ) ) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Creating RPC bindings for member node %1!u!\n", node->NodeId );
//
//
// Cycle through the target node's interfaces
//
for (ifEntry = node->InterfaceList.Flink; ifEntry != &(node->InterfaceList); ifEntry = ifEntry->Flink ) { netInterface = CONTAINING_RECORD( ifEntry, NM_INTERFACE, NodeLinkage );
network = netInterface->Network;
if (NmpIsNetworkForInternalUse(network)) { if ( (network->LocalInterface != NULL) && NmpIsInterfaceRegistered(network->LocalInterface) && NmpIsInterfaceRegistered(netInterface) ) { PNM_INTERFACE localInterface = network->LocalInterface;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Attempting to use network %1!ws! to " "create bindings for node %2!u!\n", OmObjectName(network), node->NodeId );
status = NmpSetNodeInterfacePriority( node, 0xFFFFFFFF, netInterface, 1 );
if (status == ERROR_SUCCESS) {
status = NmRpcCreateBinding( SponsorBinding, NmpJoinSequence, NmLocalNodeIdString, (LPWSTR) OmObjectId(localInterface), (LPWSTR) OmObjectId(node) );
if (status == ERROR_SUCCESS) { //
// Create RPC bindings for the target node.
//
status = NmpCreateRpcBindings(node);
if (status == ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Created binding for node " "%1!u!\n", node->NodeId ); break; }
wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent3( LOG_UNUSUAL, NM_EVENT_JOIN_BIND_OUT_FAILED, OmObjectName(node), OmObjectName(network), errorString ); ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Unable to create binding for " "node %1!u!, status %2!u!.\n", node->NodeId, status ); } else { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent3( LOG_UNUSUAL, NM_EVENT_JOIN_BIND_IN_FAILED, OmObjectName(node), OmObjectName(network), errorString ); ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Member node %1!u! failed to " "create binding to us, status %2!u!\n", node->NodeId, status ); } } else { wsprintfW(&(errorString[0]), L"%u", node->NodeId); wsprintfW(&(errorString2[0]), L"%u", status); CsLogEvent2( LOG_UNUSUAL, NM_EVENT_CLUSNET_SET_INTERFACE_PRIO_FAILED, errorString, errorString2 ); ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to set interface priorities " "for node %1!u!, status %2!u!\n", node->NodeId, status ); } } else { status = ERROR_CLUSTER_NODE_UNREACHABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] No matching local interface for " "network %1!ws!\n", OmObjectName(netInterface->Network) ); } } else { status = ERROR_CLUSTER_NODE_UNREACHABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Network %1!ws! is not used for internal " "communication.\n", OmObjectName(netInterface->Network) ); } }
if (status != ERROR_SUCCESS) { //
// Cannot make contact with this node. The join fails.
//
CsLogEvent1( LOG_CRITICAL, NM_EVENT_NODE_UNREACHABLE, OmObjectName(node) ); ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Cluster node %1!u! is not reachable. Join " "failed.\n", node->NodeId ); goto error_exit; } } }
CL_ASSERT(status == ERROR_SUCCESS);
//
// run through the active nodes again, this time establishing
// security contexts to use in signing packets
//
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Establishing security contexts with all active nodes.\n" );
for (nodeEntry = NmpNodeList.Flink; nodeEntry != &NmpNodeList; nodeEntry = nodeEntry->Flink ) { node = CONTAINING_RECORD(nodeEntry, NM_NODE, Linkage);
status = ClMsgCreateActiveNodeSecurityContext(NmpJoinSequence, node);
if ( status != ERROR_SUCCESS ) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent2( LOG_UNUSUAL, NM_EVENT_CREATE_SECURITY_CONTEXT_FAILED, OmObjectName(node), errorString ); ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Unable to establish security context for node %1!u!, status 0x%2!08X!\n", node->NodeId, status ); goto error_exit; } }
//
// Finally, petition the sponsor for membership
//
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Petitioning to join cluster membership.\n" );
#ifdef CLUSTER_TESTPOINT
TESTPT(TpFailJoinPetitionForMembership) { status = 999999; goto error_exit; }#endif
status = NmRpcPetitionForMembership( SponsorBinding, NmpJoinSequence, NmLocalNodeIdString );
if (status != ERROR_SUCCESS) { //
// Our petition was denied.
//
eventCode = NM_EVENT_PETITION_FAILED; ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Petition to join was denied %1!d!\n", status ); goto error_exit; }
#ifdef CLUSTER_TESTPOINT
TESTPT(TpFailNmJoin) { status = 999999; goto error_exit; }#endif
//
// Reset the interface priorities for all nodes to default to
// the priorities of the associated networks.
//
NmpAcquireLock();
for (ifEntry = NmpInterfaceList.Flink; ifEntry != &NmpInterfaceList; ifEntry = ifEntry->Flink ) { netInterface = CONTAINING_RECORD(ifEntry, NM_INTERFACE, Linkage); network = netInterface->Network;
if ( NmpIsNetworkForInternalUse(network) && NmpIsInterfaceRegistered(netInterface) ) { status = ClusnetSetInterfacePriority( NmClusnetHandle, netInterface->Node->NodeId, netInterface->Network->ShortId, 0 );
CL_ASSERT(status == ERROR_SUCCESS); } }
NmpState = NmStateOnline;
NmpReleaseLock();
//
// Invoke other components to create RPC bindings for each node.
//
//
// Enable our GUM update handler.
//
GumReceiveUpdates( TRUE, GumUpdateMembership, NmpGumUpdateHandler, NULL, sizeof(NmGumDispatchTable)/sizeof(GUM_DISPATCH_ENTRY), NmGumDispatchTable, NULL );
// DavidDio 8/16/2001
// Bug 456951: There is a race condition in the form path between
// the network and interface state update and setting the NM state
// to online. If the state is online but no GUM handler is registered,
// the update will not be invoked. Hence, use the following flag to
// indicate that a handler is ready.
NmpGumUpdateHandlerRegistered = TRUE;
return(ERROR_SUCCESS);
error_exit:
if (eventCode != 0) { wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent1(LOG_CRITICAL, eventCode, errorString); }
return(status);
} // NmJoinCluster
BOOLEANNmpVerifyJoinerConnectivity( IN PNM_NODE JoiningNode, OUT PNM_NODE * UnreachableNode ){ PLIST_ENTRY entry; PNM_NODE node;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Verifying connectivity to active cluster nodes\n" );
*UnreachableNode = NULL;
for (entry = NmpNodeList.Flink; entry != &NmpNodeList; entry = entry->Flink ) { node = CONTAINING_RECORD( entry, NM_NODE, Linkage );
if (NM_NODE_UP(node)) { if (!NmpVerifyNodeConnectivity(JoiningNode, node, NULL)) { *UnreachableNode = node; return(FALSE); } } }
return(TRUE);
} // NmpVerifyJoinerConnectivity
DWORDNmGetJoinSequence( VOID ){ DWORD sequence;
NmpAcquireLock();
sequence = NmpJoinSequence;
NmpReleaseLock();
return(sequence);
} // NmGetJoinSequence
DWORDNmJoinComplete( OUT DWORD *EndSeq )/*++
Routine Description:
This routine is called by the initialization sequence once a join has successfully completed and the node can transition from ClusterNodeJoining to ClusterNodeOnline.
Arguments:
None
Return Value:
ERROR_SUCCESS if successful
Win32 error otherwise.
--*/
{ DWORD Sequence; DWORD Status; PNM_JOIN_UPDATE JoinUpdate = NULL; DWORD UpdateLength; HDMKEY NodeKey = NULL; DWORD Default = 0; DWORD NumRetries=50; DWORD eventCode = 0; WCHAR errorString[12]; PNM_NETWORK_STATE_ENUM networkStateEnum = NULL; PNM_NETWORK_STATE_INFO networkStateInfo; PNM_INTERFACE_STATE_ENUM interfaceStateEnum = NULL; PNM_INTERFACE_STATE_INFO interfaceStateInfo; DWORD i; PNM_NETWORK network; PNM_INTERFACE netInterface; PLIST_ENTRY entry; DWORD moveCount;
UpdateLength = sizeof(NM_JOIN_UPDATE) + (lstrlenW(OmObjectId(NmLocalNode))+1)*sizeof(WCHAR);
JoinUpdate = LocalAlloc(LMEM_FIXED, UpdateLength);
if (JoinUpdate == NULL) { Status = ERROR_NOT_ENOUGH_MEMORY; eventCode = CS_EVENT_ALLOCATION_FAILURE; ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Unable to allocate memory.\n"); goto error_exit; }
JoinUpdate->JoinSequence = NmpJoinSequence;
lstrcpyW(JoinUpdate->NodeId, OmObjectId(NmLocalNode));
NodeKey = DmOpenKey(DmNodesKey, OmObjectId(NmLocalNode), KEY_READ);
if (NodeKey == NULL) { Status = GetLastError(); wsprintfW(&(errorString[0]), L"%u", Status); CsLogEvent2( LOG_CRITICAL, CS_EVENT_REG_OPEN_FAILED, OmObjectId(NmLocalNode), errorString ); ClRtlLogPrint( LOG_CRITICAL, "[NMJOIN] Unable to open database key to local node, status %1!u!.\n", Status ); goto error_exit; }
retry:
Status = GumBeginJoinUpdate(GumUpdateMembership, &Sequence);
if (Status != ERROR_SUCCESS) { eventCode = NM_EVENT_GENERAL_JOIN_ERROR; goto error_exit; }
//
// Get the leader node ID from the sponsor.
//
Status = NmRpcGetLeaderNodeId( CsJoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &NmpLeaderNodeId );
if (Status != ERROR_SUCCESS) { if (Status == ERROR_CALL_NOT_IMPLEMENTED) { //
// The sponsor is an NT4 node. Make this node the leader.
//
NmpLeaderNodeId = NmLocalNodeId; } else { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to get leader node ID from sponsor, status %1!u!.\n", Status ); goto error_exit; } }
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Node %1!u! is the leader.\n", NmpLeaderNodeId );
//
// Fetch the network and interface states from the sponsor
//
Status = NmRpcEnumNetworkAndInterfaceStates( CsJoinSponsorBinding, NmpJoinSequence, NmLocalNodeIdString, &networkStateEnum, &interfaceStateEnum );
if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to get network and interface state values from sponsor, status %1!u!.\n", Status ); goto error_exit; }
NmpAcquireLock();
for (i=0; i<networkStateEnum->NetworkCount; i++) { networkStateInfo = &(networkStateEnum->NetworkList[i]);
network = OmReferenceObjectById( ObjectTypeNetwork, networkStateInfo->Id );
if (network == NULL) { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Cannot find network %1!ws! to update state.\n", networkStateInfo->Id ); NmpReleaseLock(); NmpFreeNetworkStateEnum(networkStateEnum); LocalFree(JoinUpdate); DmCloseKey(NodeKey); return(ERROR_CLUSTER_NETWORK_NOT_FOUND); }
network->State = networkStateInfo->State;
OmDereferenceObject(network); }
for (i=0; i<interfaceStateEnum->InterfaceCount; i++) { interfaceStateInfo = &(interfaceStateEnum->InterfaceList[i]);
netInterface = OmReferenceObjectById( ObjectTypeNetInterface, interfaceStateInfo->Id );
if (netInterface == NULL) { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Cannot find interface %1!ws! to update state.\n", interfaceStateInfo->Id ); NmpReleaseLock(); NmpFreeInterfaceStateEnum(interfaceStateEnum); LocalFree(JoinUpdate); DmCloseKey(NodeKey); return(ERROR_CLUSTER_NETINTERFACE_NOT_FOUND); }
netInterface->State = interfaceStateInfo->State;
OmDereferenceObject(netInterface); }
NmpReleaseLock();
NmpFreeInterfaceStateEnum(interfaceStateEnum); interfaceStateEnum = NULL;
//
// Check the registry to see if we should come up paused.
//
JoinUpdate->IsPaused = Default;
Status = DmQueryDword(NodeKey, CLUSREG_NAME_NODE_PAUSED, &JoinUpdate->IsPaused, &Default);
if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Unable to query Paused value for local node, status %1!u!.\n", Status ); }
Status = GumEndJoinUpdate(Sequence, GumUpdateMembership, NmUpdateJoinComplete, UpdateLength, JoinUpdate);
if (Status != ERROR_SUCCESS) { if (Status == ERROR_CLUSTER_JOIN_ABORTED) { //
// The join was aborted by the cluster members. Don't retry.
//
CsLogEvent(LOG_CRITICAL, NM_EVENT_JOIN_ABORTED); goto error_exit; }
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] GumEndJoinUpdate with sequence %1!d! failed %2!d!\n", Sequence, Status );
if (--NumRetries == 0) { CsLogEvent(LOG_CRITICAL, NM_EVENT_JOIN_ABANDONED); ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Tried to complete join too many times. Giving up.\n" ); goto error_exit; }
goto retry; }
//
// If the cluster instance ID does not exist, create it now. The cluster
// instance ID should be in the database unless this is the first uplevel
// node.
//
NmpCreateClusterInstanceId();
//
// Derive the cluster key.
//
Status = NmpRederiveClusterKey(); if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Failed to derive cluster key, status %1!u!. " "Aborting ...\n", Status ); goto error_exit; }
NmpAcquireLock();
if (JoinUpdate->IsPaused != 0) { //
// We should be coming up paused.
//
NmLocalNode->State = ClusterNodePaused; } else { //
// Set our state to online.
//
NmLocalNode->State = ClusterNodeUp; }
//
// Start multicast for all networks.
//
Status = NmpStartMulticast(NULL, NmStartMulticastJoin); if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NM] Failed to refresh multicast configuration " "for cluster networks, status %1!u!.\n", Status ); //
// Not a fatal error.
//
Status = ERROR_SUCCESS; }
NmpReleaseLock();
//
// Finally, enable network PnP event handling.
//
// If a PnP event occured during the join process, an error code will
// be returned, which will abort startup of the service.
//
Status = NmpEnablePnpEvents();
if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] A network PnP event occurred during join - abort.\n"); goto error_exit; }
//
// Mark end sequence
*EndSeq = Sequence;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Join complete, node now online\n");
error_exit:
if (JoinUpdate != NULL) { LocalFree(JoinUpdate); }
if (NodeKey != NULL) { DmCloseKey(NodeKey); }
if (eventCode != 0) { wsprintfW(&(errorString[0]), L"%u", Status); CsLogEvent1(LOG_CRITICAL, eventCode, errorString); }
return(Status);
} // NmJoinComplete
//
// Server-side routines for sponsoring a joining node.
//
/*
Notes On Joining:
Only a single node may join the cluster at any time. A join begins with a JoinBegin global update. A join completes successfully with a JoinComplete global update. A join is aborted with a JoinAbort global update.
A timer runs on the sponsor during a join. The timer is suspended while the sponsor is performing work on behalf of the joiner. If the timer expires, a worker thread is scheduled to initiate the abort process.
If the sponsor goes down while a join is in progress, the node down handling code on each remaining node will abort the join.
*/
error_status_ts_NmRpcJoinBegin( IN handle_t IDL_handle, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, OUT LPDWORD SponsorNodeId, OUT LPDWORD JoinSequenceNumber, OUT LPWSTR * ClusnetEndpoint )/*++
Routine Description:
Called by a joining node to begin the join process. Issues a JoinBegin global update.
--*/{
DWORD status=ERROR_CLUSTER_INCOMPATIBLE_VERSIONS;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Request by node %1!ws! to begin joining, refused. Using obsolete join interface\n", JoinerNodeId );
if ( status != ERROR_SUCCESS ) { WCHAR errorCode[16];
wsprintfW( errorCode, L"%u", status );
CsLogEvent2( LOG_CRITICAL, NM_EVENT_JOIN_REFUSED, JoinerNodeId, errorCode ); }
return(status);
} // s_NmRpcJoinBegin
//
// Server-side routines for sponsoring a joining node.
//
/*
Notes On Joining:
*///#pragma optimize("", off)
DWORDNmpJoinBegin( IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, IN DWORD JoinerHighestVersion, IN DWORD JoinerLowestVersion, OUT LPDWORD SponsorNodeId, OUT LPDWORD JoinSequenceNumber, OUT LPWSTR * ClusnetEndpoint )/*++
Routine Description:
Called from s_NmRpcJoinBegin2 and s_NmRpcJoinBegin3. Contains functionality common to both JoinBegin versions.
Notes:
Called with NM lock held and NmpLockedEnterApi already called.
--*/{ DWORD status = ERROR_SUCCESS; PNM_NODE joinerNode = NULL; LPWSTR endpoint = NULL;
joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode == NULL) { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! is not a member of this cluster. Cannot join.\n", JoinerNodeId ); goto FnExit; }
endpoint = MIDL_user_allocate(NM_WCSLEN(NmpClusnetEndpoint));
if (endpoint == NULL) { status = ERROR_NOT_ENOUGH_MEMORY; goto FnExit; }
lstrcpyW(endpoint, NmpClusnetEndpoint);
if (NmpJoinBeginInProgress || NmpJoinerNodeId != ClusterInvalidNodeId) { status = ERROR_CLUSTER_JOIN_IN_PROGRESS; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! cannot join because a join is already in progress (%2!d!).\n", JoinerNodeId, NmpJoinerNodeId ); goto FnExit; }
//
//validate the nodes version's number
//ie. check to see what the cluster database
//claims this node's version is vs what the node
//itself suggests
status = NmpValidateNodeVersion( JoinerNodeId, JoinerHighestVersion, JoinerLowestVersion );
//since this node joined, its version has changed
//this may happen due to upgrades or reinstall
//if this version cant join due to versioning,fail the join
if (status == ERROR_REVISION_MISMATCH) { DWORD id = NmGetNodeId(joinerNode);
status = NmpIsNodeVersionAllowed( id, JoinerHighestVersion, JoinerLowestVersion, TRUE );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] The version of the cluster prevents Node %1!ws! from joining the cluster\n", JoinerNodeId ); goto FnExit; } } else if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] The version of Node %1!ws! cannot be validated.\n", JoinerNodeId ); goto FnExit; }
//
// Lock out other join attempts with this sponsor.
//
NmpJoinBeginInProgress = TRUE; NmpSuccessfulMMJoin = FALSE;
NmpReleaseLock();
status = GumSendUpdateEx( GumUpdateMembership, NmUpdateJoinBegin2, 5, NM_WCSLEN(JoinerNodeId), JoinerNodeId, NM_WCSLEN(JoinerNodeName), JoinerNodeName, NM_WCSLEN(NmLocalNodeIdString), NmLocalNodeIdString, sizeof(DWORD), &JoinerHighestVersion, sizeof(DWORD), &JoinerLowestVersion );
NmpAcquireLock();
CL_ASSERT(NmpJoinBeginInProgress == TRUE); NmpJoinBeginInProgress = FALSE;
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] JoinBegin2 update for node %1!ws! failed, status %2!u!\n", JoinerNodeId, status ); goto FnExit; } //
// Verify that the join is still in progress with
// this node as the sponsor.
//
if ( (NmpJoinerNodeId == joinerNode->NodeId) && (NmpSponsorNodeId == NmLocalNodeId) ) { //
// Give the joiner parameters for future
// join-related calls.
//
*SponsorNodeId = NmLocalNodeId; *JoinSequenceNumber = NmpJoinSequence;
//
// Start the join timer
//
NmpJoinTimer = NM_JOIN_TIMEOUT;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Node %1!ws! has begun the join process.\n", JoinerNodeId ); } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Begin join of node %1!ws! was aborted\n", JoinerNodeId ); }
FnExit: if (joinerNode) { OmDereferenceObject(joinerNode); }
if (status == ERROR_SUCCESS) { *ClusnetEndpoint = endpoint; } else { WCHAR errorCode[16];
if (endpoint) MIDL_user_free(endpoint);
wsprintfW( errorCode, L"%u", status );
CsLogEvent2( LOG_CRITICAL, NM_EVENT_JOIN_REFUSED, JoinerNodeId, errorCode ); }
return(status);
} // NmpJoinBegin
error_status_ts_NmRpcJoinBegin2( IN handle_t IDL_handle, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, IN DWORD JoinerHighestVersion, IN DWORD JoinerLowestVersion, OUT LPDWORD SponsorNodeId, OUT LPDWORD JoinSequenceNumber, OUT LPWSTR * ClusnetEndpoint )/*++
Routine Description:
Called by a joining node to begin the join process. Issues a JoinBegin global update.
--*/{ DWORD status = ERROR_SUCCESS;
status = FmDoesQuorumAllowJoin( JoinerNodeId ); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Quorum Characteristics prevent the node %1!ws! to from joining, Status=%2!u!.\n", JoinerNodeId, status ); return(status);
}
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Processing request by node %1!ws! to begin joining (2).\n", JoinerNodeId );
if (!NmpLockedEnterApi(NmStateOnline)) { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Cannot sponsor a joining node at this time.\n" ); NmpReleaseLock(); return(status); }
status = NmpJoinBegin( JoinerNodeId, JoinerNodeName, JoinerHighestVersion, JoinerLowestVersion, SponsorNodeId, JoinSequenceNumber, ClusnetEndpoint );
NmpLockedLeaveApi();
NmpReleaseLock();
return(status);
} // s_NmRpcJoinBegin2
error_status_ts_NmRpcJoinBegin3( IN handle_t IDL_handle, IN LPWSTR JoinerClusterInstanceId, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, IN DWORD JoinerHighestVersion, IN DWORD JoinerLowestVersion, IN DWORD JoinerMajorVersion, IN DWORD JoinerMinorVersion, IN LPWSTR JoinerCsdVersion, IN DWORD JoinerProductSuite, OUT LPDWORD SponsorNodeId, OUT LPDWORD JoinSequenceNumber, OUT LPWSTR * ClusnetEndpoint ){ DWORD status = ERROR_SUCCESS;
LPWSTR clusterInstanceId = NULL; DWORD clusterInstanceIdBufSize = 0; DWORD clusterInstanceIdSize = 0;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Processing request by node %1!ws! to begin joining (3).\n", JoinerNodeId );
status = FmDoesQuorumAllowJoin( JoinerNodeId ); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Quorum Characteristics prevent node %1!ws! from joining, Status=%2!u!.\n", JoinerNodeId, status ); return(status);
}
//
// Check our cluster instance ID against the joiner's.
//
if (NmpClusterInstanceId == NULL || lstrcmpiW(NmpClusterInstanceId, JoinerClusterInstanceId) != 0) {
WCHAR errorCode[16];
status = ERROR_CLUSTER_INSTANCE_ID_MISMATCH;
ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Sponsor cluster instance ID %1!ws! does not match joiner cluster instance id %2!ws!.\n", ((NmpClusterInstanceId == NULL) ? L"<NULL>" : NmpClusterInstanceId), JoinerClusterInstanceId );
wsprintfW( errorCode, L"%u", status ); CsLogEvent2( LOG_CRITICAL, NM_EVENT_JOIN_REFUSED, JoinerNodeId, errorCode );
return(status);
} else {
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Sponsor cluster instance ID matches joiner cluster instance id (%1!ws!).\n", JoinerClusterInstanceId ); }
NmpAcquireLock();
if (!NmpLockedEnterApi(NmStateOnline)) {
status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Cannot sponsor a joining node at this time.\n" );
} else {
status = NmpJoinBegin( JoinerNodeId, JoinerNodeName, JoinerHighestVersion, JoinerLowestVersion, SponsorNodeId, JoinSequenceNumber, ClusnetEndpoint );
NmpLockedLeaveApi(); }
NmpReleaseLock();
return(status);
} // s_NmRpcJoinBegin3
DWORDNmpUpdateJoinBegin( IN BOOL SourceNode, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, IN LPWSTR SponsorNodeId ){ DWORD status=ERROR_CLUSTER_INCOMPATIBLE_VERSIONS;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Failing update to begin join of node %1!ws! with " "sponsor %2!ws!. Using obsolete join interface.\n", JoinerNodeId, SponsorNodeId );
return(status);
} // NmpUpdateJoinBegin
DWORDNmpUpdateJoinBegin2( IN BOOL SourceNode, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerNodeName, IN LPWSTR SponsorNodeId, IN LPDWORD JoinerHighestVersion, IN LPDWORD JoinerLowestVersion ){ DWORD status = ERROR_SUCCESS; PNM_NODE sponsorNode=NULL; PNM_NODE joinerNode=NULL; HLOCALXSACTION hXsaction=NULL; BOOLEAN lockAcquired = FALSE; BOOLEAN fakeSuccess = FALSE;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Received update to begin join (2) of node %1!ws! with " "sponsor %2!ws!.\n", JoinerNodeId, SponsorNodeId );
//
// If running with -noquorum flag or if not online, don't sponsor
// any node.
//
if (CsNoQuorum || !NmpEnterApi(NmStateOnline)) { ClRtlLogPrint(LOG_NOISE, "[NM] Not in valid state to begin a join operation.\n" ); return(ERROR_NODE_NOT_AVAILABLE); }
//
// Find the sponsor node
//
sponsorNode = OmReferenceObjectById( ObjectTypeNode, SponsorNodeId );
if (sponsorNode == NULL) { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] JoinBegin update for node %1!ws! failed because " "sponsor node %2!ws! is not a member of this cluster.\n", JoinerNodeId, SponsorNodeId ); goto FnExit; }
//
// Find the joiner node
//
joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode == NULL) { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! is not a member of this cluster. " "Cannot join.\n", JoinerNodeId ); goto FnExit; }
hXsaction = DmBeginLocalUpdate();
if (hXsaction == NULL) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to start a transaction, status %1!u!\n", status ); goto FnExit; }
NmpAcquireLock(); lockAcquired = TRUE;
if (!NM_NODE_UP(sponsorNode)) { //
// [GorN 4/3/2000] See bug#98287
// This hack is a kludgy solution to a problem that
// a replay of this Gum update after the sponsor death
// will fail on all the nodes that didn't see the update.
//
fakeSuccess = TRUE; status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Sponsor node %1!ws! is not up. Join of node %2!ws! " "failed.\n", SponsorNodeId, JoinerNodeId ); goto FnExit; }
//
// Check that the joiner is really who we think it is.
//
if (lstrcmpiW( OmObjectName(joinerNode), JoinerNodeName)) { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! is not a member of this cluster. " "Cannot join.\n", JoinerNodeName ); goto FnExit; }
//
// Make sure the joiner is currently down.
//
if (joinerNode->State != ClusterNodeDown) { status = ERROR_CLUSTER_NODE_UP; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! is not down. Cannot begin join.\n", JoinerNodeId ); goto FnExit; }
//
// Make sure we aren't already in a join.
//
if (NmpJoinerNodeId != ClusterInvalidNodeId) { status = ERROR_CLUSTER_JOIN_IN_PROGRESS; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! cannot begin join because a join is " "already in progress for node %2!u!.\n", JoinerNodeId, NmpJoinerNodeId ); goto FnExit; }
//
// Perform the version compatibility check.
//
status = NmpIsNodeVersionAllowed( NmGetNodeId(joinerNode), *JoinerHighestVersion, *JoinerLowestVersion, TRUE );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] The version of the cluster prevents Node %1!ws! " "from joining the cluster\n", JoinerNodeId ); goto FnExit; }
// Fix up the joiner's version number if needed.
//
status = NmpValidateNodeVersion( JoinerNodeId, *JoinerHighestVersion, *JoinerLowestVersion );
if (status == ERROR_REVISION_MISMATCH) { //
// At this point, the registry contains the new
// versions for the joining code.
// The new node information should be reread
// from the registry before resetting the cluster
// version
// make sure the joiner gets the database from the
// sponsor after the fixups have occured
//
status = NmpJoinFixupNodeVersion( hXsaction, JoinerNodeId, *JoinerHighestVersion, *JoinerLowestVersion );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node %1!ws! failed to fixup its node version\r\n", JoinerNodeId); goto FnExit; } } else if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] The verison of Node %1!ws! could not be validated\n", JoinerNodeId); goto FnExit; }
//
//at this point we ready to calculate the cluster version
//all the node versions are in the registry, the fixups have
//been made if neccessary
//
NmpResetClusterVersion(TRUE);
//
// Enable communication to the joiner.
//
// This must be the last test that can fail before the join is allowed
// to proceed.
//
status = ClusnetOnlineNodeComm(NmClusnetHandle, joinerNode->NodeId);
if (status != ERROR_SUCCESS) { if (status != ERROR_CLUSTER_NODE_ALREADY_UP) { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to enable communication for node %1!u!, " "status %2!u!\n", JoinerNodeId, status ); goto FnExit; } else { status = ERROR_SUCCESS; } }
//
// Officially begin the join process
//
CL_ASSERT(NmpJoinTimer == 0); CL_ASSERT(NmpJoinAbortPending == FALSE); CL_ASSERT(NmpJoinerUp == FALSE); CL_ASSERT(NmpSponsorNodeId == ClusterInvalidNodeId);
NmpJoinerNodeId = joinerNode->NodeId; NmpSponsorNodeId = sponsorNode->NodeId; NmpJoinerOutOfSynch = FALSE; NmpJoinSequence = GumGetCurrentSequence(GumUpdateMembership);
joinerNode->State = ClusterNodeJoining;
ClusterEvent( CLUSTER_EVENT_NODE_JOIN, joinerNode );
NmpCleanupIfJoinAborted = TRUE;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Node %1!ws! join sequence = %2!u!\n", JoinerNodeId, NmpJoinSequence );
CL_ASSERT(status == ERROR_SUCCESS);
FnExit:
if (lockAcquired) { NmpLockedLeaveApi(); NmpReleaseLock(); } else { NmpLeaveApi(); }
if (hXsaction != NULL) { if (status == ERROR_SUCCESS) { DmCommitLocalUpdate(hXsaction); } else { DmAbortLocalUpdate(hXsaction); } }
if (joinerNode != NULL) { OmDereferenceObject(joinerNode); }
if (sponsorNode != NULL) { OmDereferenceObject(sponsorNode); }
if (fakeSuccess) { status = ERROR_SUCCESS; } return(status);
} // NmpUpdateJoinBegin2
DWORDNmpCreateRpcBindings( IN PNM_NODE Node ){ DWORD status;
//
// Create the default binding for the whole cluster service
//
status = ClMsgCreateDefaultRpcBinding( Node, &Node->DefaultRpcBindingGeneration);
if (status != ERROR_SUCCESS) { return(status); }
//
// Create private bindings for the NM's use.
// We create one for reporting network connectivity and one for
// performing network failure isolation. The NM uses the
// default binding for operations on behalf of joining nodes.
//
if (Node->ReportRpcBinding != NULL) { //
// Reuse the old binding.
//
status = ClMsgVerifyRpcBinding(Node->ReportRpcBinding);
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to verify RPC binding for node %1!u!, " "status %2!u!.\n", Node->NodeId, status ); return(status); } } else { //
// Create a new binding
//
status = ClMsgCreateRpcBinding( Node, &(Node->ReportRpcBinding), 0 );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create RPC binding for node %1!u!, " "status %2!u!.\n", Node->NodeId, status ); return(status); } }
if (Node->IsolateRpcBinding != NULL) { //
// Reuse the old binding.
//
status = ClMsgVerifyRpcBinding(Node->IsolateRpcBinding);
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to verify RPC binding for node %1!u!, " "status %2!u!.\n", Node->NodeId, status ); return(status); } } else { //
// Create a new binding
//
status = ClMsgCreateRpcBinding( Node, &(Node->IsolateRpcBinding), 0 );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create RPC binding for node %1!u!, " "status %2!u!.\n", Node->NodeId, status ); return(status); } }
//
// Call other components to create their private bindings
//
status = GumCreateRpcBindings(Node);
if (status != ERROR_SUCCESS) { return(status); }
status = EvCreateRpcBindings(Node);
if (status != ERROR_SUCCESS) { return(status); }
status = FmCreateRpcBindings(Node);
if (status != ERROR_SUCCESS) { return(status); }
return(ERROR_SUCCESS);
} // NmpCreateRpcBindings
error_status_ts_NmRpcCreateBinding( IN handle_t IDL_handle, IN DWORD JoinSequence, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerInterfaceId, IN LPWSTR MemberNodeId ){ DWORD status;
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Processing CreateBinding request from joining node %1!ws! for member node %2!ws!\n", JoinerNodeId, MemberNodeId );
if (NmpLockedEnterApi(NmStateOnlinePending)) {
PNM_NODE joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode != NULL) { if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) && (NmpSponsorNodeId == NmLocalNodeId) && !NmpJoinAbortPending ) { PNM_NODE memberNode;
CL_ASSERT(joinerNode->State == ClusterNodeJoining); CL_ASSERT(NmpJoinerUp == FALSE); CL_ASSERT(NmpJoinTimer != 0);
//
// Suspend the join timer while we are working on
// behalf of the joiner. This precludes an abort
// from occuring as well.
//
NmpJoinTimer = 0;
memberNode = OmReferenceObjectById( ObjectTypeNode, MemberNodeId );
if (memberNode != NULL) { PNM_INTERFACE netInterface = OmReferenceObjectById( ObjectTypeNetInterface, JoinerInterfaceId );
if (netInterface != NULL) { if (memberNode == NmLocalNode) { status = NmpCreateJoinerRpcBindings( joinerNode, netInterface ); } else { if (NM_NODE_UP(memberNode)) { DWORD joinSequence = NmpJoinSequence; RPC_BINDING_HANDLE binding = Session[memberNode->NodeId];
CL_ASSERT(binding != NULL);
NmpReleaseLock();
NmStartRpc(memberNode->NodeId); status = NmRpcCreateJoinerBinding( binding, joinSequence, JoinerNodeId, JoinerInterfaceId ); NmEndRpc(memberNode->NodeId); if(status != RPC_S_OK) { NmDumpRpcExtErrorInfo(status); }
NmpAcquireLock();
} else { status = ERROR_CLUSTER_NODE_DOWN; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because member node %2!ws! is down.\n", JoinerNodeId, MemberNodeId ); } }
OmDereferenceObject(netInterface); } else { status = ERROR_CLUSTER_NETINTERFACE_NOT_FOUND; ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Can't create binding for joining node %1!ws! - interface %2!ws! doesn't exist.\n", JoinerNodeId, JoinerInterfaceId ); }
OmDereferenceObject(memberNode); } else { status = ERROR_CLUSTER_NODE_NOT_FOUND; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because member node %2!ws! does not exist\n", JoinerNodeId, MemberNodeId ); }
//
// Verify that the join is still in progress
//
if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) ) { CL_ASSERT(joinerNode->State == ClusterNodeJoining); CL_ASSERT(NmpJoinerUp == FALSE); CL_ASSERT(NmpSponsorNodeId == NmLocalNodeId); CL_ASSERT(NmpJoinTimer == 0); CL_ASSERT(NmpJoinAbortPending == FALSE);
//
// Restart the join timer.
//
NmpJoinTimer = NM_JOIN_TIMEOUT; } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because the join was aborted.\n", JoinerNodeId ); } } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because the join was aborted.\n", JoinerNodeId ); }
OmDereferenceObject(joinerNode); } else { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because the node is not a member of the cluster.\n", JoinerNodeId ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Not in valid state to process the request.\n" ); }
NmpReleaseLock();
return(status);
} // s_NmRpcCreateBinding
error_status_ts_NmRpcCreateJoinerBinding( IN handle_t IDL_handle, IN DWORD JoinSequence, IN LPWSTR JoinerNodeId, IN LPWSTR JoinerInterfaceId )/*++
Notes:
The sponsor is responsible for aborting the join on failure.
--*/{ DWORD status;
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Processing CreateBinding request for joining node %1!ws!.\n", JoinerNodeId );
if (NmpLockedEnterApi(NmStateOnline)) { PNM_NODE joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode != NULL) { PNM_INTERFACE netInterface = OmReferenceObjectById( ObjectTypeNetInterface, JoinerInterfaceId );
if (netInterface != NULL) { //
// Verify that a join is still in progress.
//
if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) ) { status = NmpCreateJoinerRpcBindings( joinerNode, netInterface );
if (status != ERROR_SUCCESS) { WCHAR errorString[12];
wsprintfW(&(errorString[0]), L"%u", status); CsLogEvent3( LOG_UNUSUAL, NM_EVENT_JOINER_BIND_FAILED, OmObjectName(joinerNode), OmObjectName(netInterface->Network), errorString ); } } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failing create bindings for joining node %1!ws! because the join was aborted\n", JoinerNodeId ); }
OmDereferenceObject(netInterface); } else { status = ERROR_CLUSTER_NETINTERFACE_NOT_FOUND; ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Can't create binding for joining node %1!ws! - no corresponding interface for joiner interface %2!ws!.\n", JoinerNodeId, JoinerInterfaceId ); }
OmDereferenceObject(joinerNode); } else { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] CreateBinding call for joining node %1!ws! failed because the node is not a member of the cluster.\n", JoinerNodeId ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Not in valid state to process the request.\n" ); }
NmpReleaseLock();
return(status);
} // s_NmRpcCreateJoinerBinding
DWORDNmpCreateJoinerRpcBindings( IN PNM_NODE JoinerNode, IN PNM_INTERFACE JoinerInterface )/*++
Notes:
Called with the NmpLock held.
--*/{ DWORD status; PNM_NETWORK network = JoinerInterface->Network; CL_NODE_ID joinerNodeId = JoinerNode->NodeId;
CL_ASSERT(JoinerNode->NodeId == NmpJoinerNodeId); CL_ASSERT(JoinerNode->State == ClusterNodeJoining);
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Creating bindings for joining node %1!u! using network %2!ws!\n", joinerNodeId, OmObjectName(JoinerInterface->Network) );
//
// Make sure that this node has an interface on the target network.
//
if (NmpIsNetworkForInternalUse(network)) { if (network->LocalInterface != NULL) { if ( NmpIsInterfaceRegistered(JoinerInterface) && NmpIsInterfaceRegistered(network->LocalInterface)
) { status = NmpSetNodeInterfacePriority( JoinerNode, 0xFFFFFFFF, JoinerInterface, 1 );
if (status == ERROR_SUCCESS) { PNM_INTERFACE localInterface = network->LocalInterface;
//
// Create intracluster RPC bindings for the petitioner.
// The MM relies on these to perform the join.
//
OmReferenceObject(localInterface); OmReferenceObject(JoinerNode);
NmpReleaseLock();
status = NmpCreateRpcBindings(JoinerNode);
NmpAcquireLock();
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Unable to create RPC binding for " "joining node %1!u!, status %2!u!.\n", joinerNodeId, status ); }
OmDereferenceObject(JoinerNode); OmDereferenceObject(localInterface); } else { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to set interface priority for " "network %1!ws! (%2!ws!), status %3!u!\n", OmObjectId(network), OmObjectName(network), status ); } } else { status = ERROR_CLUSTER_NODE_UNREACHABLE; } } else { status = ERROR_CLUSTER_NETINTERFACE_NOT_FOUND; } } else { status = ERROR_CLUSTER_NODE_UNREACHABLE; }
if (status !=ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[NMJOIN] Failed to create binding for joining node %1!u! " "on network %2!ws! (%3!ws!), status %4!u!\n", joinerNodeId, OmObjectId(network), OmObjectName(network), status ); }
return(status);
} // NmpCreateJoinerRpcBinding
error_status_ts_NmRpcPetitionForMembership( IN handle_t IDL_handle, IN DWORD JoinSequence, IN LPCWSTR JoinerNodeId )/*++
Routine Description:
Server side of a join petition.
Arguments:
IDL_handle - RPC binding handle, not used.
JoinSequence - Supplies the sequence returned from NmRpcJoinBegin
JoinerNodeId - Supplies the ID of the node attempting to join.
Return Value:
ERROR_SUCCESS if successful
Win32 error otherwise.
--*/
{ DWORD status; PNM_NODE joinerNode;
#ifdef CLUSTER_TESTPOINT
TESTPT(TestpointJoinFailPetition) { return(999999); }#endif
NmpAcquireLock();
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Processing petition to join from node %1!ws!.\n", JoinerNodeId );
if (NmpLockedEnterApi(NmStateOnline)) {
joinerNode = OmReferenceObjectById(ObjectTypeNode, JoinerNodeId);
if (joinerNode != NULL) { //
// Verify that the join is still in progress
//
//
// DavidDio 6/13/2000
// There is a small window where a begin join update can
// succeed during a regroup, but the regroup ends before
// the joining node petitions to join. In this case, the
// node will be marked out of sync. Aborting the join
// after MMJoin() is much more heavyweight than before,
// so check for this condition now. (Bug 125778).
//
if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) && (NmpSponsorNodeId == NmLocalNodeId) && (!NmpJoinAbortPending) && (!NmpJoinerOutOfSynch) ) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Performing join.\n");
CL_ASSERT(joinerNode->State == ClusterNodeJoining); CL_ASSERT(NmpJoinerUp == FALSE); CL_ASSERT(NmpJoinTimer != 0);
//
// Call the MM to join this node to the cluster membership.
// Disable the join timer. Once the node becomes an active
// member, we won't need it anymore.
//
NmpJoinTimer = 0;
NmpReleaseLock();
status = MMJoin( joinerNode->NodeId, NM_CLOCK_PERIOD, NM_SEND_HB_RATE, NM_RECV_HB_RATE, NM_MM_JOIN_TIMEOUT );
NmpAcquireLock();
//
// Verify that the join is still in progress
//
if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) ) { CL_ASSERT(NmpSponsorNodeId == NmLocalNodeId); CL_ASSERT(joinerNode->State == ClusterNodeJoining); CL_ASSERT(NmpJoinTimer == 0); CL_ASSERT(NmpJoinAbortPending == FALSE);
// GorN 3/22/2000
// We hit a case when MMJoin has succeeded after a regroup
// that killed one of the nodes (not joiner and not sponsor)
// thus leaving the joiner out of sync
// We need to abourt the join in this case too
if (status != MM_OK || NmpJoinerOutOfSynch) { status = MMMapStatusToDosError(status);
if (NmpJoinerOutOfSynch) { status = ERROR_CLUSTER_JOIN_ABORTED; }
//
// Abort the join
//
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Petition to join by node %1!ws! failed, status %2!u!.\n", JoinerNodeId, status ); //
// If MMJoin was unsuccessful it initiates a banishing
// regroup. This regroup will deliver node down events
// on all nodes that saw hb's from the joiner.
//
// Calling MMBlockIfRegroupIsInProgress here will guarantee that
// Phase2 cleanup is complete on all nodes, before we
// call NmpJoinAbort.
//
NmpReleaseLock(); MMBlockIfRegroupIsInProgress(); NmpAcquireLock();
NmpJoinAbort(status, joinerNode); } else { NmpSuccessfulMMJoin = TRUE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Petition to join by node %1!ws! succeeded.\n", JoinerNodeId ); }
#ifdef MM_IN_CLUSNET
if (status == MM_OK) {
status = NmJoinNodeToCluster(joinerNodeId);
if (status != ERROR_SUCCESS) {
DWORD clusnetStatus;
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!ws! failed, status %2!u!.\n", JoinerNodeId, status );
CL_LOGFAILURE( status );
NmpReleaseLock();
MMEject(joinerNodeId);
NmpAcquireLock();
clusnetStatus = ClusnetOfflineNodeComm( NmClusnetHandle, joinerNodeId ); CL_ASSERT( (status == ERROR_SUCCESS) || (status == ERROR_CLUSTER_NODE_ALREADY_DOWN ); } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join completed successfully.\n" ); } }
#endif // MM_IN_CLUSNET
} else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Petition to join by node %1!ws! failed because the join was aborted.\n", JoinerNodeId ); } } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Petition by node %1!ws! failed because the join was aborted\n", JoinerNodeId ); }
OmDereferenceObject(joinerNode); } else { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Petition to join by %1!ws! failed because the node is not a cluster member\n", JoinerNodeId ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Not in valid state to process the request.\n" ); }
NmpReleaseLock();
return(status);
} // s_NmRpcPetitionForMembership
error_status_ts_NmRpcGetLeaderNodeId( IN handle_t IDL_handle, IN DWORD JoinSequence, OPTIONAL IN LPWSTR JoinerNodeId, OPTIONAL OUT LPDWORD LeaderNodeId ){ DWORD status = ERROR_SUCCESS; PNM_NODE joinerNode = NULL;
NmpAcquireLock();
if (NmpLockedEnterApi(NmStateOnline)){ joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode != NULL) { if ( (JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) && (NmpSponsorNodeId == NmLocalNodeId) && !NmpJoinAbortPending ) { CL_ASSERT(joinerNode->State == ClusterNodeJoining);
*LeaderNodeId = NmpLeaderNodeId; } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] GetLeaderNodeId call for joining node %1!ws! failed because the join was aborted.\n", JoinerNodeId ); }
OmDereferenceObject(joinerNode); } else { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] GetLeaderNodeId call for joining node %1!ws! failed because the node is not a member of the cluster.\n", JoinerNodeId ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Not in valid state to process GetLeaderNodeId request.\n" ); }
NmpReleaseLock();
return(status);
} // s_NmRpcGetLeaderNodeId
DWORDNmpUpdateJoinComplete( IN PNM_JOIN_UPDATE JoinUpdate ){ DWORD status;
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Processing JoinComplete update from node %1!ws!\n", JoinUpdate->NodeId );
if (NmpLockedEnterApi(NmStateOnline)) { PNM_NODE joinerNode; LPWSTR joinerIdString = JoinUpdate->NodeId;
joinerNode = OmReferenceObjectById(ObjectTypeNode, joinerIdString);
if (joinerNode != NULL) {
CL_ASSERT(joinerNode != NmLocalNode);
//
// Verify that the join is still in progress and nothing has
// changed.
//
if ( (JoinUpdate->JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) && (joinerNode->State == ClusterNodeJoining) && NmpJoinerUp && !NmpJoinerOutOfSynch ) { PNM_INTERFACE netInterface; PNM_NETWORK network; PLIST_ENTRY ifEntry;
NmpJoinerNodeId = ClusterInvalidNodeId; NmpSponsorNodeId = ClusterInvalidNodeId; NmpJoinTimer = 0; NmpJoinAbortPending = FALSE; NmpJoinSequence = 0; NmpJoinerUp = FALSE;
if (JoinUpdate->IsPaused != 0) { //
// This node is coming up in the paused state.
//
joinerNode->State = ClusterNodePaused; } else { joinerNode->State = ClusterNodeUp; }
joinerNode->ExtendedState = ClusterNodeJoining;
ClusterEvent(CLUSTER_EVENT_NODE_UP, (PVOID)joinerNode);
//
// Reset the interface priorities for this node.
//
for (ifEntry = joinerNode->InterfaceList.Flink; ifEntry != &joinerNode->InterfaceList; ifEntry = ifEntry->Flink ) { netInterface = CONTAINING_RECORD( ifEntry, NM_INTERFACE, NodeLinkage );
network = netInterface->Network;
if ( NmpIsNetworkForInternalUse(network) && NmpIsInterfaceRegistered(netInterface) ) { status = ClusnetSetInterfacePriority( NmClusnetHandle, joinerNode->NodeId, network->ShortId, 0 );
CL_ASSERT(status == ERROR_SUCCESS); } }
status = ERROR_SUCCESS; } else { status = ERROR_CLUSTER_JOIN_ABORTED; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!ws! cannot complete because the join was aborted\n", joinerIdString ); }
OmDereferenceObject(joinerNode); } else { status =ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!ws! cannot complete because the node is not a cluster member.\n", joinerIdString ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Not in valid state to process JoinComplete update.\n" ); }
NmpReleaseLock();
// DavidDio 10/27/2000
// Bug 213781: NmpUpdateJoinComplete must always return ERROR_SUCCESS.
// Otherwise, there is a small window whereby GUM sequence numbers on
// remaining cluster nodes can fall out of sync. If the join should
// be aborted, return ERROR_SUCCESS but poison the joiner out-of-band.
if (status != ERROR_SUCCESS) { DWORD dwJoinerId;
if (JoinUpdate->NodeId != NULL) { dwJoinerId = wcstoul(JoinUpdate->NodeId, NULL, 10); ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!u! failed with status %2!u!. Initiating banishment.\n", dwJoinerId, status ); NmAdviseNodeFailure(dwJoinerId, status); } else { dwJoinerId = ClusterInvalidNodeId; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!ws! failed with status %2!u!. Cannot initiate banishment as node id is unknown.\n", dwJoinerId, status ); } }
return(ERROR_SUCCESS);
} // NmpUpdateJoinComplete
DWORDNmpUpdateJoinAbort( IN BOOL SourceNode, IN LPDWORD JoinSequence, IN LPWSTR JoinerNodeId )/*++
Notes:
--*/{ DWORD status = ERROR_SUCCESS;
NmpAcquireLock();
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Received update to abort join sequence %1!u! (joiner id %2!ws!).\n", *JoinSequence, JoinerNodeId );
if (NmpLockedEnterApi(NmStateOnline)) { PNM_NODE joinerNode = OmReferenceObjectById( ObjectTypeNode, JoinerNodeId );
if (joinerNode != NULL) { //
// Check if the specified join is still in progress.
//
if ( (*JoinSequence == NmpJoinSequence) && (NmpJoinerNodeId == joinerNode->NodeId) ) { CL_ASSERT(NmpSponsorNodeId != ClusterInvalidNodeId); CL_ASSERT(joinerNode->State == ClusterNodeJoining);
//
// Assumption:
//
// An abort cannot occur during the MM join process.
// If the joiner is not already up, it cannot come up
// during the abort processing.
//
// Assert condition may not be true with the current MM join code.
// Some nodes might have got monitor node and set
// NmpJoinerUp state to TRUE by the time the sponsor issued
// an abort update
//
//CL_ASSERT(NmpJoinerUp == FALSE);
if (NmpCleanupIfJoinAborted) {
NmpCleanupIfJoinAborted = FALSE;
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Issuing a node down event for %1!u!.\n", joinerNode->NodeId );
//
// This node is not yet active in the membership.
// Call the node down event handler to finish the abort.
//
//
// We will not call NmpMsgCleanup1 and NmpMsgCleanup2,
// because we cannot guarantee that they will get executed
// in a barrier style fashion
//
// !!! Lock will be acquired by NmpNodeDownEventHandler
// second time. Is it OK?
//
NmpNodeDownEventHandler(joinerNode); } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Node down was already issued for %1!u!.\n", joinerNode->NodeId ); } } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Ignoring old join abort update with sequence %1!u!.\n", *JoinSequence ); }
OmDereferenceObject(joinerNode); status = ERROR_SUCCESS; } else { status = ERROR_CLUSTER_NODE_NOT_MEMBER; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!ws! cannot be aborted because the node is not a cluster member.\n", JoinerNodeId ); }
NmpLockedLeaveApi(); } else { status = ERROR_NODE_NOT_AVAILABLE; ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Not in valid state to process JoinAbort update.\n" ); }
NmpReleaseLock();
return(status);
} // NmpUpdateJoinAbort
VOIDNmpJoinAbort( DWORD AbortStatus, PNM_NODE JoinerNode )/*++
Routine Description:
Issues a JoinAbort update.
Notes:
Called with the NmpLock held.
--*/{ DWORD status; DWORD joinSequence = NmpJoinSequence; WCHAR errorString[12];
// GorN 10/31/2001 #488486
// By the time we get to the NmpJoinAbort code
// we can have a node down event processed
// that clears up NmpJoinerNodeId.
// In this case just log the message and bail out
// CL_ASSERT(NmpJoinerNodeId != ClusterInvalidNodeId);
// CL_ASSERT(NmpSponsorNodeId == NmLocalNodeId);
// CL_ASSERT(JoinerNode->State == ClusterNodeJoining);
if (AbortStatus == ERROR_TIMEOUT) { wsprintfW(&(errorString[0]), L"%u", AbortStatus); CsLogEvent1( LOG_CRITICAL, NM_EVENT_JOIN_TIMED_OUT, OmObjectName(JoinerNode) ); } else { wsprintfW(&(errorString[0]), L"%u", AbortStatus); CsLogEvent2( LOG_CRITICAL, NM_EVENT_SPONSOR_JOIN_ABORTED, OmObjectName(JoinerNode), errorString ); }
//
// Assumption:
//
// An abort cannot occur during the MM join process. If the joiner
// is not already up, it cannot come up during the abort processing.
//
if (NmpSuccessfulMMJoin == FALSE) { //
// The joining node has not become active yet. Issue
// an abort update.
//
joinSequence = NmpJoinSequence;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Issuing update to abort join of node %1!u!.\n", NmpJoinerNodeId );
NmpReleaseLock();
status = GumSendUpdateEx( GumUpdateMembership, NmUpdateJoinAbort, 2, sizeof(DWORD), &joinSequence, NM_WCSLEN(OmObjectId(JoinerNode)), OmObjectId(JoinerNode) );
NmpAcquireLock(); } else { //
// The joining node is already active in the membership.
// Ask the MM to kick it out. The node down event will
// finish the abort process.
//
CL_NODE_ID joinerNodeId = NmpJoinerNodeId;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Ejecting joining node %1!u! from the cluster membership.\n", NmpJoinerNodeId );
NmpReleaseLock();
status = MMEject(joinerNodeId);
NmpAcquireLock(); }
if (status != MM_OK) { status = MMMapStatusToDosError(status); ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Update to abort join of node %1!u! failed, status %2!u!\n", JoinerNode->NodeId, status );
//
// If the join is still pending, and this is the sponsor node,
// force a timeout to retry the abort. If we aren't the sponsor,
// there isn't much we can do.
//
if ( (joinSequence == NmpJoinSequence) && (NmpJoinerNodeId == JoinerNode->NodeId) && (NmpSponsorNodeId == NmLocalNodeId) ) { NmpJoinTimer = 1; NmpJoinAbortPending = FALSE; } }
return;
} // NmpJoinAbort
VOIDNmpJoinAbortWorker( IN PCLRTL_WORK_ITEM WorkItem, IN DWORD Status, IN DWORD BytesTransferred, IN ULONG_PTR IoContext )/*++
Routine Description:
Worker thread for aborting a join.
--*/{ DWORD joinSequence = PtrToUlong(WorkItem->Context);
NmpAcquireLock();
//
// The active thread count was bumped up when this item was scheduled.
// No need to call NmpEnterApi().
//
//
// If the join is still pending, begin the abort process.
//
if ( (joinSequence == NmpJoinSequence) && (NmpJoinerNodeId != ClusterInvalidNodeId) && NmpJoinAbortPending ) { PNM_NODE joinerNode = NmpIdArray[NmpJoinerNodeId];
if (joinerNode != NULL) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Worker thread initiating abort of joining node %1!u!\n", NmpJoinerNodeId );
NmpJoinAbort(ERROR_TIMEOUT, joinerNode); } else { CL_ASSERT(joinerNode != NULL); } } else { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Skipping join abort, sequence to abort %1!u!, current join sequence %2!u!, " "joiner node: %3!u! sponsor node: %4!u!\n", joinSequence, NmpJoinSequence, NmpJoinerNodeId, NmpSponsorNodeId ); }
NmpLockedLeaveApi();
NmpReleaseLock();
LocalFree(WorkItem);
return;
} // NmpJoinAbortWorker
VOIDNmpJoinTimerTick( IN DWORD MsTickInterval )/*++
Notes: Called with NmpLock held.
--*/{ if (NmpLockedEnterApi(NmStateOnline)) { //
// If we are sponsoring a join, update the timer.
//
if ( (NmpJoinerNodeId != ClusterInvalidNodeId) && (NmpSponsorNodeId == NmLocalNodeId) && !NmpJoinAbortPending && (NmpJoinTimer != 0) ) { //ClRtlLogPrint(LOG_NOISE,
// "[NMJOIN] Timer tick (%1!u! ms)\n",
// Interval
// );
if (NmpJoinTimer > MsTickInterval) { NmpJoinTimer -= MsTickInterval; } else { //
// The join has timed out. Schedule a worker thread to
// carry out the abort process.
//
PCLRTL_WORK_ITEM workItem;
ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Join of node %1!u! has timed out.\n", NmpJoinerNodeId );
workItem = LocalAlloc(LMEM_FIXED, sizeof(CLRTL_WORK_ITEM));
if (workItem != NULL) { DWORD status;
ClRtlInitializeWorkItem( workItem, NmpJoinAbortWorker, ULongToPtr(NmpJoinSequence) );
status = ClRtlPostItemWorkQueue( CsDelayedWorkQueue, workItem, 0, 0 );
if (status == ERROR_SUCCESS) { //
// Stop the timer, flag that an abort is in progress,
// and account for the thread we just scheduled.
//
NmpJoinTimer = 0; NmpJoinAbortPending = TRUE; NmpActiveThreadCount++; } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to schedule abort of join, status %1!u!.\n", status ); LocalFree(workItem); } } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] Failed to allocate memory for join abort.\n" ); } } } NmpLockedLeaveApi(); }
return;
} // NmpJoinTimerTick
DWORDNmpNetworkTimerThread( PVOID Parameter )/*++
Routine Description:
Forked as the cluster is initializing, calls NmTimerTick to drive the NM timers.
Separated from the MM timer thread for bug 742997. Code is modeled after MM timer, except that we assume that the tick interval cannot change.
Arguments:
Parameter - points to event that is signaled when thread should stop
Return value:
SUCCESS
--*/{ HANDLE stopEvent = (HANDLE) Parameter; DWORD status;
do {
// Wait on the stop event. If the stop event is
// signalled, clussvc is exiting and we break out
// of the loop. Otherwise, the timeout will fire
// and we drive the NM timers.
status = WaitForSingleObject( stopEvent, RGP_CLOCK_PERIOD ); if (status == WAIT_TIMEOUT) { NmTimerTick(RGP_CLOCK_PERIOD); } } while (status == WAIT_TIMEOUT);
if (status != WAIT_OBJECT_0) { DWORD error = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] NM timer thread exiting with " "status %1!u!, last error %2!u!. Removing " "self from cluster.\n", status, error ); // We cannot continue without the NM timer.
CsInconsistencyHalt(status); } else { ClRtlLogPrint(LOG_NOISE, "[NM] NM timer thread exiting.\n" ); }
return(status); } // NmpNetworkTimerThread
DWORDNmpStartNetworkTimerThread( VOID )/*++
Routine Description:
Starts the NM timer thread.
Arguments:
None.
Return value:
Status of event and thread creation.
--*/{ DWORD status; DWORD threadID = 0; NmpNetworkTimerThreadHandle = NULL; NmpNetworkTimerThreadStopEvent = NULL; NmpNetworkTimerThreadStopEvent = CreateEvent( NULL, // No security
FALSE, // Not autoreset
FALSE, // Not initially signalled
NULL // No name
); if (NmpNetworkTimerThreadStopEvent == NULL) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create NM timer thread stop event, " "error %1!u!.\n", status ); goto error_exit; }
NmpNetworkTimerThreadHandle = CreateThread( 0, // security
0, // stack size - use same as primary thread
(LPTHREAD_START_ROUTINE) NmpNetworkTimerThread, (PVOID) NmpNetworkTimerThreadStopEvent, // stop event is parameter
CREATE_SUSPENDED, // start it after setting prio
&threadID // thread ID returned here
); if (NmpNetworkTimerThreadHandle == NULL) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to create NM timer thread, " "error %1!u!.\n", status ); goto error_exit; }
// Set the priority, just as done to the MM timer thread
if (!SetThreadPriority( NmpNetworkTimerThreadHandle, NM_TIMER_THREAD_PRIORITY )) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to set NM timer thread priority to %1!u!, " "error %2!u!.\n", NM_TIMER_THREAD_PRIORITY, status ); goto error_exit; }
// Start the thread.
if (ResumeThread(NmpNetworkTimerThreadHandle) == -1) { status = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[NM] Failed to resume NM timer thread, " "error %1!u!.\n", status ); goto error_exit; }
ClRtlLogPrint(LOG_NOISE, "[NM] Started NM timer thread, thread id 0x%1!x!.\n", threadID );
return(ERROR_SUCCESS);
error_exit:
if (NmpNetworkTimerThreadStopEvent != NULL) { CloseHandle(NmpNetworkTimerThreadStopEvent); NmpNetworkTimerThreadStopEvent = NULL; }
if (NmpNetworkTimerThreadHandle != NULL) { CloseHandle(NmpNetworkTimerThreadHandle); NmpNetworkTimerThreadHandle = NULL; }
return(status); } // NmpStartNetworkTimerThread
VOIDNmpStopNetworkTimerThread( VOID )/*++
Routine Description:
Stops the NM timer thread by signaling the stop event. Waits maximum of 60 seconds before giving up and letting process cleanup sort out the mess (this is okay because the NM timer thread is stopped only when the clussvc process is exiting).
Arguments:
None.
Return value:
None.
--*/{ DWORD status; if (NmpNetworkTimerThreadStopEvent != NULL && NmpNetworkTimerThreadHandle != NULL) {
// Signal the thread to stop.
SetEvent(NmpNetworkTimerThreadStopEvent);
// Wait for the thread to exit.
status = WaitForSingleObject(NmpNetworkTimerThreadHandle, 60 * 1000); if (status != WAIT_OBJECT_0) { DWORD error = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[NM] NM network timer thread could not be gracefully " "stopped, status %1!u!, last error %2!u!.\n", status, error ); }
CloseHandle(NmpNetworkTimerThreadStopEvent); CloseHandle(NmpNetworkTimerThreadHandle); }
NmpNetworkTimerThreadStopEvent = NULL; NmpNetworkTimerThreadHandle = NULL;
return; } // NmpStopNetworkTimerThread
VOIDNmTimerTick( IN DWORD MsTickInterval )/*++
Routine Description:
Implements all of the NM timers. Called on every tick of the common NM/MM timer - currently every 300ms.
Arguments:
MsTickInterval - The number of milliseconds that have passed since the last tick.
ReturnValue:
None.
--*/{ NmpAcquireLock();
NmpNetworkTimerTick(MsTickInterval);
NmpJoinTimerTick(MsTickInterval);
#if DBG
// Addition for checking for hung RPC threads.
NmpRpcTimerTick(MsTickInterval);
#endif // DBG
NmpReleaseLock();
return;
} // NmTimerTick
error_status_ts_JoinAddNode4( IN handle_t IDL_handle, IN LPCWSTR lpszNodeName, IN DWORD dwNodeHighestVersion, IN DWORD dwNodeLowestVersion, IN DWORD dwNodeProductSuite, IN BOOL IsNodeRunningWin64, IN DWORD dwNodeProcessorArchitecture )/*++
Routine Description:
Adds a new node to the cluster.
Arguments:
IDL_handle - RPC binding handle, not used.
lpszNodeName - Supplies the name of the new node.
dwNodeHighestVersion - The highest cluster version number that the new node can support.
dwNodeLowestVersion - The lowest cluster version number that the new node can support.
dwNodeProductSuite - The product suite type identifier for the new node.
Return Value:
ERROR_SUCCESS if successful
Win32 error code otherwise.
Notes:
This is a new routine in Whistler. It will be invoked only by Whistler Setup.
The cluster registry APIs cannot be called while holding the NmpLock, or a deadlock may occur.
--*/{ DWORD status; DWORD registryNodeLimit; SYSTEM_INFO SystemInfo;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Received request to add node '%1!ws!' to the cluster.\n", lpszNodeName );
//
// Read the necessary registry parameters before acquiring
// the NM lock.
//
status = DmQueryDword( DmClusterParametersKey, CLUSREG_NAME_MAX_NODES, ®istryNodeLimit, NULL );
if (status != ERROR_SUCCESS) { registryNodeLimit = 0; }
NmpAcquireLock();
if (NmpLockedEnterApi(NmStateOnline)) { DWORD retryCount = 0;
//if this is the last node and it has been evicted
//but the cleanup hasnt completed and hence the
//service is up, then it should not entertain
//any new join requests
if (NmpLastNodeEvicted) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] This node was evicted and hence is not in a valid state to process a " "request to add a node to the cluster.\n" ); status = ERROR_NODE_NOT_AVAILABLE; NmpLockedLeaveApi(); goto FnExit; }
#if 0
status = ClRtlCheck64BitCompatibility(ClRtlIsProcessRunningOnWin64(GetCurrentProcess), IsNodeRunningWin64); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] This joining node doesnt match the 32/64 bitness of the current cluster\n" ); NmpLockedLeaveApi(); goto FnExit; }
#endif
GetSystemInfo(&SystemInfo); status = ClRtlCheckProcArchCompatibility(SystemInfo.wProcessorArchitecture, (WORD)dwNodeProcessorArchitecture); if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] This joining node doesnt meet the processor architecture compatibility with this cluster\n" ); NmpLockedLeaveApi(); goto FnExit; }
while (TRUE) { if (NmpLeaderNodeId == NmLocalNodeId) { //
// This node is the leader, call the internal
// handler directly.
//
status = NmpAddNode( lpszNodeName, dwNodeHighestVersion, dwNodeLowestVersion, dwNodeProductSuite, registryNodeLimit ); } else { //
// Forward the request to the leader.
//
RPC_BINDING_HANDLE binding = Session[NmpLeaderNodeId];
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Forwarding request to add node '%1!ws!' " "to the cluster to the leader (node %2!u!).\n", lpszNodeName, NmpLeaderNodeId );
CL_ASSERT(binding != NULL);
NmpReleaseLock();
status = NmRpcAddNode( binding, lpszNodeName, dwNodeHighestVersion, dwNodeLowestVersion, dwNodeProductSuite );
NmpAcquireLock(); }
//
// Check for the error codes that indicate either that
// another AddNode operation is in progress or that the
// leadership is changing. We will retry in these cases.
//
if ( (status != ERROR_CLUSTER_JOIN_IN_PROGRESS) && (status != ERROR_NODE_NOT_AVAILABLE) ) { break; }
//
// Sleep for 3 seconds and try again. We will give up and
// return the error after retrying for 2 minutes.
//
if (++retryCount > 40) { break; }
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] AddNode operation for node '%1!ws! delayed " "waiting for competing AddNode operation to complete.\n", lpszNodeName );
NmpReleaseLock();
Sleep(3000);
NmpAcquireLock();
} // end while(TRUE)
NmpLockedLeaveApi(); } else { ClRtlLogPrint(LOG_UNUSUAL, "[NMJOIN] This system is not in a valid state to process a " "request to add a node to the cluster.\n" ); status = ERROR_NODE_NOT_AVAILABLE; }
FnExit: NmpReleaseLock();
return(status);
} // s_NmJoinAddNode4
error_status_ts_JoinAddNode3( IN handle_t IDL_handle, IN LPCWSTR lpszNodeName, IN DWORD dwNodeHighestVersion, IN DWORD dwNodeLowestVersion, IN DWORD dwNodeProductSuite )/*++
Routine Description:
Adds a new node to the cluster.
Arguments:
IDL_handle - RPC binding handle, not used.
lpszNodeName - Supplies the name of the new node.
dwNodeHighestVersion - The highest cluster version number that the new node can support.
dwNodeLowestVersion - The lowest cluster version number that the new node can support.
dwNodeProductSuite - The product suite type identifier for the new node.
Return Value:
ERROR_SUCCESS if successful
Win32 error code otherwise.
Notes:
This is a new routine in NT5. It performs the AddNode operation correctly. It will never be invoked by an NT4 system. It cannot be invoked if an NT4 node is in the cluster without violating the license agreement.
The cluster registry APIs cannot be called while holding the NmpLock, or a deadlock may occur.
--*/{ return(s_JoinAddNode4(IDL_handle, lpszNodeName, dwNodeHighestVersion, dwNodeLowestVersion,dwNodeProductSuite, FALSE, (DWORD)PROCESSOR_ARCHITECTURE_INTEL));} // s_NmJoinAddNode3
// This is used by setup of all highest major versions post 1.0
error_status_ts_JoinAddNode2( IN handle_t IDL_handle, IN LPCWSTR lpszNodeName, IN DWORD dwNodeHighestVersion, IN DWORD dwNodeLowestVersion )/*++
Routine Description:
Adds a new node to the cluster database.
Arguments:
IDL_handle - RPC binding handle, not used.
lpszNodeName - Supplies the name of the new node.
Return Value:
ERROR_SUCCESS if successful
Win32 error code otherwise.
Notes:
This routine was defined in NT4-SP4. JoinAddNode3 is used by NT5. Since it is impossible to install clustering using the NT4-SP4 software, this routine should never be invoked.
--*/
{ CL_ASSERT(FALSE);
return(ERROR_CLUSTER_INCOMPATIBLE_VERSIONS);}
error_status_ts_JoinAddNode( IN handle_t IDL_handle, IN LPCWSTR lpszNodeName )/*++
Routine Description:
Adds a new node to the cluster database.
Arguments:
IDL_handle - RPC binding handle, not used.
lpszNodeName - Supplies the name of the new node.
Return Value:
ERROR_SUCCESS if successful
Win32 error code otherwise.
Notes:
This is the routine that NT4-SP3 setup invokes to add a new node to a cluster. The combination of NT4-SP3 and NT5 is not supported.
--*/
{ return(ERROR_CLUSTER_INCOMPATIBLE_VERSIONS);}
//
// The rest of the code is currently unused.
//
error_status_ts_NmRpcDeliverJoinMessage( IN handle_t IDL_handle, IN UCHAR * Message, IN DWORD MessageLength )/*++
Routine Description:
Server side of the RPC interface for delivering membership join messages.
Arguments:
IDL_handle - RPC binding handle, not used.
buffer - Supplies a pointer to the message data.
length - Supplies the length of the message data.
Return Value:
ERROR_SUCCESS
--*/
{ DWORD status = ERROR_SUCCESS;
#ifdef MM_IN_CLUSNET
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Delivering join message to Clusnet.\n" );
status = ClusnetDeliverJoinMessage( NmClusnetHandle, Message, MessageLength );
#endif
return(status);}
#ifdef MM_IN_CLUSNET
DWORDNmpSendJoinMessage( IN ULONG DestNodeMask, IN PVOID Message, IN ULONG MessageLength ){ DWORD status = ERROR_SUCCESS; CL_NODE_ID node;
CL_ASSERT(NmMaxNodeId != ClusterInvalidNodeId);
for ( node = ClusterMinNodeId; node <= NmMaxNodeId; node++, (DestNodeMask >>= 1) ) {
if (DestNodeMask & 0x1) { if (node != NmLocalNodeId) {
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Sending join message to node %1!u!.\n", node );
status = NmRpcDeliverJoinMessage( Session[node->NodeId], Message, MessageLength );
if (status == RPC_S_CALL_FAILED_DNE) { //
// Try again since the first call to a restarted
// RPC server will fail.
//
status = NmRpcDeliverJoinMessage( Session[node->NodeId], Message, MessageLength ); } } else { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Delivering join message to local node.\n" );
status = ClusnetDeliverJoinMessage( NmClusnetHandle, Message, MessageLength ); }
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] send of join message to node %1!u! failed, status %2!u!\n", node, status ); break; } } }
return(status);
} // NmpSendJoinMessage
DWORDNmJoinNodeToCluster( CL_NODE_ID joinerNodeId ){ DWORD status; PVOID message = NULL; ULONG messageLength; ULONG destMask; CLUSNET_JOIN_PHASE phase;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] Joining node %1!u! to the cluster.\n", joinerNodeId );
for (phase = ClusnetJoinPhase1; phase <= ClusnetJoinPhase4; phase++) {
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] JoinNode phase %1!u!\n", phase );
status = ClusnetJoinCluster( NmClusnetHandle, joinerNodeId, phase, NM_MM_JOIN_TIMEOUT, &message, &messageLength, &destMask );
if (status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[NMJOIN] JoinNode phase %1!u! failed, status %2!u!\n", phase, status );
break; }
status = NmpSendJoinMessage( destMask, message, messageLength );
if (status != ERROR_SUCCESS) { DWORD abortStatus;
ClRtlLogPrint(LOG_NOISE, "[NMJOIN] send join message failed %1!u!, aborting join of node %2!u!.\n", status, joinerNodeId );
abortStatus = ClusnetJoinCluster( NmClusnetHandle, joinerNodeId, ClusnetJoinPhaseAbort, NM_MM_JOIN_TIMEOUT, &message, &messageLength, &destMask );
if (abortStatus == ERROR_SUCCESS) { (VOID) NmpSendJoinMessage( destMask, message, messageLength ); }
break; } }
if (message != NULL) { ClusnetEndJoinCluster(NmClusnetHandle, message); }
return(status);
} // NmJoinNodeToCluster
#endif // MM_IN_CLUSNET
|
