/*++ Copyright (c) 1996 Microsoft Corporation Module Name: dmlog.c Abstract: Contains the quorum logging related functions for the cluster registry. Author: Sunita Shrivastava (sunitas) 24-Apr-1996 Revision History: --*/ #include "dmp.h" #include "tchar.h" #include "clusudef.h" /**** @doc EXTERNAL INTERFACES CLUSSVC DM ****/ //global static data HLOG ghQuoLog=NULL; //pointer to the quorum log DWORD gbIsQuoResOnline = FALSE; DWORD gbNeedToCheckPoint = FALSE; DWORD gbIsQuoResEnoughSpace = TRUE; HLOG ghNewQuoLog = NULL; //pointer to the new quorum resource //global data extern HANDLE ghQuoLogOpenEvent; extern BOOL gbIsQuoLoggingOn; extern HANDLE ghDiskManTimer; extern HANDLE ghCheckpointTimer; extern PFM_RESOURCE gpQuoResource; //set when DmFormNewCluster is complete extern BOOL gbDmInited; #if NO_SHARED_LOCKS extern CRITICAL_SECTION gLockDmpRoot; #else extern RTL_RESOURCE gLockDmpRoot; #endif //forward definitions void DmpLogCheckPointCb(); /**** @func DWORD | DmPrepareQuorumResChange| When the quorum resource is changed, the FM invokes this api on the owner node of the new quorum resource to create a new quorum log file. @parm IN PVOID | pResource | The new quorum resource. @parm IN LPCWSTR | lpszPath | The path for temporary cluster files. @parm IN DWORD | dwMaxQuoLogSize | The maximum size limit for the quorum log file. @comm When a quorum resource is changed, the fm calls this funtion before it updates the quorum resource. If a new log file needs to be created, a checkpoint is taken. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD DmPrepareQuorumResChange( IN PVOID pResource, IN LPCWSTR lpszPath, IN DWORD dwMaxQuoLogSize) { DWORD dwError=ERROR_SUCCESS; PFM_RESOURCE pNewQuoRes; WCHAR szFileName1[MAX_PATH]; //for new quorum log,for tombstonefile LSN FirstLsn; WCHAR szFileName2[MAX_PATH]; //for old quorum log, for temp tombstone DWORD dwCurLogSize; DWORD dwMaxLogSize; DWORD dwChkPtSequence; WIN32_FIND_DATA FindData; QfsHANDLE hSrchTmpFiles; pNewQuoRes = (PFM_RESOURCE)pResource; ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - Entry\r\n"); //the resource is already online at this point //if the directory doesnt exist create it dwError = QfsClRtlCreateDirectory(lpszPath); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - Failed to create directory, Status=%1!u!\r\n", dwError); goto FnExit; } lstrcpyW(szFileName1, lpszPath); lstrcatW(szFileName1, cszQuoFileName); //if the log file is open here //this implies that the new quorum resource is the on the same node //as the old one if (ghQuoLog) { LogGetInfo(ghQuoLog, szFileName2, &dwCurLogSize, &dwMaxLogSize); //if the file is the same as the new log file, simply set the size if (!lstrcmpiW(szFileName2, szFileName1)) { LogSetInfo(ghQuoLog, dwMaxQuoLogSize); ghNewQuoLog = ghQuoLog; goto FnExit; } } //delele all the quorum logging related files //delete the log if it exits QfsDeleteFile(szFileName1); //delete all checkpoint files lstrcpyW(szFileName2, lpszPath); lstrcatW(szFileName2, L"*.tmp"); hSrchTmpFiles = QfsFindFirstFile(szFileName2, & FindData); if (QfsIsHandleValid(hSrchTmpFiles)) { lstrcpyW(szFileName2, lpszPath); lstrcatW(szFileName2, FindData.cFileName); QfsDeleteFile(szFileName2); while (QfsFindNextFile( hSrchTmpFiles, & FindData)) { lstrcpyW(szFileName2, lpszPath); lstrcatW(szFileName2, FindData.cFileName); QfsDeleteFile(szFileName2); } QfsFindClose(hSrchTmpFiles); } dwError = QfsSetFileSecurityInfo(lpszPath, GENERIC_ALL, GENERIC_ALL, 0); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - ClRtlSetObjSecurityInfo Failed, Status=%1!u!\r\n", dwError); goto FnExit; } //open the new log file ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange: the name of the quorum file is %1!ls!\r\n", szFileName1); //open the log file ghNewQuoLog = LogCreate(szFileName1, dwMaxQuoLogSize, (PLOG_GETCHECKPOINT_CALLBACK)DmpGetSnapShotCb, NULL, TRUE, &FirstLsn); if (!ghNewQuoLog) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmPrepareQuorumResChange: Quorum log could not be opened, error = %1!u!\r\n", dwError); CsLogEventData1( LOG_CRITICAL, CS_DISKWRITE_FAILURE, sizeof(dwError), &dwError, szFileName1 ); CsInconsistencyHalt(ERROR_QUORUMLOG_OPEN_FAILED); } //create a checkpoint in the new place dwError = DmpGetSnapShotCb(lpszPath, NULL, szFileName1, &dwChkPtSequence); if (dwError != ERROR_SUCCESS) { CL_LOGFAILURE(dwError); CsInconsistencyHalt(ERROR_QUORUMLOG_OPEN_FAILED); goto FnExit; } dwError = LogCheckPoint(ghNewQuoLog, TRUE, szFileName1, dwChkPtSequence); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - failed to take chkpoint, error = %1!u!\r\n", dwError); goto FnExit; } ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - checkpoint taken\r\n"); // // Call the checkpoint manager to copy over any checkpoint files // if ( !( CsNoQuorum ) || ( gpQuoResource->State == ClusterResourceOnline ) ) { dwError = CpCopyCheckpointFiles(lpszPath, FALSE); if (dwError != ERROR_SUCCESS) { goto FnExit; } } else { ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange: Skip copying checkpoint files from old quorum, FixQuorum=%1!u!, QuoState=%2!u!...\n", CsNoQuorum, gpQuoResource->State); } //create the tombstone and tmp file names lstrcpyW(szFileName1, lpszPath); lstrcatW(szFileName1, cszQuoTombStoneFile); lstrcpyW(szFileName2, lpszPath); lstrcatW(szFileName2, cszTmpQuoTombStoneFile); //rename the quorum tomstone file,it if it exists if (!QfsMoveFileEx(szFileName1, szFileName2, MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH)) { //this may fail if the tombstone doesnt exist, ignore error ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmPrepareQuorumResChange:tombstone doesnt exist,movefilexW failed, error=0x%1!08lx!\r\n", GetLastError()); } FnExit: if (dwError != ERROR_SUCCESS) { //if not sucess, clean up the new file if (ghNewQuoLog) { LogClose(ghNewQuoLog); ghNewQuoLog = NULL; } ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - Exit, error=0x%1!08lx!\r\n", dwError); } else { ClRtlLogPrint(LOG_NOISE, "[DM] DmPrepareQuorumResChange - Exit, status=0x%1!08lx!\r\n", dwError); } return(dwError); } // DmPrepareQuorumResChange /**** @func void | DmDwitchToNewQuorumLog| This is called to switch to a new quorum log when the quorum resource is changed. @comm When a quorum resource is successfully changed, this function is to switch quorum logs. The synchronous notifications for the old resource are unhooked and those for the new resource file are hooked. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ void DmSwitchToNewQuorumLog( IN LPCWSTR lpszQuoLogPath, IN DWORD dwNewQuorumResourceCharacteristics) { WCHAR szTmpQuoTombStone[MAX_PATH]; DWORD dwError = ERROR_SUCCESS; ClRtlLogPrint(LOG_NOISE, "[DM] DmSwitchQuorumLogs - Entry\r\n"); //unhook notifications with the old quorum resource DmpUnhookQuorumNotify(); //ask the dm to register with the new quorum resource DmpHookQuorumNotify(); //if the new log file exists... this is the owner of the new quorum resource. //the new log file may be the same as the old one if (ghNewQuoLog) { if (ghQuoLog && (ghQuoLog != ghNewQuoLog)) { LogClose(ghQuoLog); //take another checkpoint to the new quorum file, //so that the last few updates make into it if ((dwError = LogCheckPoint(ghNewQuoLog, TRUE, NULL, 0)) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[DM] DmSwitchQuorumLogs - Failed to take a checkpoint\r\n"); CL_UNEXPECTED_ERROR(dwError); } ClRtlLogPrint(LOG_NOISE, "[DM] DmSwitchQuorumLogs - taken checkpoint\r\n"); ghQuoLog = NULL; } ghQuoLog = ghNewQuoLog; ghNewQuoLog = NULL; // if the old tombstome was replace by a tmp file at the beginning //of change quorum resource delete it now //get the tmp file for the new quorum resource lstrcpyW(szTmpQuoTombStone, lpszQuoLogPath); lstrcatW(szTmpQuoTombStone, cszTmpQuoTombStoneFile); QfsDeleteFile(szTmpQuoTombStone); } else { //if the old log file is open, owner of the old quorum resource if (ghQuoLog) { LogClose(ghQuoLog); ghQuoLog = NULL; } } if (FmDoesQuorumAllowLogging(dwNewQuorumResourceCharacteristics) != ERROR_SUCCESS) { //this is not enough to ensure the dm logging will cease //the ghQuoLog parameter must be NULL CsNoQuorumLogging = TRUE; if (ghQuoLog) { LogClose(ghQuoLog); ghQuoLog = NULL; } } else if ( !CsUserTurnedOffQuorumLogging ) { // // If the user did not turn off quorum logging explicitly, then turn it back on since // the new quorum resource is not local quorum. // CsNoQuorumLogging = FALSE; } ClRtlLogPrint(LOG_NOISE, "[DM] DmSwitchQuorumLogs - Exit!\r\n"); return; } /**** @func DWORD | DmReinstallTombStone| If the change to a new quorum resource fails, the new log is closed and the tombstone is reinstalled. @parm IN LPCWSTR | lpszQuoLogPath | The path for maintenance cluster files. @comm The old quorum log file is deleted and a tomstone file is created in its place. If this tombstone file is detected in the quorum path, the node is not allowed to do a form. It must do a join to find about the new quorum resource from the node that knows about the most recent quorum resource. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD DmReinstallTombStone( IN LPCWSTR lpszQuoLogPath ) { DWORD dwError=ERROR_SUCCESS; WCHAR szQuoTombStone[MAX_PATH]; WCHAR szTmpQuoTombStone[MAX_PATH]; ClRtlLogPrint(LOG_NOISE, "[DM] DmReinstallTombStone - Entry\r\n"); if (ghNewQuoLog) { //get the tmp file for the new quorum resource lstrcpyW(szTmpQuoTombStone, lpszQuoLogPath); lstrcatW(szTmpQuoTombStone, cszTmpQuoTombStoneFile); //create the tombstone file or replace the previous one with a new one lstrcpyW(szQuoTombStone, lpszQuoLogPath); lstrcatW(szQuoTombStone, cszQuoTombStoneFile); //restore the tombstone if (!QfsMoveFileEx(szTmpQuoTombStone, szQuoTombStone, MOVEFILE_REPLACE_EXISTING|MOVEFILE_WRITE_THROUGH)) { //this may fail if the tombstone doesnt exist, ignore error ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmReinstallTombStone :Warning-MoveFileExW failed, error=0x%1!08lx!\r\n", GetLastError()); } // if this is not the same as the old log file, close it if (ghNewQuoLog != ghQuoLog) { LogClose(ghNewQuoLog); } ghNewQuoLog = NULL; } return(dwError); } /**** @func DWORD | DmCompleteQuorumResChange| This is called on the quorum resource if the old quorum log file is not the same as the new one. @parm IN PVOID | pOldQuoRes | The new quorum resource. @parm IN LPCWSTR | lpszPath | The path for temporary cluster files. @parm IN DWORD | dwMaxQuoLogSize | The maximum size limit for the quorum log file. @comm The old quorum log file is deleted and a tomstone file is created in its place. If this tombstone file is detected in the quorum path, the node is not allowed to do a form. It must do a join to find about the new quorum resource from the node that knows about the most recent quorum resource. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD DmCompleteQuorumResChange( IN LPCWSTR lpszOldQuoResId, IN LPCWSTR lpszOldQuoLogPath ) { DWORD dwError=ERROR_SUCCESS; WCHAR szOldQuoFileName[MAX_PATH]; QfsHANDLE hTombStoneFile; WCHAR szQuorumTombStone[MAX_PATH]; PQUO_TOMBSTONE pTombStone = NULL; DWORD dwBytesWritten; WIN32_FIND_DATA FindData; QfsHANDLE hSrchTmpFiles; ClRtlLogPrint(LOG_NOISE, "[DM] DmCompleteQuorumResChange - Entry\r\n"); //the old log file name lstrcpyW(szOldQuoFileName, lpszOldQuoLogPath); lstrcatW(szOldQuoFileName, cszQuoFileName); //create the tombstone file or replace the previous one with a new one lstrcpyW(szQuorumTombStone, lpszOldQuoLogPath); lstrcatW(szQuorumTombStone, cszQuoTombStoneFile); pTombStone = LocalAlloc(LMEM_FIXED, sizeof(QUO_TOMBSTONE)); if (!pTombStone) { CL_LOGFAILURE(ERROR_NOT_ENOUGH_MEMORY); CsLogEvent(LOG_UNUSUAL, DM_TOMBSTONECREATE_FAILED); goto DelOldFiles; } hTombStoneFile = QfsCreateFile(szQuorumTombStone, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, CREATE_ALWAYS, 0, NULL); if (!QfsIsHandleValid(hTombStoneFile) ) { //dont return failure CL_LOGFAILURE(dwError); CsLogEvent(LOG_UNUSUAL, DM_TOMBSTONECREATE_FAILED); goto DelOldFiles; } //write the old quorum path to it. lstrcpyn(pTombStone->szOldQuoResId, lpszOldQuoResId, MAXSIZE_RESOURCEID); lstrcpy(pTombStone->szOldQuoLogPath, lpszOldQuoLogPath); //write the tombstones if (! QfsWriteFile(hTombStoneFile, pTombStone, sizeof(QUO_TOMBSTONE), &dwBytesWritten, NULL)) { CL_LOGFAILURE(GetLastError()); CsLogEvent(LOG_UNUSUAL, DM_TOMBSTONECREATE_FAILED); goto DelOldFiles; } CL_ASSERT(dwBytesWritten == sizeof(QUO_TOMBSTONE)); ClRtlLogPrint(LOG_NOISE, "[DM] DmCompleteQuorumResChange: tombstones written\r\n"); DelOldFiles: // //delete the old quorum files // if (!QfsDeleteFile(szOldQuoFileName)) CL_LOGFAILURE(GetLastError()); //delele other tmp files in there lstrcpyW(szOldQuoFileName, lpszOldQuoLogPath); lstrcatW(szOldQuoFileName, L"*.tmp"); hSrchTmpFiles = QfsFindFirstFile(szOldQuoFileName, & FindData); if (QfsIsHandleValid(hSrchTmpFiles)) { lstrcpyW(szQuorumTombStone, lpszOldQuoLogPath); lstrcatW(szQuorumTombStone, FindData.cFileName); QfsDeleteFile(szQuorumTombStone); while (QfsFindNextFile( hSrchTmpFiles, & FindData)) { lstrcpyW(szQuorumTombStone, lpszOldQuoLogPath); lstrcatW(szQuorumTombStone, FindData.cFileName); QfsDeleteFile(szQuorumTombStone); } QfsFindClose(hSrchTmpFiles); } // // Clean up the old registry checkpoint files // CpCompleteQuorumChange(lpszOldQuoLogPath); QfsCloseHandleIfValid(hTombStoneFile); if (pTombStone) LocalFree(pTombStone); return(dwError); } /**** @func DWORD | DmWriteToQuorumLog| When a transaction to the cluster database is completed successfully, this function is invoked. @parm DWORD | dwSequence | The sequnece number of the transaction. @parm PVOID | pData | A pointer to a record data. @parm DWORD | dwSize | The size of the record data in bytes. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD WINAPI DmWriteToQuorumLog( IN DWORD dwGumDispatch, IN DWORD dwSequence, IN DWORD dwType, IN PVOID pData, IN DWORD dwSize) { DWORD dwError=ERROR_SUCCESS; //dmupdate is coming before the DmUpdateJoinCluster is called. //at this point we are not the owner of quorum in any case if (!gpQuoResource) goto FnExit; ClRtlLogPrint(LOG_NOISE, "[DM] DmWriteToQuorumLog Entry Seq#=%1!u! Type=%2!u! Size=%3!u!\r\n", dwSequence, dwType, dwSize); // // Chittur Subbaraman (chitturs) - 6/3/99 // // Make sure the gLockDmpRoot is held before LogCheckPoint is called // so as to maintain the ordering between this lock and the log lock. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); //if I am the owner of the quorum logs, just write the record if (gbIsQuoLoggingOn && ghQuoLog && gbIsQuoResOnline && AMIOWNEROFQUORES(gpQuoResource)) { if (dwGumDispatch == PRE_GUM_DISPATCH) { //make sure the logger has enough space to commit this else //refuse this GUM transaction dwError = LogCommitSize(ghQuoLog, RMRegistryMgr, dwSize); if (dwError != ERROR_SUCCESS) { if (dwError == ERROR_CLUSTERLOG_NOT_ENOUGH_SPACE) { //map error CL_LOGCLUSERROR(LM_DISKSPACE_LOW_WATERMARK); gbIsQuoResEnoughSpace = FALSE; } } else { if (!gbIsQuoResEnoughSpace) gbIsQuoResEnoughSpace = TRUE; } } else if (dwGumDispatch == POST_GUM_DISPATCH) { if (LogWrite(ghQuoLog, dwSequence, TTCompleteXsaction, RMRegistryMgr, dwType, pData, dwSize) == NULL_LSN) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmWriteToQuorumLog failed, error=0x%1!08lx!\r\n", dwError); } } } RELEASE_LOCK(gLockDmpRoot); FnExit: return (dwError); } /**** @func DWORD | DmpChkQuoTombStone| This checks the quorum logs to ensure that it is the most recent one before rolling in the changes. @rdesc Returns a result code. ERROR_SUCCESS on success. @comm This looks for the tombstone file and if one exists. It checks if this quorum file is marked as dead in there. @xref ****/ DWORD DmpChkQuoTombStone() { DWORD dwError=ERROR_SUCCESS; WCHAR szQuorumLogPath[MAX_PATH]; WCHAR szQuorumTombStone[MAX_PATH]; QfsHANDLE hTombStoneFile = QfsINVALID_HANDLE_VALUE; PQUO_TOMBSTONE pTombStone = NULL; DWORD dwBytesRead; ClRtlLogPrint(LOG_NOISE, "[DM] DmpChkQuoTombStone - Entry\r\n"); dwError = DmGetQuorumLogPath(szQuorumLogPath, sizeof(szQuorumLogPath)); if (dwError) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone - DmGetQuorumLogPath failed,error=0x%1!08lx!\n", dwError); goto FnExit; } lstrcpyW(szQuorumTombStone, szQuorumLogPath); lstrcatW(szQuorumTombStone, L"\\quotomb.stn"); pTombStone = LocalAlloc(LMEM_FIXED, sizeof(QUO_TOMBSTONE)); if (!pTombStone) { dwError = ERROR_NOT_ENOUGH_MEMORY; goto FnExit; } hTombStoneFile = QfsCreateFile(szQuorumTombStone, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ|FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); if (!QfsIsHandleValid(hTombStoneFile) ) { //there is no tombstone file, not a problem-we can proceed with the form goto FnExit; } //found a tombstone file //read the file if (! QfsReadFile(hTombStoneFile, pTombStone, sizeof(QUO_TOMBSTONE), &dwBytesRead, NULL)) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone - Couldn't read the tombstone,error=0x%1!08lx!\n", dwError); //dont return an error, we can proceed with form?? goto FnExit; } if (dwBytesRead != sizeof(QUO_TOMBSTONE)) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone - Couldn't read the entire tombstone\r\n"); //dont return an error, we can proceed with form?? goto FnExit; } if ((!lstrcmpW(OmObjectId(gpQuoResource), pTombStone->szOldQuoResId)) && (!lstrcmpiW(szQuorumLogPath, pTombStone->szOldQuoLogPath))) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone:A tombstone for this resource, and quorum log file was found here.\r\n"); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone:This is node is only allowed to do a join, make sure another node forms\r\n"); //log something into the eventlog CL_LOGCLUSERROR(SERVICE_MUST_JOIN); //we exit with succes because this is by design and we dont want //clusprxy to retry starting unnecessarily ExitProcess(dwError); goto FnExit; } else { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpChkQuoTombStone: Bogus TombStone ??\r\n"); #if DBG if (IsDebuggerPresent()) DebugBreak(); #endif goto FnExit; } FnExit: QfsCloseHandleIfValid(hTombStoneFile); if (pTombStone) LocalFree(pTombStone); ClRtlLogPrint(LOG_NOISE, "[DM] DmpChkQuoTombStone: Exit, returning 0x%1!08lx!\r\n", dwError); return(dwError); } /**** @func DWORD | DmpApplyChanges| When dm is notified that the cluster form is occuring, it calls DmpApplyChanges to apply the quorum logs to the cluster database. @rdesc Returns a result code. ERROR_SUCCESS on success. @comm This opens the quorum file. Note that it doesnt close the quorum file. @xref ****/ DWORD DmpApplyChanges() { LSN FirstLsn; DWORD dwErr = ERROR_SUCCESS; DWORD dwSequence; DM_LOGSCAN_CONTEXT DmAppliedChangeContext; if (ghQuoLog == NULL) { return(ERROR_QUORUMLOG_OPEN_FAILED); } //find the current sequence number from the registry dwSequence = DmpGetRegistrySequence(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpApplyChanges: The current registry sequence number %1!d!\r\n", dwSequence); // upload a database if the current sequence number is lower or equal to // the one in the database OR if the user is forcing a restore database // operation. // find the lsn of the record from which we need to start applying changes // if null there are no changes to apply dwErr = DmpLogFindStartLsn(ghQuoLog, &FirstLsn, &dwSequence); if (dwErr != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpApplyChanges: DmpLogFindStartLsn failed, error=0x%1!08lx!\r\n", dwErr); goto FnExit; } //dwSequence now contains the current sequence number in the registry DmAppliedChangeContext.dwSequence = dwSequence; if (FirstLsn != NULL_LSN) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpApplyChanges: The LSN of the record to apply changes from 0x%1!08lx!\r\n", FirstLsn); if (dwErr = LogScan(ghQuoLog, FirstLsn, TRUE,(PLOG_SCAN_CALLBACK)DmpLogApplyChangesCb, &DmAppliedChangeContext) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpApplyChanges: LogScan failed, error=0x%1!08lx!\r\n", dwErr); } //if the more changes have been applied if (DmAppliedChangeContext.dwSequence != dwSequence) { //set the gum sequence number to the trid that has been applied GumSetCurrentSequence(GumUpdateRegistry, DmAppliedChangeContext.dwSequence); //update the registry with this sequence number DmpUpdateSequence(); //set the gum sequence number to one higher for the next transaction GumSetCurrentSequence(GumUpdateRegistry, (DmAppliedChangeContext.dwSequence + 1)); ClRtlLogPrint(LOG_NOISE, "[DM] DmpApplyChanges: Gum sequnce number set to = %1!d!\r\n", (DmAppliedChangeContext.dwSequence + 1)); } } FnExit: ClRtlLogPrint(LOG_NOISE, "[DM] DmpApplyChanges: Exit, returning 0x%1!08lx!\r\n", dwErr); return(dwErr); } /**** @func DWORD | DmpFindStartLsn| Uploads the last checkpoint from the quorum and returns the LSN of the record from which the changes should be applied. @parm IN HLOG | hQuoLog | the log file handle. @parm OUT LSN *| pStartScanLsn | Returns the LSN of the record in the quorum log from which changes must be applied is returned here. NULL_LSN is returned if no changes need to be applied. @parm IN OUT LPDWORD | *pdwSequence | Should be set to the current sequence number is the cluster registry. If a new chkpoint is uploaded, the sequence number corresponding to that is returned. @rdesc Returns ERROR_SUCCESS if a valid LSN is returned. This may be NULL_LSN. Returns the error code if the database cannot be uploaded from the last chkpoint or if something horrible happens. @comm This finds the last valid check point in the log file. The data base is synced with this checkpoint and the gum sequence number is set to one plus the sequence number of that checkpoint. If no checkpoint record is found, a checkpoint is taken and NULL_LSN is returned. @xref ****/ DWORD DmpLogFindStartLsn( IN HLOG hQuoLog, OUT LSN *pStartScanLsn, IN OUT LPDWORD pdwSequence) { LSN ChkPtLsn; LSN StartScanLsn; DWORD dwChkPtSequence=0; DWORD dwError = ERROR_SUCCESS; WCHAR szChkPtFileName[LOG_MAX_FILENAME_LENGTH]; DM_LOGSCAN_CONTEXT DmAppliedChangeContext; *pStartScanLsn = NULL_LSN; ChkPtLsn = NULL_LSN; //read the last check point record if any and the transaction id till that //checkpoint dwError = LogGetLastChkPoint(hQuoLog, szChkPtFileName, &dwChkPtSequence, &ChkPtLsn); if (dwError != ERROR_SUCCESS) { //no chk point record found ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogFindStartLsn: LogGetLastChkPoint failed, error=0x%1!08lx!\r\n", dwError ); // this can happen either due to the fact that the log file was just created, // and hence there is no checkpoint or because log file was messed up // and the mount process corrected it but removed the checkpoint. // If it is the second case, then logpmountlog should put something in the // event log if (dwError == ERROR_CLUSTERLOG_CHKPOINT_NOT_FOUND) { // // Chittur Subbaraman (chitturs) - 6/3/99 // // Make sure the gLockDmpRoot is held before LogCheckPoint is called // so as to maintain the ordering between this lock and the log lock. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); //take a checkpoint, so that this doesnt happen the next time dwError = LogCheckPoint(hQuoLog, TRUE, NULL, 0); RELEASE_LOCK(gLockDmpRoot); if (dwError != ERROR_SUCCESS) { //check point could not be taken ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogFindStartLsn: Checkpoint on first form failed, error=0x%1!08lx!\r\n", dwError ); goto FnExit; } } else { //there were other errors goto FnExit; } } else { //found check point record ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: LogGetLastChkPt rets, Seq#=%1!d! ChkPtLsn=0x%2!08lx!\r\n", dwChkPtSequence, ChkPtLsn); // // Chittur Subbaraman (chitturs) - 10/18/98 // // If the user is forcing a database restore from backup, then // do not check whether the current sequence number in the registry // is younger than the checkpoint sequence number in the quorum log. // Just, go ahead and load the checkpoint from restored database. // if ( CsDatabaseRestore == TRUE ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: User forcing a chkpt upload from quorum log...\r\n"); } else { //if the sequence number is greater than the check point sequence number //plus one, that implies..that only changes from that sequence number //need to be applied.(this node may not have been the first one to die) //We dont always apply the database because if logging is mostly off //and the two nodes die simultaneosly we want to prevent losing all the //changes //else if the checkpoint sequence is one below the current //current sequence number, then the locker node could have died after updating //get the current checkpoint irrespective of what the current sequence number is //this is because a checkpoint with the same sequence number may have //a change that is different from whats there in the current registry. //if node 'a'(locker and logger dies in the middle of logging trid=x+1, //the other node,'b' will take over logging and checkpoint the database //at trid=x. If 'a' comes back up, it needs to throw aways its x+1 change //and apply changes from the log from chk pt x. if (*pdwSequence > (dwChkPtSequence + 1)) { //the current sequence number is less than or equal to chkpt Seq + 1 ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: ChkPt not applied, search for next seq\r\n"); DmAppliedChangeContext.dwSequence = *pdwSequence; DmAppliedChangeContext.StartLsn = NULL_LSN; //find the LSN from which to apply changes if (dwError = LogScan(ghQuoLog, ChkPtLsn, TRUE,(PLOG_SCAN_CALLBACK)DmpLogFindStartLsnCb, &DmAppliedChangeContext) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogFindStartLsn: LogScan failed, no changes will be applied, error=0x%1!08lx!\r\n", dwError); goto FnExit; } *pStartScanLsn = DmAppliedChangeContext.StartLsn; goto FnExit; } } // // The current registry sequence number is less than or equal // to chkpt Seq + 1 OR the user is forcing a database restore // from the backup area. // ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: Uploading chkpt from quorum log\r\n"); //make sure that no keys are added to the key list because of opens/creates ACQUIRE_EXCLUSIVE_LOCK(gLockDmpRoot); //hold the key lock as well EnterCriticalSection(&KeyLock); //invalidate all open keys DmpInvalidateKeys(); if ((dwError = DmInstallDatabase(szChkPtFileName, NULL, FALSE)) != ERROR_SUCCESS) { //couldnt install the database //bad ! ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogFindStartLsn: DmpInstallDatabase failed, error=0x%1!08lx!\r\n", dwError); CsLogEventData( LOG_CRITICAL, DM_CHKPOINT_UPLOADFAILED, sizeof(dwError), &dwError ); DmpReopenKeys(); //release the locks LeaveCriticalSection(&KeyLock); RELEASE_LOCK(gLockDmpRoot); goto FnExit; } else { //the current sequence number is less than or equal to chkpt Seq + 1 ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: chkpt uploaded from quorum log\r\n"); //since we downloaded the database, we should start //aplying changes from ChkPtLsn *pStartScanLsn = ChkPtLsn; *pdwSequence = dwChkPtSequence; //set the gum sequence number to be the next one //ss: the next logged transaction shouldnt have the same //transaction id GumSetCurrentSequence(GumUpdateRegistry, (dwChkPtSequence+1)); //reopen the keys DmpReopenKeys(); //release the locks LeaveCriticalSection(&KeyLock); RELEASE_LOCK(gLockDmpRoot); goto FnExit; } } FnExit: ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogFindStartLsn: LSN=0x%1!08lx!, returning 0x%2!08lx!\r\n", *pStartScanLsn, dwError); return(dwError); } /**** @func DWORD | DmpLogFindStartLsnCb| The callback tries to find the first record with a transaction id that is larger than the sequence number of the local database. @parm PVOID | pContext| A pointer to a DM_STARTLSN_CONTEXT structure. @parm LSN | Lsn| The LSN of the record. @parm RMID | Resource | The resource manager for this transaction. @parm RMID | ResourceType | The resource manager for this transaction. @parm TRID | Transaction | The transaction number of this record. @parm PVOID | pLogData | The log data for this record. @parm DWORD | DataLength | The length of the record. @rdesc Returns TRUE to continue scan. FALSE to stop. @comm This function returns true if the sequence number of the record being scanned is higher than the seqence number passed in the context. @xref ****/ BOOL WINAPI DmpLogFindStartLsnCb( IN PVOID pContext, IN LSN Lsn, IN RMID Resource, IN RMTYPE ResourceFlags, IN TRID Transaction, IN TRTYPE TrType, IN const PVOID pLogData, IN DWORD DataLength) { PDM_LOGSCAN_CONTEXT pDmStartLsnContext= (PDM_LOGSCAN_CONTEXT) pContext; CL_ASSERT(pDmStartLsnContext); if (Transaction > (int)pDmStartLsnContext->dwSequence) { pDmStartLsnContext->StartLsn = Lsn; return (FALSE); } return(TRUE); } /**** @func DWORD | DmpHookQuorumNotify| This hooks a callback to be invoked whenever the state of the quorum resource changes. @rdesc Returns a result code. ERROR_SUCCESS on success. @comm This is used to monitor the state of @xref ****/ DWORD DmpHookQuorumNotify() { DWORD dwError = ERROR_SUCCESS; if (dwError = FmFindQuorumResource(&gpQuoResource)) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmUpdateFormNewCluster: FmFindQuorumResource failed, error=0x%1!08lx!\r\n", dwError); goto FnExit; } dwError = OmRegisterNotify(gpQuoResource, NULL, NOTIFY_RESOURCE_POSTONLINE| NOTIFY_RESOURCE_PREOFFLINE | NOTIFY_RESOURCE_OFFLINEPENDING | NOTIFY_RESOURCE_POSTOFFLINE | NOTIFY_RESOURCE_FAILED, DmpQuoObjNotifyCb); FnExit: return(dwError); } /**** @func DWORD | DmpUnhookQuorumNotify| This unhooks the callback function that is registered with the object. @parm PVOID | pContext| A pointer to a DMLOGRECORD structure. @parm PVOID | pObject| A pointer to quorum resource object. @parm DWORD | dwNotification| A pointer to a DMLOGRECORD structure. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ DWORD DmpUnhookQuorumNotify() { DWORD dwError = ERROR_SUCCESS; if (gpQuoResource) { dwError = OmDeregisterNotify(gpQuoResource, DmpQuoObjNotifyCb); OmDereferenceObject(gpQuoResource); } return(ERROR_SUCCESS); } /**** @func DWORD | DmpQuoObjNotifyCb| This is a callback that is called on change of state on quorum resource. @parm PVOID | pContext| A pointer to a DMLOGRECORD structure. @parm PVOID | pObject| A pointer to quorum resource object. @parm DWORD | dwNotification| A pointer to a DMLOGRECORD structure. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ void DmpQuoObjNotifyCb( IN PVOID pContext, IN PVOID pObject, IN DWORD dwNotification) { switch(dwNotification) { case NOTIFY_RESOURCE_POSTONLINE: gbIsQuoResOnline = TRUE; ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Quorum resource is online\r\n"); //if this is the owner of the quorum resource //and the log is not open, open the log if (AMIOWNEROFQUORES(gpQuoResource) && !CsNoQuorumLogging) { //ToDo: the quorum file name should be obtained from the setup //for now obtain the value from the cluster registry. WCHAR szQuorumFileName[MAX_PATH]; LSN FirstLsn; DWORD dwError; DWORD dwType; DWORD dwLength; DWORD dwMaxQuoLogSize; DWORD bForceReset = FALSE; ULONG OldHardErrorValue; //bug# :106647 //SS: HACKHACK disabling hard error pop ups so that disk corruption //is caught somewhere else.. //atleast the pop-ups must be disabled for the whole process ! //me thinks this is covering up the problem of disk corruption //disk corruption should not occur! RtlSetThreadErrorMode(RTL_ERRORMODE_FAILCRITICALERRORS, &OldHardErrorValue); ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Own quorum resource, try open the quorum log\r\n"); if (DmGetQuorumLogPath(szQuorumFileName, sizeof(szQuorumFileName)) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Quorum log file is not configured\r\n"); } else { BOOL fSetSecurity = FALSE; QfsHANDLE hFindFile = QfsINVALID_HANDLE_VALUE; WIN32_FIND_DATA FindData; hFindFile = QfsFindFirstFile( szQuorumFileName, &FindData ); if ( !QfsIsHandleValid(hFindFile) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: FindFirstFile on path %1!ws! failed, Error=%2!d! !!!\n", szQuorumFileName, dwError); if ( dwError == ERROR_PATH_NOT_FOUND ) { fSetSecurity = TRUE; } } else { QfsFindClose( hFindFile ); } //if the directory doesnt exist create it dwError = QfsClRtlCreateDirectory(szQuorumFileName); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[DM] DmpQuoObjNotifyCb: Failed to create directory %1!ws!, error=0x%2!08lx!...\n", szQuorumFileName, dwError); CL_UNEXPECTED_ERROR(dwError); CsInconsistencyHalt(dwError); } if ( fSetSecurity == TRUE ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Attempting to set security on directory %1!ws!...\n", szQuorumFileName); dwError = QfsSetFileSecurityInfo( szQuorumFileName, GENERIC_ALL, // for Admins GENERIC_ALL, // for Owner 0 ); // for Everyone if ( dwError != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_CRITICAL, "[DM] DmpQuoObjNotifyCb: ClRtlSetObjSecurityInfo failed for file %1!ws!, Status=%2!u!\r\n", szQuorumFileName, dwError); CL_LOGFAILURE( dwError ); CsInconsistencyHalt( dwError ); } } DmGetQuorumLogMaxSize(&dwMaxQuoLogSize); // If the resource monitor dies and comes back up, this can happen if (ghQuoLog != NULL) { HLOG hQuoLog; // // Make sure the ghQuoLog variable is NULLed out with lock held exclusively BEFORE the log // is closed. This will prevent race cases in which another thread reads the ghQuoLog variable and // assumes blindly that the log is open. // ACQUIRE_EXCLUSIVE_LOCK( gLockDmpRoot ); hQuoLog = ghQuoLog; ghQuoLog = NULL; RELEASE_LOCK( gLockDmpRoot ); LogClose( hQuoLog ); } if (gbIsQuoLoggingOn) gbNeedToCheckPoint = TRUE; // // Chittur Subbaraman (chitturs) - 10/16/98 // // Check whether you need to restore the database from a // user-supplied backup directory to the quorum disk. This // restore operation is done only once when the Dm has // not been fully initialized. Note that this function // is called whenever the state of the quorum resource // changes but the restore operation is only done once. // if ( ( gbDmInited == FALSE ) && ( CsDatabaseRestore == TRUE ) ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Beginning DB restoration from %1!ws!...\r\n", CsDatabaseRestorePath); if ( ( dwError = DmpRestoreClusterDatabase ( szQuorumFileName ) ) != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpQuoObjNotifyCb: DB restore operation from %1!ws! failed! Error=0x%2!08lx!\r\n", CsDatabaseRestorePath, dwError); CL_LOGFAILURE( dwError ); CsDatabaseRestore = FALSE; CsInconsistencyHalt( dwError ); } ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: DB restoration from %1!ws! successful...\r\n", CsDatabaseRestorePath); CL_LOGCLUSINFO( SERVICE_CLUSTER_DATABASE_RESTORE_SUCCESSFUL ); } lstrcat(szQuorumFileName, cszQuoFileName); ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: the name of the quorum file is %1!ls!\r\n", szQuorumFileName); // // Chittur Subbaraman (chitturs) - 12/4/99 // // If the quorum log file is found to be missing or corrupt, // reset it only under the following conditions, else // fail the log creation and halt the node. // // (1) A freshly formed cluster, // (2) The user has chosen to reset the log since the user // does not have a backup. // (3) After the quorum resource has successfully come // online on this node and the DM has been initialized // successfully. This is because the sanity of the // quorum log file has already been verified at // initialization and the chances of the quorum log // missing or getting corrputed after that are not // so high (due to it being held open by the cluster // service) and so it is not worth halting the node // during run-time. // if ((CsFirstRun && !CsUpgrade) || (CsResetQuorumLog) || (gbDmInited == TRUE)) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Will try to reset Quorum log if file not found or if corrupt\r\n"); bForceReset = TRUE; } // open the log file ghQuoLog = LogCreate(szQuorumFileName, dwMaxQuoLogSize, (PLOG_GETCHECKPOINT_CALLBACK)DmpGetSnapShotCb, NULL, bForceReset, &FirstLsn); if (!ghQuoLog) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpQuoObjNotifyCb: Quorum log could not be opened, error = 0x%1!08lx!\r\n", dwError); CL_LOGFAILURE(dwError); CsInconsistencyHalt(ERROR_QUORUMLOG_OPEN_FAILED); } else { ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Quorum log opened\r\n"); } if (gbNeedToCheckPoint && ghQuoLog) { //take a checkpoint and set the flag to FALSE. gbNeedToCheckPoint = FALSE; //get a checkpoint database ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb - taking a checkpoint\r\n"); // // Chittur Subbaraman (chitturs) - 6/3/99 // // Make sure the gLockDmpRoot is held before LogCheckPoint is called // so as to maintain the ordering between this lock and the log lock. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); dwError = LogCheckPoint(ghQuoLog, TRUE, NULL, 0); RELEASE_LOCK(gLockDmpRoot); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[DM] DmpEventHandler - Failed to take a checkpoint in the log file, error = 0x%1!08lx!\r\n", dwError); CL_UNEXPECTED_ERROR(dwError); CsInconsistencyHalt(dwError); } } //if the checkpoint timer doesnt already exist //check if the timer has already been created - we might // get two post online notifications //and dont cause a timer leak if (!ghCheckpointTimer) { ghCheckpointTimer = CreateWaitableTimer(NULL, FALSE, NULL); if (!ghCheckpointTimer) { CL_UNEXPECTED_ERROR(dwError = GetLastError()); } else { DWORD dwCheckpointInterval; dwError = DmpGetCheckpointInterval(&dwCheckpointInterval); CL_ASSERT(dwError == ERROR_SUCCESS); //add a timer to take periodic checkpoints AddTimerActivity(ghCheckpointTimer, dwCheckpointInterval, 1, DmpCheckpointTimerCb, &ghQuoLog); } } } //SS:completion of hack, revert to enabling pop-ups RtlSetThreadErrorMode(OldHardErrorValue, NULL); } if (ghQuoLogOpenEvent) { //this is the first notification after the form //allow the initialization to continue after rolling //back the changes SetEvent(ghQuoLogOpenEvent); } break; case NOTIFY_RESOURCE_FAILED: case NOTIFY_RESOURCE_PREOFFLINE: case NOTIFY_RESOURCE_OFFLINEPENDING: ClRtlLogPrint(LOG_NOISE, "[DM] DmpQuoObjNotifyCb: Quorum resource offline/offlinepending/preoffline\r\n"); gbIsQuoResOnline = FALSE; if (ghQuoLog) { HLOG hQuoLog; //stop the checkpoint timer if (ghCheckpointTimer) { RemoveTimerActivity(ghCheckpointTimer); ghCheckpointTimer = NULL; } // // Make sure the ghQuoLog variable is NULLed out with lock held exclusively BEFORE the log // is closed. This will prevent race cases in which another thread reads the ghQuoLog variable and // assumes blindly that the log is open. // ACQUIRE_EXCLUSIVE_LOCK( gLockDmpRoot ); hQuoLog = ghQuoLog; ghQuoLog = NULL; RELEASE_LOCK( gLockDmpRoot ); LogClose( hQuoLog ); //dont try and log after this gbIsQuoLoggingOn = FALSE; } if (ghQuoLogOpenEvent) { //this is the first notification after the form //allow the initialization to continue after rolling //back the changes SetEvent(ghQuoLogOpenEvent); } break; } } /**** @func DWORD | DmpHookEventHandler| This hooks a callback to be invoked whenever the state of the quorum resource changes. @rdesc Returns a result code. ERROR_SUCCESS on success. @comm This is used to monitor the state of nodes and turn quorum logging on or off. @xref ****/ DWORD DmpHookEventHandler() { DWORD dwError; dwError = EpRegisterEventHandler(CLUSTER_EVENT_ALL,DmpEventHandler); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmHookEventHandler: EpRegisterEventHandler failed, error=0x%1!08lx!\r\n", dwError); CL_UNEXPECTED_ERROR( dwError ); } return(dwError); } /**** @func DWORD | DmpEventHandler| This routine handles events for the Cluster Database Manager. @parm CLUSTER_EVENT | Event | The event to be processed. Only one event at a time. If the event is not handled, return ERROR_SUCCESS. @parm PVOID| pContext | A pointer to context associated with the particular event. @rdesc Returns ERROR_SUCCESS else a Win32 error code on other errors. @comm This is used to monitor the state of nodes and turn quorum logging on or off. @xref ****/ DWORD WINAPI DmpEventHandler( IN CLUSTER_EVENT Event, IN PVOID pContext ) { DWORD dwError=ERROR_SUCCESS; BOOL bAreAllNodesUp; switch ( Event ) { case CLUSTER_EVENT_NODE_UP: bAreAllNodesUp = TRUE; if ((dwError = OmEnumObjects(ObjectTypeNode, DmpNodeObjEnumCb, &bAreAllNodesUp, NULL)) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM]DmpEventHandler : OmEnumObjects returned, error=0x%1!08lx!\r\n", dwError); } else { if (bAreAllNodesUp) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpEventHandler - node is up, turning quorum logging off\r\n"); gbIsQuoLoggingOn = FALSE; } } break; case CLUSTER_EVENT_NODE_DOWN: if (!gbIsQuoLoggingOn) { HANDLE hThread = NULL; DWORD dwThreadId; // // Chittur Subbaraman (chitturs) - 7/23/99 // // Create a new thread to handle the checkpointing on a // node down. This is necessary since we don't want the // DM node down handler to be blocked in any fashion. If // it is blocked since FmCheckQuorumState couldn't get the // quorum group lock and some other thread got the group // lock and is waiting for the GUM lock, then we have // an immediate deadlock. Only after this node down // handler finishes, any subsequent future node down // processing can be started. // ClRtlLogPrint(LOG_NOISE, "[DM] DmpEventHandler - Node is down, turn quorum logging on...\r\n"); gbIsQuoLoggingOn = TRUE; ClRtlLogPrint(LOG_NOISE, "[DM] DmpEventHandler - Create thread to handle node down event...\r\n"); hThread = CreateThread( NULL, 0, DmpHandleNodeDownEvent, NULL, 0, &dwThreadId ); if ( hThread == NULL ) { dwError = GetLastError(); ClRtlLogPrint(LOG_CRITICAL, "[DM] DmpEventHandler - Unable to create thread to handle node down event. Error=0x%1!08lx!\r\n", dwError); CsInconsistencyHalt( dwError ); } CloseHandle( hThread ); } break; case CLUSTER_EVENT_NODE_CHANGE: break; case CLUSTER_EVENT_NODE_ADDED: break; case CLUSTER_EVENT_NODE_DELETED: break; case CLUSTER_EVENT_NODE_JOIN: break; } return(dwError); } // DmpEventHandler /**** @func DWORD | DmpNodeObjEnumCb| This is a callback that is called when node objects are enumberate by the dm. @parm PVOID | pContext| A pointer to a DMLOGRECORD structure. @parm PVOID | pObject| A pointer to quorum resource object. @parm DWORD | dwNotification| A pointer to a DMLOGRECORD structure. @rdesc Returns a result code. ERROR_SUCCESS on success. @xref ****/ BOOL DmpNodeObjEnumCb(IN BOOL *pbAreAllNodesUp, IN PVOID pContext2, IN PVOID pNode, IN LPCWSTR szName) { if ((NmGetNodeState(pNode) != ClusterNodeUp) && (NmGetNodeState(pNode) != ClusterNodePaused)) *pbAreAllNodesUp = FALSE; //if any of the nodes is down fall out return(*pbAreAllNodesUp); } /**** @func BOOL | DmpGetSnapShotCb| This callback is invoked when the logger is asked to take a checkpoint record for the cluster registry. @parm PVOID| pContext | The checkpoint context passed into LogCreate. @parm LPWSTR | szChkPtFile | The name of the file in which to take a checkpoint. @parm LPDWORD | pdwChkPtSequence | The sequence number related with this checkpoint is returned in this. @rdesc Returns a result code. ERROR_SUCCESS on success. If the file corresponding to this checkpoint already exists, it will return ERROR_ALREADY_EXISTS and szChkPtFile will be set to the name of the file. @comm LogCheckPoint() calls this function when the log manager is asked to checkpoint the dm database. @xref ****/ DWORD WINAPI DmpGetSnapShotCb(IN LPCWSTR szPathName, IN PVOID pContext, OUT LPWSTR szChkPtFile, OUT LPDWORD pdwChkPtSequence) { DWORD dwError = ERROR_SUCCESS; WCHAR szFilePrefix[MAX_PATH] = L"chkpt"; WCHAR szTempFile[MAX_PATH] = L""; ACQUIRE_SHARED_LOCK( gLockDmpRoot ); szChkPtFile[0] = L'\0'; // // Chittur Subbaraman (chitturs) - 5/1/2000 // // Checkpoint file name is based on registry sequence number. It is possible that two // or more consecutive calls to this function to take checkpoints may read the same // registry sequence number. Thus, if DmGetDatabase fails for some reason, it is possible // that an existing checkpoint file will get corrupted. Thus, even though the quorum log // marks a 'start checkpoint record' and an 'end checkpoint record', it could turn out // to be useless if this function manages to corrupt an existing checkpoint file. To solve // this problem, we first generate a temp file, take a cluster hive snapshot as this temp // file, then atomically move the temp file to the final checkpoint file using the MoveFileEx // function. // // // Create a new unique temp file name // if ( !QfsGetTempFileName( szPathName, szFilePrefix, 0, szTempFile ) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[LM] DmpGetSnapShotCb: Failed to generate a temp file name, PathName=%1!ls!, FilePrefix=%2!ls!, Error=0x%3!08lx!\r\n", szPathName, szFilePrefix, dwError); goto FnExit; } dwError = DmCommitRegistry(); // Ensure up-to-date snapshot if ( dwError != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_CRITICAL, "[LM] DmpGetSnapShotCb: DmCommitRegistry() failed, Error=0x%1!08lx!\r\n", dwError); goto FnExit; } dwError = DmGetDatabase( DmpRoot, szTempFile ); ClRtlLogPrint(LOG_NOISE, "[DM] DmpGetSnapShotCb: DmpGetDatabase returned 0x%1!08lx!\r\n", dwError); if ( dwError == ERROR_SUCCESS ) { *pdwChkPtSequence = DmpGetRegistrySequence(); // // Create a checkpoint file name based on the registry sequence number // if ( !QfsGetTempFileName( szPathName, szFilePrefix, *pdwChkPtSequence, szChkPtFile ) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[LM] DmpGetSnapShotCb: Failed to generate a chkpt file name, PathName=%1!ls!, FilePrefix=%2!ls!, Error=0x%3!08lx!\r\n", szPathName, szFilePrefix, dwError); // // Reset the file name to null, as this information will be used to determine // if the checkpoint was taken // szChkPtFile[0] = L'\0'; goto FnExit; } ClRtlLogPrint(LOG_NOISE, "[LM] DmpGetSnapshotCb: Checkpoint file name=%1!ls! Seq#=%2!d!\r\n", szChkPtFile, *pdwChkPtSequence); if ( !QfsMoveFileEx( szTempFile, szChkPtFile, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH ) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[LM] DmpGetSnapShotCb: Failed to move the temp file to checkpoint file, TempFileName=%1!ls!, ChkPtFileName=%2!ls!, Error=0x%3!08lx!\r\n", szTempFile, szChkPtFile, dwError); // // Reset the file name to null, as this information will be used to determine // if the checkpoint was taken // szChkPtFile[0] = L'\0'; goto FnExit; } } FnExit: RELEASE_LOCK(gLockDmpRoot); if ( dwError != ERROR_SUCCESS ) { QfsDeleteFile( szTempFile ); } return ( dwError ); } /**** @func BOOL WINAPI | DmpLogApplyChangesCb| This callback walks through the records in the quorum logs and applies changes to the local database. @parm PVOID | pContext | The event to be processed. Only one event at a time. If the event is not handled, return ERROR_SUCCESS. @parm LSN | Lsn | Lsn of the record. @parm RMID | Resource | The resource id of the entity that logged this record. @parm RMTYPE | ResourceType | The record type that is specific to the resource id. @parm TRID | Transaction | The sequence number of the transaction. @parm const PVOID | pLogData | A pointer to the record data. @parm DWORD | DataLength | The length of the data in bytes. @rdesc Returns TRUE to continue scan else returns FALSE. @comm This function is called at initialization when a cluster is being formed to apply transactions from the quorum log to the local cluster database. @xref ****/ BOOL WINAPI DmpLogApplyChangesCb( IN PVOID pContext, IN LSN Lsn, IN RMID Resource, IN RMTYPE ResourceType, IN TRID Transaction, IN TRTYPE TransactionType, IN const PVOID pLogData, IN DWORD DataLength) { DWORD Status; PDM_LOGSCAN_CONTEXT pDmAppliedChangeContext = (PDM_LOGSCAN_CONTEXT) pContext; TRSTATE trXsactionState; BOOL bRet = TRUE; CL_ASSERT(pDmAppliedChangeContext); //if the resource id is not the same as dm..ignore..go to the next one switch(TransactionType) { case TTStartXsaction: Status = LogFindXsactionState(ghQuoLog, Lsn, Transaction, &trXsactionState); if (Status != ERROR_SUCCESS) { //there was an error ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogApplyChangesCb ::LogFindXsaction failed, error=0x%1!08lx!\r\n", Status); //assume unknown state CL_LOGFAILURE(Status); trXsactionState = XsactionUnknown; } //if the transaction is successful apply it, else continue if (trXsactionState == XsactionCommitted) { Status = LogScanXsaction(ghQuoLog, Lsn, Transaction, DmpApplyTransactionCb, NULL); if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogApplyChangesCb :LogScanTransaction for committed record failed, error=0x%1!08lx!\r\n", Status); bRet = FALSE; CL_LOGFAILURE(Status); break; } pDmAppliedChangeContext->dwSequence = Transaction; } else { ClRtlLogPrint(LOG_NOISE, "[DM] TransactionState = %1!u!\r\n", trXsactionState); } break; case TTCompleteXsaction: bRet = DmpApplyTransactionCb(NULL, Lsn, Resource, ResourceType, Transaction, pLogData, DataLength); pDmAppliedChangeContext->dwSequence = Transaction; break; default: CL_ASSERT(FALSE); } return(bRet); } BOOL WINAPI DmpApplyTransactionCb( IN PVOID pContext, IN LSN Lsn, IN RMID Resource, IN RMTYPE ResourceType, IN TRID TransactionId, IN const PVOID pLogData, IN DWORD dwDataLength) { DWORD Status; switch(ResourceType) { case DmUpdateCreateKey: ClRtlLogPrint(LOG_NOISE,"[DM] DmpLogScanCb::DmUpdateCreateKey\n"); //SS: we dont care at this point as to where the update originated Status = DmpUpdateCreateKey(FALSE, GET_ARG(pLogData,0), GET_ARG(pLogData,1), GET_ARG(pLogData,2)); break; case DmUpdateDeleteKey: ClRtlLogPrint(LOG_NOISE,"[DM] DmUpdateDeleteKey \n"); Status = DmpUpdateDeleteKey(FALSE, (PDM_DELETE_KEY_UPDATE)((PBYTE)pLogData)); break; case DmUpdateSetValue: ClRtlLogPrint(LOG_NOISE,"[DM] DmUpdateSetValue \n"); Status = DmpUpdateSetValue(FALSE, (PDM_SET_VALUE_UPDATE)((PBYTE)pLogData)); break; case DmUpdateDeleteValue: ClRtlLogPrint(LOG_NOISE,"[DM] DmUpdateDeleteValue\n"); Status = DmpUpdateDeleteValue(FALSE, (PDM_DELETE_VALUE_UPDATE)((PBYTE)pLogData)); break; case DmUpdateJoin: ClRtlLogPrint(LOG_UNUSUAL,"[DM] DmUpdateJoin\n"); Status = ERROR_SUCCESS; break; default: ClRtlLogPrint(LOG_UNUSUAL,"[DM] DmpLogScanCb:uType = %1!u!\r\n", ResourceType); Status = ERROR_INVALID_DATA; CL_UNEXPECTED_ERROR(ERROR_INVALID_DATA); break; } return(TRUE); } /**** @func WORD| DmpLogCheckPtCb| A callback fn for DM to take a checkpoint to the log if the quorum resource is online on this node. @rdesc Returns ERROR_SUCCESS for success, else returns the error code. @comm This callback is called when the quorum resource is online on this node. Since the quorum resource synchronous callbacks are called before the resource state changes are propagated, if the quorum is online the log must be open. @xref ****/ void DmpLogCheckPointCb() { DWORD dwError; // // Chittur Subbaraman (chitturs) - 9/22/99 // // If the quorum logging switch is off, don't do anything. // if (CsNoQuorumLogging) return; //once it is online the log file should be open //SS:BUGS: should we log something in the eventlog if (ghQuoLog) { // // Chittur Subbaraman (chitturs) - 6/3/99 // // Make sure the gLockDmpRoot is held before LogCheckPoint is called // so as to maintain the ordering between this lock and the log lock. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); //get a checkpoint database dwError = LogCheckPoint(ghQuoLog, TRUE, NULL, 0); RELEASE_LOCK(gLockDmpRoot); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[DM] DmpLogCheckPointCb - Failed to take a checkpoint in the log file, error=0x%1!08lx!\r\n", dwError); CL_UNEXPECTED_ERROR(dwError); } ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogCheckPointCb - taken checkpoint\r\n"); } else { CsInconsistencyHalt(ERROR_QUORUMLOG_OPEN_FAILED); } } /**** @func WORD| DmGetQuorumLogPath| Reads the quorum log file path configured in the registry during setup. @parm LPWSTR | szQuorumLogPath | A pointer to a wide string of size MAX_PATH. @parm DWORD | dwSize | The size of szQuorumLogPath in bytes. @rdesc Returns ERROR_SUCCESS for success, else returns the error code. @comm If the quorum resource is not cabaple of logging this should not be set. @xref ****/ DWORD DmGetQuorumLogPath(LPWSTR szQuorumLogPath, DWORD dwSize) { DWORD Status; Status = DmQuerySz( DmQuorumKey, cszPath, &szQuorumLogPath, &dwSize, &dwSize); if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmGetQuorumLogPath failed, error=%1!u!\n", Status); goto FnExit; } FnExit: return(Status); } /**** @func WORD| DmpGetCheckpointInterval| Reads the checkpoint interval from the registry, else returns the default. @parm LPDWORD | pdwCheckpointInterval | A pointer to DWORD where the checkpoint interval, in secs, is returned. @rdesc Returns ERROR_SUCCESS for success, else returns the error code. @comm The default checkpoint interval is 4 hours. The registry must be configured in units of hours. @xref ****/ DWORD DmpGetCheckpointInterval( OUT LPDWORD pdwCheckpointInterval) { DWORD dwDefCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL; DWORD dwStatus = ERROR_SUCCESS; dwStatus = DmQueryDword( DmQuorumKey, CLUSREG_NAME_CHECKPOINT_INTERVAL, pdwCheckpointInterval, &dwDefCheckpointInterval); if (dwStatus != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmGetCheckpointInterval Failed, error=%1!u!\n", dwStatus); goto FnExit; } //the checkpoint interval cant be less than 1 hour or more than 1 day if ((*pdwCheckpointInterval < 1) || (*pdwCheckpointInterval>24)) *pdwCheckpointInterval = DEFAULT_CHECKPOINT_INTERVAL; //convert to msecs *pdwCheckpointInterval = *pdwCheckpointInterval * 60 * 60 * 1000; FnExit: return(dwStatus); } /**** @func WORD| DmGetQuorumLogMaxSize| Reads the quorum log file max size. @parm LPDWORD | pdwMaxLogSize| A pointer to a dword containing the size. @rdesc Returns ERROR_SUCCESS for success, else returns the error code. @comm If the quorum resource is not cabaple of logging this should not be set. @xref ****/ DWORD DmGetQuorumLogMaxSize(LPDWORD pdwMaxLogSize) { DWORD Status; DWORD dwDefaultLogMaxSize = CLUSTER_QUORUM_DEFAULT_MAX_LOG_SIZE; Status = DmQueryDword( DmQuorumKey, cszMaxQuorumLogSize, pdwMaxLogSize, &dwDefaultLogMaxSize); if (Status != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmGetQuorumLogMaxSize failed, error=%1!u!\n",Status); } return(Status); } /**** @func DWORD | DmpCheckDiskSpace| Called to check for the disk space on the quorum resource after it is brought online and logs are rolled up. @rdesc ERROR_SUCCESS if successful. Win32 error code if something horrible happened. @comm This function checks if there is enough disk space and sets up a periodic timer to monitor the disk space. @xref ****/ DWORD DmpCheckDiskSpace() { DWORD dwError = ERROR_SUCCESS; WCHAR szQuoLogPathName[MAX_PATH]; ULARGE_INTEGER liNumTotalBytes; ULARGE_INTEGER liNumFreeBytes; //if you own the quorum resource, try to check the size if (gpQuoResource && AMIOWNEROFQUORES(gpQuoResource) && gbIsQuoResOnline) { //get the path if ((dwError = DmGetQuorumLogPath(szQuoLogPathName, sizeof(szQuoLogPathName))) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpCheckDiskSpace: Quorum log file is not configured, error=%1!u!\r\n", dwError); //log something in the event log CL_LOGFAILURE(dwError); goto FnExit; } //check the minimum space on the quorum disk if (!QfsGetDiskFreeSpaceEx(szQuoLogPathName, &liNumFreeBytes, &liNumTotalBytes, NULL)) { dwError = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpCheckDiskSpace: GetDiskFreeSpace returned error=0x%1!08lx!\r\n", dwError); goto FnExit; } //if not available, log something in the event log and bail out if ((liNumFreeBytes.HighPart == 0) && (liNumFreeBytes.LowPart < DISKSPACE_INIT_MINREQUIRED)) { CL_LOGCLUSWARNING(LM_DISKSPACE_HIGH_WATERMARK); dwError = ERROR_CLUSTERLOG_NOT_ENOUGH_SPACE; goto FnExit; } } FnExit: return(dwError); } /**** @func DWORD | DmpDiskManage | This is the callback registered to perform periodic disk check functions on the quorum resource. @comm If the disk space has dipped below the lowwatermark, this gracefully shuts the cluster service. If the disk space dips below the high watermark, it sends an alert to registered recipients. @xref ****/ void WINAPI DmpDiskManage( IN HANDLE hTimer, IN PVOID pContext) { DWORD dwError; WCHAR szQuoLogPathName[MAX_PATH]; ULARGE_INTEGER liNumTotalBytes; ULARGE_INTEGER liNumFreeBytes; static DWORD dwNumWarnings=0; if (!gpQuoResource || (!AMIOWNEROFQUORES(gpQuoResource)) || (!gbIsQuoResOnline || (CsNoQuorumLogging))) { //the owner of the quorum resource checks the disk space //the quorum disk shouldnt go offline //skip checking if no quorum logging is required return; } //get the path if ((dwError = DmGetQuorumLogPath(szQuoLogPathName, sizeof(szQuoLogPathName))) != ERROR_SUCCESS) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpDiskManage: Quorum log file is not configured, error=%1!u!\r\n", dwError); //log something in the event log CL_UNEXPECTED_ERROR(dwError); goto FnExit; } //check the minimum space on the quorum disk if (!QfsGetDiskFreeSpaceEx(szQuoLogPathName, &liNumFreeBytes, &liNumTotalBytes, NULL)) { dwError = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpDiskManage: GetDiskFreeSpace returned error=0x%1!08lx!\r\n", dwError); CL_LOGFAILURE(dwError); goto FnExit; } if ((liNumFreeBytes.HighPart == 0) && (liNumFreeBytes.LowPart < DISKSPACE_LOW_WATERMARK)) { //reached the low water mark dwNumWarnings++; //ss: we can control the rate at which we put things in the //event log but once every five minutes is not bad. //ss: post an event ??? ClRtlLogPrint(LOG_NOISE, "[DM] DmpDiskManage: GetDiskFreeSpace - Not enough disk space, Avail=0x%1!08lx!\r\n", liNumFreeBytes.LowPart); CL_LOGCLUSWARNING(LM_DISKSPACE_LOW_WATERMARK); } else { gbIsQuoResEnoughSpace = TRUE; dwNumWarnings = 0; } FnExit: return; } /**** @func DWORD | DmpCheckpointTimerCb | This is the callback registered to perform periodic checkpointing on the quorum log. @parm IN HANDLE| hTimer| The timer associated with checkpointing interval. @parm IN PVOID | pContext | A pointer to the handle for the quorum log file. @comm This helps in backups. If you want to take a cluster backup by making a copy of the quorum.log and checkpoint files, then if both nodes have been up for a long time both the files can be old. By taking a periodic checkpoint we guarantee that they are not more than n hours old. ****/ void WINAPI DmpCheckpointTimerCb( IN HANDLE hTimer, IN PVOID pContext) { HLOG hQuoLog; DWORD dwError; // // Chittur Subbaraman (chitturs) - 6/3/99 // // Make sure the gLockDmpRoot is held before LogCheckPoint is called // so as to maintain the ordering between this lock and the log lock. // In addition, we want to read the pContext safely. This is because // pContext is a pointer to the log and could change via the SetClusterQuorumResource // API. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); hQuoLog = *((HLOG *)pContext); if (hQuoLog && gbDmInited) { //get a checkpoint database ClRtlLogPrint(LOG_NOISE, "[DM]DmpCheckpointTimerCb- taking a checkpoint\r\n"); dwError = LogReset(hQuoLog); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_CRITICAL, "[DM]DmpCheckpointTimerCb - Failed to reset log, error=%1!u!\r\n", dwError); CL_UNEXPECTED_ERROR(dwError); } } RELEASE_LOCK(gLockDmpRoot); } /**** @func DWORD | DmBackupClusterDatabase | Take a fresh checkpoint and copy the quorum log and the checkpoint file to the supplied path name. This function is called with gQuoLock held. @parm IN LPCWSTR | lpszPathName | The directory path name where the files have to be backed up. This path must be visible to the node on which the quorum resource is online (i.e., this node in this case). @comm This function first takes a fresh checkpoint, updates the quorum log file and then copies the two files to a backup area. @rdesc Returns a Win32 error code on failure. ERROR_SUCCESS on success. @xref ****/ DWORD DmBackupClusterDatabase( IN LPCWSTR lpszPathName) { QfsHANDLE hFindFile = QfsINVALID_HANDLE_VALUE; WIN32_FIND_DATA FindData; DWORD status = ERROR_SUCCESS; LPWSTR szDestPathName = NULL; DWORD dwLen; // // Chittur Subbaraman (chitturs) - 10/12/98 // dwLen = lstrlenW( lpszPathName ); // // It is safer to use dynamic memory allocation for user-supplied // path since we don't want to put restrictions on the user // on the length of the path that can be supplied. However, as // far as our own quorum disk path is concerned, it is system-dependent // and static memory allocation for that would suffice. // szDestPathName = (LPWSTR) LocalAlloc ( LMEM_FIXED, ( dwLen + 5 ) * sizeof ( WCHAR ) ); if ( szDestPathName == NULL ) { status = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmBackupClusterDatabase: Error %1!d! in allocating memory for %2!ws! !!!\n", status, lpszPathName); CL_LOGFAILURE( status ); goto FnExit; } lstrcpyW( szDestPathName, lpszPathName ); // // If the client-supplied path is not already terminated with '\', // then add it. // if ( szDestPathName [dwLen-1] != L'\\' ) { szDestPathName [dwLen++] = L'\\'; } // // Add a wild character at the end to search for any file in the // supplied directory // szDestPathName[dwLen++] = L'*'; szDestPathName[dwLen] = L'\0'; // // Find out whether you can access the supplied path by // trying to find some file in the directory. // hFindFile = QfsFindFirstFile( szDestPathName, &FindData ); if ( !QfsIsHandleValid(hFindFile) ) { status = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmBackupClusterDatabase: Supplied path %1!ws! does not exist, Error=%2!d! !!!\n", szDestPathName, status); goto FnExit; } // // Check whether the log is open. It must be since we already // verified that the quorum resource is online on this node and // quorum logging is turned on. // if ( ghQuoLog ) { // // Remove the '*' so the same variable can be used. // szDestPathName [dwLen-1] = L'\0'; ClRtlLogPrint(LOG_NOISE, "[DM] DmBackupClusterDatabase: Attempting to take a checkpoint and then backup to %1!ws!..\n", szDestPathName); // // The gLockDmpRoot needs to be acquired here since otherwise // you will get the log lock in the LogCheckPoint( ) // function and someone else could get the gLockDmpRoot. // After you get the log lock, you also try to acquire // the gLockDmpRoot in the function DmCommitRegistry. // This is a potential deadlock situation and is avoided here. // ACQUIRE_SHARED_LOCK(gLockDmpRoot); status = DmpLogCheckpointAndBackup ( ghQuoLog, szDestPathName ); RELEASE_LOCK(gLockDmpRoot); if ( status == ERROR_SUCCESS ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmBackupClusterDatabase: Successfully finished backing up to %1!ws!...\n", szDestPathName); } } else { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmBackupClusterDatabase: Quorum log could not be opened...\r\n"); status = ERROR_QUORUMLOG_OPEN_FAILED; } FnExit: QfsFindCloseIfValid ( hFindFile ); LocalFree ( szDestPathName ); return ( status ); } /**** @func DWORD | DmpLogCheckpointAndBackup | Takes a checkpoint, updates the quorum log and then copies the files to the supplied path. This function is called with the gQuoLock and the gLockDmpRoot held. @parm IN HLOG | hLogFile | An identifier for the quorum log file. @parm IN LPWSTR | lpszPathName | The path for storing the quorum log file, the recent checkpoint file, and the resource registry checkpoint files. This path must be visible from this node. @comm Called by DmpBackupQuorumLog() to take a checkpoint and then take a backup of the cluster database including resource registry checkpoint files. @rdesc Returns a Win32 error code on failure. ERROR_SUCCESS on success. @xref ****/ DWORD DmpLogCheckpointAndBackup( IN HLOG hLogFile, IN LPWSTR lpszPathName) { DWORD dwError; DWORD dwLen; WCHAR szChkPointFilePrefix[MAX_PATH]; WCHAR szQuoLogPathName[MAX_PATH]; LPWSTR szDestFileName = NULL; WCHAR szSourceFileName[MAX_PATH]; LPWSTR szDestPathName = NULL; LPWSTR lpChkPointFileNameStart; LSN Lsn; TRID Transaction; QfsHANDLE hFile = QfsINVALID_HANDLE_VALUE; // // Chittur Subbaraman (chitturs) - 10/12/1998 // // // Initiate a checkpoint process. Allow a log file reset, if necessary. // if ( ( dwError = LogCheckPoint( hLogFile, TRUE, NULL, 0 ) ) != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Callback failed to return a checkpoint. Error=%1!u!\r\n", dwError); CL_LOGFAILURE( dwError ); LogClose( hLogFile ); goto FnExit; } // // Get the name of the most recent checkpoint file // szChkPointFilePrefix[0] = TEXT('\0'); if ( ( dwError = LogGetLastChkPoint( hLogFile, szChkPointFilePrefix, &Transaction, &Lsn ) ) != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::No check point found in the log file. Error=%1!u!\r\n", dwError); CL_LOGFAILURE( dwError ); LogClose( hLogFile ); goto FnExit; } dwError = DmGetQuorumLogPath( szQuoLogPathName, sizeof( szQuoLogPathName ) ); if ( dwError != ERROR_SUCCESS ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::DmGetQuorumLogPath failed, Error = %1!d!\r\n", dwError); CL_LOGFAILURE( dwError ); goto FnExit; } // // It is safer to use dynamic memory allocation for user-supplied // path since we don't want to put restrictions on the user // on the length of the path that can be supplied. However, as // far as our own quorum disk path is concerned, it is system-dependent // and static memory allocation for that would suffice. // szDestPathName = (LPWSTR) LocalAlloc ( LMEM_FIXED, ( lstrlenW ( lpszPathName ) + 1 ) * sizeof ( WCHAR ) ); if ( szDestPathName == NULL ) { dwError = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogCheckpointAndBackup: Error %1!d! in allocating memory for %2!ws! !!!\n", dwError, lpszPathName); CL_LOGFAILURE( dwError ); goto FnExit; } // // Get the user-supplied destination path name // lstrcpyW( szDestPathName, lpszPathName ); szDestFileName = (LPWSTR) LocalAlloc ( LMEM_FIXED, ( lstrlenW ( szDestPathName ) + 1 + LOG_MAX_FILENAME_LENGTH ) * sizeof ( WCHAR ) ); if ( szDestFileName == NULL ) { dwError = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpLogCheckpointAndBackup: Error %1!d! in allocating memory for chkpt file name !!!\n", dwError); CL_LOGFAILURE( dwError ); goto FnExit; } // // Make an attempt to delete the CLUSBACKUP.DAT file // lstrcpyW( szDestFileName, szDestPathName ); lstrcatW( szDestFileName, L"CLUSBACKUP.DAT" ); // // Set the file attribute to normal. Continue even if you // fail in this step, but log an error. (Note that you are // countering the case in which a destination file with // the same name exists in the backup directory already and // you are trying to delete it.) // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_NORMAL ) ) { dwError = GetLastError(); if ( dwError != ERROR_FILE_NOT_FOUND ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to NORMAL, Error = %2!d!\n", szDestFileName, dwError); } } if ( !QfsDeleteFile( szDestFileName ) ) { dwError = GetLastError(); if ( dwError != ERROR_FILE_NOT_FOUND ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::CLUSBACKUP.DAT exists, but can't delete it, Error = %1!d!\n", dwError); CL_LOGFAILURE( dwError ); goto FnExit; } } // // Just get the checkpoint file name without any path added. // Note that szQuoLogPathName includes the '\' // dwLen = lstrlenW ( szQuoLogPathName ); lpChkPointFileNameStart = &szChkPointFilePrefix[dwLen]; // // Now, create the path-included destination file name // lstrcpyW( szDestFileName, szDestPathName ); lstrcatW( szDestFileName, lpChkPointFileNameStart ); // // And, the path-included source file name // lstrcpyW( szSourceFileName, szChkPointFilePrefix ); // // Set the file attribute to normal. Continue even if you // fail in this step, but log an error. (Note that you are // countering the case in which a destination file with // the same name exists in the backup directory already and // you are trying to overwrite it.) // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_NORMAL ) ) { dwError = GetLastError(); if ( dwError != ERROR_FILE_NOT_FOUND ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to NORMAL, Error = %2!d!\n", szDestFileName, dwError); } } // // Copy the checkpoint file to the destination // dwError = QfsClRtlCopyFileAndFlushBuffers( szSourceFileName, szDestFileName ); if ( !dwError ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Unable to copy file %1!ws! to %2!ws!, Error = %3!d!\n", szSourceFileName, szDestFileName, dwError); CL_LOGFAILURE( dwError ); goto FnExit; } // // Set the file attribute to read-only. Continue even if you // fail in this step, but log an error. // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_READONLY ) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to READONLY, Error = %2!d!\n", szDestFileName, dwError); } // // Now, create the path-included destination file name // lstrcpyW( szDestFileName, szDestPathName ); lstrcatW( szDestFileName, cszQuoFileName ); // // And, the path-included source file name // lstrcpyW( szSourceFileName, szQuoLogPathName ); lstrcatW( szSourceFileName, cszQuoFileName ); // // Set the destination file attribute to normal. Continue even if you // fail in this step, but log an error. (Note that you are // countering the case in which a destination file with // the same name exists in the backup directory already and // you are trying to overwrite it.) // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_NORMAL ) ) { dwError = GetLastError(); if ( dwError != ERROR_FILE_NOT_FOUND ) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to NORMAL, Error = %2!d!\n", szDestFileName, dwError); } } // // Copy the quorum log file to the destination // dwError = QfsCopyFile( szSourceFileName, szDestFileName, FALSE ); if ( !dwError ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Unable to copy file %1!ws! to %2!ws!, Error = %3!d!\n", szSourceFileName, szDestFileName, dwError); CL_LOGFAILURE( dwError ); goto FnExit; } // // Set the destination file attribute to read-only. Continue even // if you fail in this step, but log an error // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_READONLY ) ) { dwError = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to READONLY, Error = %2!d!\n", szDestFileName, dwError); } // // Now copy the resource chkpt files to the destination. Note that // we call this function with both gQuoLock and gLockDmpRoot held. // The former lock prevents any checkpoint being read or written // via CppReadCheckpoint() and CppWriteCheckpoint() while the // following function is executing. // // Note: However, the CpDeleteRegistryCheckPoint() function is // unprotected and poses a potential danger here. // // Note: Also, currently the following function returns ERROR_SUCCESS // in all cases. // dwError = CpCopyCheckpointFiles( szDestPathName, TRUE ); if (dwError != ERROR_SUCCESS) { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Unable to copy resource checkpoint files, Error = %1!d!\n", dwError); goto FnExit; } // // Now create an empty READONLY, HIDDEN, file in the destination // directory which marks the successful ending of the backup. // lstrcpyW( szDestFileName, szDestPathName ); lstrcatW( szDestFileName, L"CLUSBACKUP.DAT"); hFile = QfsCreateFile(szDestFileName, GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_NEW, FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_READONLY, NULL ); if ( !QfsIsHandleValid(hFile) ) { dwError = GetLastError(); CL_LOGFAILURE( dwError ); goto FnExit; } dwError = ERROR_SUCCESS; FnExit: LocalFree ( szDestFileName ); LocalFree ( szDestPathName ); QfsCloseHandleIfValid ( hFile ); return ( dwError ); } /**** @func DWORD | DmpRestoreClusterDatabase | Copy the quorum log and all the checkpoint files from CsDatabaseRestorePath to the quorum log path in the quorum disk. @parm IN LPCWSTR | lpszQuoLogPathName | The quorum directory path where the backed up files have to be copied to. @rdesc Returns a Win32 error code on failure. ERROR_SUCCESS on success. @xref ****/ DWORD DmpRestoreClusterDatabase( IN LPCWSTR lpszQuoLogPathName ) { QfsHANDLE hFindFile = QfsINVALID_HANDLE_VALUE; WIN32_FIND_DATA FindData; DWORD status; WCHAR szDestFileName[MAX_PATH]; LPWSTR szSourceFileName = NULL; LPWSTR szSourcePathName = NULL; DWORD dwLen; WCHAR szChkptFileNameStart[4]; WCHAR szTempFileName[MAX_PATH]; // // Chittur Subbaraman (chitturs) - 10/20/98 // dwLen = lstrlenW ( CsDatabaseRestorePath ); // // It is safer to use dynamic memory allocation for user-supplied // path since we don't want to put restrictions on the user // on the length of the path that can be supplied. However, as // far as our own quorum disk path is concerned, it is system-dependent // and static memory allocation for that would suffice. // szSourcePathName = (LPWSTR) LocalAlloc ( LMEM_FIXED, ( dwLen + 25 ) * sizeof ( WCHAR ) ); if ( szSourcePathName == NULL ) { status = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Error %1!d! in allocating memory for %2!ws! !!!\n", status, CsDatabaseRestorePath); CL_LOGFAILURE( status ); goto FnExit; } lstrcpyW ( szSourcePathName, CsDatabaseRestorePath ); // // If the client-supplied path is not already terminated with '\', // then add it. // if ( szSourcePathName [dwLen-1] != L'\\' ) { szSourcePathName [dwLen++] = L'\\'; szSourcePathName[dwLen] = L'\0'; } lstrcatW ( szSourcePathName, L"CLUSBACKUP.DAT" ); // // Try to find the CLUSBACKUP.DAT file in the directory // hFindFile = QfsFindFirstFile( szSourcePathName, &FindData ); // // Reuse the source path name variable // szSourcePathName[dwLen] = L'\0'; if ( !QfsIsHandleValid(hFindFile) ) { status = GetLastError(); if ( status != ERROR_FILE_NOT_FOUND ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Path %1!ws! unavailable, Error = %2!d! !!!\n", szSourcePathName, status); } else { status = ERROR_DATABASE_BACKUP_CORRUPT; ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Backup procedure not fully successful, can't restore DB, Error = %1!d! !!!\n", status); } CL_LOGFAILURE( status ); goto FnExit; } QfsFindClose ( hFindFile ); szSourcePathName[dwLen++] = L'*'; szSourcePathName[dwLen] = L'\0'; // // Try to find any file in the directory // hFindFile = QfsFindFirstFile( szSourcePathName, &FindData ); // // Reuse the source path name variable // szSourcePathName[dwLen-1] = L'\0'; if ( !QfsIsHandleValid(hFindFile) ) { status = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Error %2!d! in trying to find file in path %1!ws!\r\n", szSourcePathName, status); CL_LOGFAILURE( status ); goto FnExit; } szSourceFileName = (LPWSTR) LocalAlloc ( LMEM_FIXED, ( lstrlenW ( szSourcePathName ) + 1 + LOG_MAX_FILENAME_LENGTH ) * sizeof ( WCHAR ) ); if ( szSourceFileName == NULL ) { status = GetLastError(); ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Error %1!d! in allocating memory for source file name !!!\n", status); CL_LOGFAILURE( status ); goto FnExit; } status = ERROR_SUCCESS; // // Now, find and copy all relevant files from the backup area // to the quorum disk. Note that only one of the copied chk*.tmp // files will be used as the valid checkpoint. However, we copy // all chk*.tmp files to make this implementation simple and // straightforward to comprehend. // while ( status == ERROR_SUCCESS ) { if ( FindData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) { if ( FindData.cFileName[0] == L'.' ) { if ( FindData.cFileName[1] == L'\0' || FindData.cFileName[1] == L'.' && FindData.cFileName[2] == L'\0' ) { goto skip; } } // // Since the found file is infact a directory, check // whether it is one of the resource checkpoint directories. // If so copy the relevant checkpoint files to the quorum // disk. // if ( ( status = CpRestoreCheckpointFiles( szSourcePathName, FindData.cFileName, lpszQuoLogPathName ) ) != ERROR_SUCCESS ) { ClRtlLogPrint(LOG_NOISE, "[DM] DmpRestoreClusterDatabase: Error %1!d! in copying resource cp files !!!\n", status); CL_LOGFAILURE( status ); goto FnExit; } } else { lstrcpyW ( szTempFileName, FindData.cFileName ); szTempFileName[3] = L'\0'; mbstowcs( szChkptFileNameStart, "chk", 4 ); if ( ( lstrcmpW ( szTempFileName, szChkptFileNameStart ) == 0 ) || ( lstrcmpW ( FindData.cFileName, cszQuoFileName ) == 0 ) ) { lstrcpyW( szSourceFileName, szSourcePathName ); lstrcatW( szSourceFileName, FindData.cFileName ); lstrcpyW( szDestFileName, lpszQuoLogPathName ); lstrcatW( szDestFileName, FindData.cFileName ); status = QfsCopyFile( szSourceFileName, szDestFileName, FALSE ); if ( !status ) { status = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpRestoreClusterDatabase: Unable to copy file %1!ws! to %2!ws!, Error = %3!d!\n", szSourceFileName, szDestFileName, status); CL_LOGFAILURE( status ); goto FnExit; } // // Set the file attribute to normal. There is no reason // to fail in this step since the quorum disk is ours // and we succeeded in copying the file. // if ( !QfsSetFileAttributes( szDestFileName, FILE_ATTRIBUTE_NORMAL ) ) { status = GetLastError(); ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpLogCheckpointAndBackup::Error in changing %1!ws! attribute to NORMAL, error = %2!u!\n", szDestFileName, status); CL_LOGFAILURE( status ); goto FnExit; } } } skip: if ( QfsFindNextFile( hFindFile, &FindData ) ) { status = ERROR_SUCCESS; } else { status = GetLastError(); } } if ( status == ERROR_NO_MORE_FILES ) { status = ERROR_SUCCESS; } else { ClRtlLogPrint(LOG_UNUSUAL, "[DM] DmpRestoreClusterDatabase: FindNextFile failed! Error = %1!u!\n", status); } FnExit: QfsFindCloseIfValid ( hFindFile ); LocalFree ( szSourceFileName ); LocalFree ( szSourcePathName ); return ( status ); } /**** @func DWORD | DmpHandleNodeDownEvent | Handle the node down event for DM. @parm IN LPVOID | NotUsed | Unused parameter. @rdesc Returns ERROR_SUCCESS. @xref ****/ DWORD DmpHandleNodeDownEvent( IN LPVOID NotUsed ) { // // Chittur Subbaraman (chitturs) - 7/23/99 // // This function handles the DM node down processing as a separate // thread. The reasons for creating this thread are outlined in // DmpEventHandler. // ClRtlLogPrint(LOG_NOISE, "[DM] DmpHandleNodeDownEvent - Entry...\r\n"); // // SS: I am not the owner of the quorum resource as yet, but I might // be after rearbitration, in that case, just set a flag saying we // need to checkpoint. It will be looked at when the quorum resource // comes online. The following function in FM checks if the // quorum is online on this node and if it is, it calls // the checkpoint callback function. If it is not, it sets the // global boolean variable passed to TRUE. // FmCheckQuorumState( DmpLogCheckPointCb, &gbNeedToCheckPoint ); ClRtlLogPrint(LOG_NOISE, "[DM] DmpHandleNodeDownEvent - Exit...\r\n"); return( ERROR_SUCCESS ); }