//+--------------------------------------------------------------------------- // // Microsoft Windows // Copyright (C) Microsoft Corporation, 1991 - 1998. // // File: CHANGLOG.CXX // // Contents: Methods to make DocQueue persistent // // Classes: CDocQueue // // History: 08-Feb-91 DwightKr Created // 24-Feb-97 SitaramR Push filtering // //---------------------------------------------------------------------------- #include #pragma hdrstop #include #include #include #include #include #include #include "changlog.hxx" #include "fresh.hxx" #include "resman.hxx" #include "notxact.hxx" //+--------------------------------------------------------------------------- // // Function: CDocChunk::UpdateMaxUsn // // Synopsis: Updates the max usn flushed info // // Arguments: [aUsnFlushInfo] -- Array of usn info to be updated // // History: 05-07-97 SitaramR Created // //---------------------------------------------------------------------------- void CDocChunk::UpdateMaxUsn( CCountedDynArray & aUsnFlushInfo ) { for ( unsigned i=0; iVolumeId() == volumeId ) { // // Usn's need not be monotonic because there can be notifications from // different scopes on same usn volume // if ( usn > aUsnFlushInfo[j]->UsnHighest() ) aUsnFlushInfo[j]->SetUsnHighest( usn ); fFound = TRUE; } } if ( !fFound ) { CUsnFlushInfo *pUsnInfo = new CUsnFlushInfo( volumeId, usn ); XPtr xUsnInfo( pUsnInfo ); aUsnFlushInfo.Add( pUsnInfo, aUsnFlushInfo.Count() ); xUsnInfo.Acquire(); } } } } //+--------------------------------------------------------------------------- // // Function: AcquireAndAppend // // Synopsis: Acquire the chunks from the given "newList" and append these // chunks to the current list. // // Arguments: [newList] -- List to acquire from. // // History: 5-26-94 srikants Created // // Notes: The newList will be emptied. // //---------------------------------------------------------------------------- void CChunkList::AcquireAndAppend( CChunkList & newList ) { while ( !newList.IsEmpty() ) { CDocChunk * pChunk = newList.Pop(); Win4Assert( pChunk ); Append( pChunk ); } } //+--------------------------------------------------------------------------- // // Function: Constructor // // Synopsis: Constructs the CChangeLog class by figuring out the number of // chunks present in the change log. // // Arguments: [widChangeLog] -- Workid of changlog // [storage] -- Storage // [type] -- Primary or secondary changlog type // // History: 5-26-94 srikants Created // 2-24-97 SitaramR Push filtering // //---------------------------------------------------------------------------- CChangeLog::CChangeLog( WORKID widChangeLog, PStorage & storage, PStorage::EChangeLogType type ) : _sigChangeLog(eSigChangeLog), _pResManager( 0 ), _widChangeLog(widChangeLog), _storage(storage), _type(type), _PersDocQueue(_storage.QueryChangeLog(_widChangeLog,type)), _oChunkToRead(0), _cChunksAvail(0), _cChunksTotal(0), _fUpdatesEnabled( FALSE ), _fPushFiltering( FALSE ) { Win4Assert( IsPrimary() || IsSecondary() ); } //+--------------------------------------------------------------------------- // // Function: Destructor // // History: 8-Nov-94 DwightKr Created // //---------------------------------------------------------------------------- CChangeLog::~CChangeLog() { } //+--------------------------------------------------------------------------- // // Function: LokVerifyConsistency // // History: 12-15-94 srikants Created // //---------------------------------------------------------------------------- #if CIDBG==1 void CChangeLog::LokVerifyConsistency() { PRcovStorageObj & persDocQueue = *_PersDocQueue; CRcovStorageHdr & hdr = persDocQueue.GetHeader(); Win4Assert( _cChunksAvail + _oChunkToRead == _cChunksTotal ); ULONG ulDataSize = hdr.GetUserDataSize( hdr.GetPrimary() ); ULONG cTotPrim = ulDataSize / (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum()); ULONG cPersTotalRec = hdr.GetCount( hdr.GetPrimary() ); Win4Assert( cPersTotalRec == cTotPrim ); Win4Assert( cPersTotalRec * (cDocInChunk * sizeof(CDocNotification) + CRcovStrmIter::SizeofChecksum()) == ulDataSize); Win4Assert((ulDataSize % (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum())) == 0); ULONG cTotBackup = hdr.GetCount( hdr.GetBackup() ); Win4Assert( 0 == _cChunksTotal || _cChunksTotal == cTotPrim ); } #else inline void CChangeLog::LokVerifyConsistency(){} #endif //+--------------------------------------------------------------------------- // // Function: LokEmpty, public // // Synopsis: Initializes the change log by empting it and setting status // to eCIDiskFullScan // // History: 15-Nov-94 DwightKr Created // //---------------------------------------------------------------------------- void CChangeLog::LokEmpty() { CImpersonateSystem impersonate; PRcovStorageObj & persDocQueue = *_PersDocQueue; CRcovStrmWriteTrans xact( persDocQueue ); persDocQueue.GetHeader().SetCount(persDocQueue.GetHeader().GetBackup(), 0); xact.Empty(); xact.Commit(); _oChunkToRead = 0; _cChunksAvail = 0; _cChunksTotal = 0; LokVerifyConsistency(); } //+--------------------------------------------------------------------------- // // Function: InitSize // // Synopsis: Initializes the size of the change log by reading in the // size information from the change log header. // // History: May-26-94 srikants Rewrite and move from CDocQueue // Nov-08-94 DwightKr Added dirty flag set on startup // //---------------------------------------------------------------------------- void CChangeLog::InitSize() { ULONG ulDataSize = 0; // // Two copies of the content scan data must fit into the recoverable // stream header in the user-data section. Verify that we haven't // grown too large. // CImpersonateSystem impersonate; PRcovStorageObj & persDocQueue = *_PersDocQueue; CRcovStorageHdr & hdr = persDocQueue.GetHeader(); persDocQueue.VerifyConsistency(); // // Do a read transaction to complete any in-complete transactions. // { CRcovStrmReadTrans xact( persDocQueue ); } // // Determine the # of records in the persistent log and do some // consistency checks. // ulDataSize = hdr.GetUserDataSize( hdr.GetPrimary() ); ULONG cRecords = hdr.GetCount( hdr.GetPrimary() ); ULONG cTotRecord = ulDataSize / (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum()); // // If the number of records in the file is not the same as the // number of records in the header, we have an inconsistancy. // if ( cRecords != cTotRecord ) { Win4Assert( !"Corrupt changelog" ); _storage.ReportCorruptComponent( L"ChangeLog2" ); THROW( CException( CI_CORRUPT_DATABASE ) ); } _oChunkToRead = 0; _cChunksAvail = cTotRecord; _cChunksTotal = _cChunksAvail; } //+--------------------------------------------------------------------------- // // Function: DeSerialize // // Synopsis: Reads the specified number of chunks from the change log and // appends them to the list passed. // // Arguments: [list] -- List to append to. // [cChunksToRead] -- Number of chunks to read. // // Returns: Number of chunks read. // // History: 5-26-94 srikants Created // //---------------------------------------------------------------------------- ULONG CChangeLog::DeSerialize( CChunkList & list, ULONG cChunksToRead ) { CImpersonateSystem impersonate; CChunkList newChunks; Win4Assert( cChunksToRead <= _cChunksAvail ); cChunksToRead = min (cChunksToRead, _cChunksAvail); PRcovStorageObj & persDocQueue = _PersDocQueue.Get(); CRcovStorageHdr & hdr = persDocQueue.GetHeader(); // // We seem to be losing writes for the header stream. Just check the // consistency before reading the data from disk. // LokVerifyConsistency(); { // Begin transaction CRcovStrmReadTrans xact( persDocQueue ); CRcovStrmReadIter iter( xact, cDocInChunk * sizeof(CDocNotification) ); iter.Seek( _oChunkToRead ); for ( unsigned i=0; i < cChunksToRead ; i++ ) { CDocChunk *pChunk = new CDocChunk; iter.GetRec( pChunk->GetArray() ); newChunks.Append(pChunk); } } // End-Transaction Win4Assert( newChunks.Count() == cChunksToRead ); // // Acquire the newly created list and append it to the existing list. // list.AcquireAndAppend(newChunks); // // Update the internal state. // _oChunkToRead += cChunksToRead; _cChunksAvail -= cChunksToRead; if ( _cChunksAvail + _oChunkToRead != _cChunksTotal ) { Win4Assert( ! "Data Corruption && ChangeLog::_cChunksAvail + _oChunktoRead != _cChunksTotal" ); _storage.ReportCorruptComponent( L"ChangeLog3" ); THROW( CException( CI_CORRUPT_DATABASE ) ); } LokVerifyConsistency(); return(cChunksToRead); } //+--------------------------------------------------------------------------- // // Function: Serialize // // Synopsis: Serializes the given list of CDocChunks to the disk. // // Arguments: [listToSerialize] -- The list to be serialized // [aUsnFlushInfo] -- Usn flush info returned here // // Returns: Number of chunks serialized. // // History: 5-26-94 srikants Created // //---------------------------------------------------------------------------- ULONG CChangeLog::Serialize( CChunkList & listToSerialize, CCountedDynArray & aUsnFlushInfo ) { CImpersonateSystem impersonate; unsigned cRecords = 0; USN updateUSN = 0; CRcovStorageHdr & hdr = _PersDocQueue.Get().GetHeader(); LokVerifyConsistency(); { // // Begin Transaction // // STACKSTACK // XPtr xTrans( new CRcovStrmAppendTrans( _PersDocQueue.Get() ) ); CRcovStrmAppendIter iter( xTrans.GetReference(), cDocInChunk * sizeof(CDocNotification) ); for ( CDocChunk * pChunk = listToSerialize.GetFirst() ; 0 != pChunk; pChunk = pChunk->GetNext() ) { pChunk->UpdateMaxUsn( aUsnFlushInfo ); iter.AppendRec( pChunk->GetArray() ); cRecords++; } xTrans->Commit(); } // EndTransaction ciDebugOut( (DEB_ITRACE, "SerializeRange: %d records serialized to disk\n", cRecords) ); _cChunksAvail += cRecords; _cChunksTotal += cRecords; // // Extra level of consistency checking for the lengths and record // counts in the on-disk version vs. in-memory values. // ULONG ulDataSize = hdr.GetUserDataSize( hdr.GetPrimary() ); ULONG cTotRecord = ulDataSize / (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum()); ULONG cRecCount = hdr.GetCount( hdr.GetPrimary() ); if ( (cTotRecord != cRecCount) || ((ulDataSize % (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum())) != 0) || (_cChunksTotal != cTotRecord) || (_oChunkToRead + _cChunksAvail != cTotRecord) ) { ciDebugOut(( DEB_ERROR, "cTotRecord %d, cRecCount %d, ulDataSize %d\n" "sizeof(CDocNotifications) %d, CRcovStrmIter::SizeofChecksum() %d\n" "cDocInChunk %d, _cChunksTotal %d, cTotRecord %d\n" "_oChunkToRead %d, _cChunksAvail %d\n", cTotRecord, cRecCount, ulDataSize, sizeof(CDocNotification), CRcovStrmIter::SizeofChecksum(), cDocInChunk, _cChunksTotal, cTotRecord, _oChunkToRead, _cChunksAvail )); Win4Assert( "Data Corruption" && cTotRecord == cRecCount); Win4Assert( "Data Corruption" && (ulDataSize % (sizeof(CDocNotification) * cDocInChunk + CRcovStrmIter::SizeofChecksum())) == 0); Win4Assert( "Data Corruption" && _cChunksTotal == cTotRecord ); Win4Assert( "Data Corruption" && _oChunkToRead + _cChunksAvail == _cChunksTotal ); _storage.ReportCorruptComponent( L"ChangeLog5" ); THROW( CException( CI_CORRUPT_DATABASE ) ); } ciDebugOut(( DEB_ITRACE, "Serialize: cRecords=%d ulDataSize/sizeof(record)=%d\n", cRecCount, cTotRecord )); return cRecords; } //Serialize //+--------------------------------------------------------------------------- // // Function: LokDeleteWIDsInPersistentIndexes // // Synopsis: This method computes the number of chunks that can be // deleted from the front of the changelog and truncates the // log appropriately. // // All documents that have been filtered and successfully made // into a persistent index need not be filtered after restart. // This method iterates over the filtered chunks from the front // of the log and when it detects a document that has not yet // made it into a persistent index, it stops the iteration and // truncates the log upto such a chunk ( not including the chunk // that had a document in a volatile index ). // // Arguments: [cFilteredChunks] -- Number of chunks that have been // filtered. // [freshTestLatest] -- Latest freshTest // [freshTestAtMerge] -- Fresh test snapshoted at time of shadow // merge (can be same as freshTestLatest) // [docList] -- Resman's Doc list // [notifTrans] -- Notification transaction // // Returns: The number of chunks that have been truncated from the // front. // // History: 5-27-94 SrikantS Created // 2-24-97 SitaramR Push filtering // //---------------------------------------------------------------------------- ULONG CChangeLog::LokDeleteWIDsInPersistentIndexes( ULONG cFilteredChunks, CFreshTest & freshTestLatest, CFreshTest & freshTestAtMerge, CDocList & docList, CNotificationTransaction & notifTrans ) { CImpersonateSystem impersonate; // // Create the recoverable storage object. // PRcovStorageObj & persDocQueue = _PersDocQueue.Get(); CRcovStorageHdr & hdr = persDocQueue.GetHeader(); ULONG ulDataSize = hdr.GetUserDataSize( hdr.GetPrimary() ); ULONG cTotRecord = 0; LokVerifyConsistency(); #if CIDBG == 1 ULONG cPersTotalRec = hdr.GetCount( hdr.GetPrimary() ); Win4Assert( cPersTotalRec >= cFilteredChunks ); #endif ULONG cRecord = 0; BOOL fDocInPersIndex = TRUE; { // Begin Transaction CRcovStrmReadTrans xact( persDocQueue ); CRcovStrmReadIter iter( xact, cDocInChunk * sizeof(CDocNotification) ); cTotRecord = iter.UserRecordCount( ulDataSize ); CDocChunk Chunk; // // Determine the first chunk in the filtered set which has a document // that nas not made it to a persistent index. // while ( fDocInPersIndex && ( cRecord < cFilteredChunks ) ) { // // Read the next chunk from the change log. // iter.GetRec( Chunk.GetArray() ); // // Determine if all the documents in this chunk have made it to // persistent indexes or not. // for ( ULONG oRetrieve = 0; oRetrieve < cDocInChunk; oRetrieve++ ) { CDocNotification * pRetrieve = Chunk.GetDoc(oRetrieve); WORKID wid = pRetrieve->Wid(); USN usn = pRetrieve->Usn(); if ( wid == widInvalid || ( _fPushFiltering && _pResManager->LokIsWidAborted( wid, usn ) ) ) { // // An invalid wid means that the changlog did not have enough // entries to fill Chunk and hence the remaining entries were set // to widinvalid. AbortedWid means that the wid was // aborted during filtering and not refiled. Hence we don't // have to check whether this wid made it to the persistent index // or not. // continue; } if ( IsWidInDocList( wid, docList ) ) { // // If the wid is in the doclist then it means that it // did not necessarily make it to the persistent index. // fDocInPersIndex = FALSE; break; } if ( pRetrieve->IsDeletion() ) { // // For deletions, check if the wid is in iidDeleted using // freshTestAtMerge. iidDeleted is the index for both deletions // in wordlists and persistent indexes, and so by using the // frest test at merge we can be sure that we are doing the // proper check. Also, check if the wid is in iidInvalid, // because after a master merge all wids (even deleted ones) // will be in iidInvalid. Also, check if the wid is in a persistent // index. This is for the case of delete followed by an add of the // same wid. Note: since iidDeleted1, iidDeleted2, iidInvalid pass the // IsPersistent test, all the above tests can be simplified to just // checking for IsPersistent index. // INDEXID iid = freshTestAtMerge.Find( wid ); CIndexId IndexID ( iid ); if ( !IndexID.IsPersistent() ) { fDocInPersIndex = FALSE; break; } } else { // // Add or modify case. We use frestTestLatest because if the // wid is in the wordlist and an earlier version of the same // wid in a persistent index, then we'll assume that the wid // has not yet made it to the persistent index. Master index // is iidinvalid, which is a persistent index. // INDEXID iid = freshTestLatest.Find( wid ); if ( iid == iidDeleted1 || iid == iidDeleted2 ) { // // An add/modify may actually be a deletion that was requeued as modify. For // deletions we should use freshTestAtMerge as described above. // iid = freshTestAtMerge.Find( wid ); if ( iid != iidDeleted1 && iid != iidDeleted2 ) { fDocInPersIndex = FALSE; break; } } else { CIndexId IndexID ( iid ); if ( !IndexID.IsPersistent() ) { // // Wid is in a wordlist // fDocInPersIndex = FALSE; break; } } } // if/else( pRetrieve->IsDeletion() ) } // for loop if ( cDocInChunk == oRetrieve ) { // // All the documents in this chunk are in persistent // indexes. These can be deleted from the change log, // and the documents are added to the commited list // used in the push/simple filtering model, or they are // added to the aborted list used in push/simple filtering // model. // Win4Assert( fDocInPersIndex ); if ( _fPushFiltering ) { for ( ULONG iDoc = 0; iDoc < cDocInChunk; iDoc++ ) { CDocNotification * pDoc = Chunk.GetDoc(iDoc); if ( pDoc->Wid() != widInvalid ) { // // An invalid wid means that the changlog did not have enough // entries to fill Chunk and hence the remaining entries were set // to widinvalid. Hence an invalid wid can be ignored. // if ( _pResManager->LokIsWidAborted( pDoc->Wid(), pDoc->Usn() ) ) { // // Needs to be removed from the aborted wids list // notifTrans.RemoveAbortedWid( pDoc->Wid(), pDoc->Usn() ); } else { // // Needs to be added to the committed wids list // notifTrans.AddCommittedWid( pDoc->Wid() ); } } } } cRecord++; } } // while loop } // End transaction ciDebugOut( (DEB_ITRACE, "Truncating %d of %d record(s) from persistent changelog\n", cRecord, cTotRecord) ); if (cRecord > 0) { // // Shrink the change log stream from the front by "cRecord" amount. // CRcovStrmMDTrans xact( persDocQueue, CRcovStrmMDTrans::mdopFrontShrink, cRecord * (cDocInChunk * sizeof(CDocNotification) + CRcovStrmIter::SizeofChecksum()) ); cTotRecord -= cRecord; hdr.SetCount( hdr.GetBackup(), cTotRecord); xact.Commit(); // // Commit wids in push filtering model. This needs to be done after the CRcovSTrmMDTrans // because it should be done only if that transaction is successfully committed // notifTrans.Commit(); } // // Update the internal state of offset. // Win4Assert( cRecord <= _oChunkToRead ); _oChunkToRead -= cRecord; _cChunksTotal -= cRecord; LokVerifyConsistency(); return(cRecord); } //+--------------------------------------------------------------------------- // // Function: LokDisableUpdates // // Synopsis: Disable further updates to changelog // // History: 12-15-94 srikants Created // 2-24-97 SitaramR Push filtering // //---------------------------------------------------------------------------- void CChangeLog::LokDisableUpdates() { _fUpdatesEnabled = FALSE; if ( !_fPushFiltering ) { // // In pull (i.e. not push) filtering, reset the in-memory part of // changlog // _cChunksTotal = _cChunksAvail = _oChunkToRead = 0; LokVerifyConsistency(); } } //+--------------------------------------------------------------------------- // // Function: LokEnableUpdates // // Synopsis: Enables updates to changelog // // Arguments: [fFirstTimeUpdatesAreEnabled] -- Is this being called for the // first time ? // // History: 12-15-94 srikants Created // 2-24-97 SitaramR Push filtering // //---------------------------------------------------------------------------- void CChangeLog::LokEnableUpdates( BOOL fFirstTimeUpdatesAreEnabled ) { _fUpdatesEnabled = TRUE; if ( fFirstTimeUpdatesAreEnabled || !_fPushFiltering ) { // // In pull (i.e. not push) filtering, initialize the in-memory part // of changlog. Also, if EnableUpdates is being called for the first time, // then initialize the in-memory datastructure (i.e. do the // initialization in the case of push filtering also). // InitSize(); LokVerifyConsistency(); } } //+--------------------------------------------------------------------------- // // Function: SetResman // // Synopsis: Initializes ptr to resman // // Arguments: [pResManager] -- Resource manager // [fPushFiltering] -- Using push model of filtering ? // // History: 12-15-94 srikants Created // //---------------------------------------------------------------------------- void CChangeLog::SetResMan( CResManager * pResManager, BOOL fPushFiltering ) { _pResManager = pResManager; _fPushFiltering = fPushFiltering; if ( fPushFiltering ) { // // In push filtering, nuke the changelog because on shutdown all // client notifications are aborted, and it's the clients // responsibility to re-notify us after startup // CImpersonateSystem impersonate; PRcovStorageObj & persDocQueue = *_PersDocQueue; CRcovStrmWriteTrans xact( persDocQueue ); persDocQueue.GetHeader().SetCount(persDocQueue.GetHeader().GetBackup(), 0); xact.Empty(); xact.Commit(); Win4Assert( _oChunkToRead == 0 ); Win4Assert( _cChunksAvail == 0 ); Win4Assert( _cChunksTotal == 0 ); } } //+--------------------------------------------------------------------------- // // Function: IsWidInDocList // // Synopsis: Check if the given wid is in the doc list // // Arguments: [wid] -- Workid to check // [docList] -- Doclist // // History: 24-Feb-97 SitaramR Created // // Notes: Use simple sequential search because there are atmost 16 // wids in docList // //---------------------------------------------------------------------------- BOOL CChangeLog::IsWidInDocList( WORKID wid, CDocList & docList ) { for ( unsigned i=0; i