windows-server-2003/inetsrv/query/cindex/resman.cxx

//+---------------------------------------------------------------------------
//
//  Microsoft Windows
//  Copyright (C) Microsoft Corporation, 1991 - 2000.
//
//  File:       RESMAN.CXX
//
//  Contents:   Resource Manager
//
//  Classes:    CResManager
//
//  History:    08-Apr-91   BartoszM    Created
//              4-Jan-95    BartoszM    Separated Filter Manager
//              Jan-08-97   mohamedn    CFwEventItem and CDmFwEventItem
//              24-Feb-97   SitaramR    Push filtering
//
//----------------------------------------------------------------------------

#include <pch.cxx>
#pragma hdrstop

#include <cci.hxx>
#include <xact.hxx>
#include <pstore.hxx>
#include <cifailte.hxx>
#include <ciole.hxx>
#include <regevent.hxx>
#include <ciregkey.hxx>
#include <eventlog.hxx>
#include <cievtmsg.h>
#include <cifrmcom.hxx>
#include <fwevent.hxx>
#include <pidmap.hxx>
#include <identran.hxx>
#include <psavtrak.hxx>

#include "resman.hxx"
#include "ci.hxx"
#include "partn.hxx"
#include "pindex.hxx"
#include "mindex.hxx"
#include "idxids.hxx"
#include "indxact.hxx"
#include "merge.hxx"
#include "mmerglog.hxx"
#include "pendcur.hxx"
#include "fltstate.hxx"
#include "idle.hxx"
#include "notxact.hxx"
#include "lowres.hxx"

const ULONG lowDiskWaterMark = 3 * 512 * 1024; // 1.5 MB
const ULONG highDiskWaterMark = lowDiskWaterMark + 512 * 1024; // 2.0 MB
const ULONG minDiskFreeForMerge = lowDiskWaterMark;

class CRevertBoolValue {
public:
    CRevertBoolValue( BOOL & rfValue, BOOL fValue ) :
        _rfVal( rfValue ),
        _fRevert( TRUE )
    {
        Win4Assert( rfValue != fValue );
        _fPrevVal = _rfVal;
        _rfVal = fValue;
    }

    ~CRevertBoolValue()
    {
        if (_fRevert)
            _rfVal = _fPrevVal;
    }

    void Revert()
    {
        _rfVal = _fPrevVal;
        _fRevert = FALSE;
    }

private:
    BOOL &  _rfVal;
    BOOL    _fPrevVal;
    BOOL    _fRevert;
};


CMergeThreadKiller::CMergeThreadKiller( CResManager & resman )
    : _resman(resman), _fKill(TRUE)
{
}

CMergeThreadKiller::~CMergeThreadKiller()
{
    if ( _fKill )
    {
        _resman._fStopMerge = TRUE;
        _resman._thrMerge.Resume();
        _resman.StopMerges();
    }
}

//+---------------------------------------------------------------------------
//
//  Class:      CPendQueueTrans
//
//  Purpose:    Transaction for flushing the pending queue if there is a
//              failure while extracting entries out of the pending queue
//              and adding them to the changelog.
//
//  History:    9-11-95   srikants   Created
//
//----------------------------------------------------------------------------

class CPendQueueTrans : public CTransaction
{
public:

    CPendQueueTrans( CPendingQueue & pendQueue ) : _pendQueue( pendQueue )
    {
    }

    ~CPendQueueTrans()
    {
        if ( CTransaction::XActAbort == _status )
        {
            CLock lock( _pendQueue.GetMutex() );

            _pendQueue.LokFlushCompletedEntries();
        }
    }

private:

    CPendingQueue &     _pendQueue;

};

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::CResManager, public
//
//  Arguments:  [cat] -- catalog
//              [xact] -- transaction
//
//  History:    08-Apr-91   BartoszM    Created
//              Jan-07-96   mohamedn    CFwEventItem
//
//----------------------------------------------------------------------------

CResManager::CResManager(
        PStorage &storage,
        CCiFrameworkParams & params,
        ICiCDocStore * pDocStore,
        CI_STARTUP_INFO const & startupInfo,
        IPropertyMapper * pPropMapper,
        CTransaction& xact,
        XInterface<CIndexNotificationTable> & xIndexNotifTable )
:   _sigResman(eSigResman),
    _frmwrkParams( params ),
    _mergeTime(0),
    _storage ( storage ),
    _sKeyList(0),
    _idxTab( _storage.QueryIndexTable( xact ) ),
    _partList ( storage, *_idxTab, _sKeyList, xact, params ),
    _fresh ( storage, xact, _partList ),
    _partidToMerge ( partidInvalid ),
    _fStopMerge(FALSE),
    _pMerge(0),
#pragma warning( disable : 4355 )           // this used in base initialization
    _thrMerge ( MergeThread, this, TRUE ),  // create suspended
    _mergeKiller( *this ),
#pragma warning( default : 4355 )
    _activeDeletedIndex( _idxTab->GetDeletedIndex() ),
    _cQuery(0),
    _cFilteredDocuments(0),
    _pBackupWorkItem(0),
    _isBeingEmptied(FALSE),
    _isLowOnDiskSpace(FALSE),
    _isDismounted(FALSE),
    _isOutOfDate(FALSE),
    _isCorrupt(FALSE),
    _fFirstTimeUpdatesAreEnabled( TRUE ),
    _fPushFiltering( FALSE ),
    _fFlushWorkerActive( FALSE ),
    _configFlags( startupInfo.startupFlags ),
    _xIndexNotifTable( xIndexNotifTable.Acquire() ),
    _dwFilteringState( 0 ),
    _pFilterAgent( 0 )
{
    Win4Assert( 0 != pDocStore );

    //
    // Look for a resource monitor.  If none exists, use the default.
    //

    _xLowResDefault.Set( new CLowRes(_frmwrkParams) );

    SCODE sc = pDocStore->QueryInterface( IID_ICiCResourceMonitor,
                                          _xLowRes.GetQIPointer() );

    if ( FAILED(sc) )
    {
        _xLowResDefault->AddRef();
        _xLowRes.Set( _xLowResDefault.GetPointer() );
    }

    //


    //
    // The presence or absence of an actual pointer in _xIndexNotifTable,
    // indicates whether the client has specified pull filtering or push
    // filtering.
    //

    _fPushFiltering = !_xIndexNotifTable.IsNull();

    //
    // Initialize the resman in all the partitions.
    //
    CPartIter iter;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        CPartition * pPart = iter.LokGet();
        pPart->SetResMan( this, FPushFiltering() );
    }

    if ( _fPushFiltering )
    {
        //
        // In push filtering, an identity translator is used
        //

        CIdentityNameTranslator *pIdentityTrans = new CIdentityNameTranslator();

        sc = pIdentityTrans->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
                                             _translator.GetQIPointer() );
        Win4Assert( SUCCEEDED( sc ) );

        _translator->Release();              // QI does an AddRef
    }
    else
    {
        //
        // In pull filtering, the client provides the translator, which may
        // be the Ex version.
        //

        sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslatorEx,
                                        _translator2.GetQIPointer() );

        if ( SUCCEEDED(sc) )
            sc = _translator2->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
                                               _translator.GetQIPointer() );
        else
            sc = pDocStore->QueryInterface( IID_ICiCDocNameToWorkidTranslator,
                                            _translator.GetQIPointer() );
    }

    if ( S_OK != sc )
        THROW( CException(sc) );

    //
    // Create a workIdToDocName converter object.
    //
    _xWorkidToDocName.Set( new CWorkIdToDocName( _translator.GetPointer(), _translator2.GetPointer() ) );

    ICiCAdviseStatus * pAdviseStatus;
    sc = pDocStore->QueryInterface( IID_ICiCAdviseStatus,
                                    (void **) & pAdviseStatus );
    if ( S_OK != sc )
        THROW( CException(sc) );

    _adviseStatus.Set( pAdviseStatus );
    _prfCounter.SetAdviseStatus( pAdviseStatus );

    ICiCPropertyStorage * pPropStore;
    sc = pDocStore->QueryInterface( IID_ICiCPropertyStorage,
                                    (void **) & pPropStore );
    if ( S_OK != sc )
        THROW( CException(sc) );

    _propStore.Set( pPropStore );

    pDocStore->AddRef();
    _docStore.Set( pDocStore );

    pPropMapper->AddRef();
    _mapper.Set( pPropMapper );

    ULONG mergeTime = _frmwrkParams.GetMasterMergeTime();
    _mergeTime = CMasterMergePolicy::ComputeMidNightMergeTime(mergeTime);

    if ( -1 == _mergeTime )
    {
       CFwEventItem   item(EVENTLOG_ERROR_TYPE,
                           MSG_CI_BAD_SYSTEM_TIME,
                           0);

       item.ReportEvent( _adviseStatus.GetReference() );

       THROW( CException( STATUS_INVALID_PARAMETER ) );
    }

    //
    // Restore the information associated with any master merge that
    // was stopped when the drive was dismounted.
    //
    _partList.RestoreMMergeState(*this, _storage);

    _partIter.LokInit( _partList );

    _isLowOnDiskSpace = LokCheckIfDiskLow( *this,
                                           _storage,
                                           _isLowOnDiskSpace,
                                           _adviseStatus.GetReference() );

    _thrMerge.SetPriority ( _frmwrkParams.GetThreadPriorityMerge() );

    _mergeKiller.Defuse();
    _thrMerge.Resume();

    //
    //  Set the merge progress indicator to 0, and refresh other counters
    //

    _prfCounter.Update( CI_PERF_MERGE_PROGRESS, 0 );

    LokUpdateCounters();

    //
    // Enable or disables updates/notifications based on disk space available
    // and whether the catalog is readonly
    //
    if ( _isLowOnDiskSpace || _storage.IsReadOnly() )
    {
        DisableUpdates( partidDefault );
        _state.LokSetState( eUpdatesDisabled );
    }
    else
    {
        EnableUpdates( partidDefault );
        SCODE sc = _docStore->EnableUpdates();
        if ( SUCCEEDED( sc ) )
           _state.LokSetState( eSteady );
        else
           _state.LokSetState( eUpdatesToBeEnabled );
    }
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::~CResManager, public
//
//  Synopsis:   Shuts down merge(s)
//
//  History:    13-Aug-93 KyleP     Created
//
//--------------------------------------------------------------------------

CResManager::~CResManager()
{
    _workMan.AbortWorkItems();
    LokDeleteFlushWorkItems();
    _workMan.WaitForDeath();

    StopMerges();

    #if CIDBG==1
    // ======================================
    {
        CPriLock    lock(_mutex);
        Win4Assert( 0 == _pBackupWorkItem );
    }
    // ======================================
    #endif  // CIDBG==1
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::MergeThread, public
//
//  Synopsis:   Entry point for the thread responsible
//              for asynchronous merges
//
//  History:    05-Mar-92   BartoszM    Created
//
//----------------------------------------------------------------------------
DWORD WINAPI CResManager::MergeThread( void* self )
{
    if ( !((CResManager*)self)->_fStopMerge )
        ((CResManager*)self)->DoMerges();

    return 0;
}

//==============
// STATE CHANGES
//==============

//+---------------------------------------------------------------------------
//
//  Function:   Dismount
//
//  Synopsis:   Dismounts the catalog by stopping any in-progress merge
//              and also finishing off any pending writes ( eg. ChangeLog).
//
//  History:    6-20-94   srikants   Created
//
//----------------------------------------------------------------------------

NTSTATUS CResManager::Dismount()
{
    ciDebugOut(( DEB_ITRACE, "*** CI: Initiating DisMount ***\n" ));

    NTSTATUS status = STATUS_SUCCESS;

    TRY
    {
        //
        // Inform all the partitions to cleanup. We have to abort any
        // in-progress merges without having to take a lock. There could
        // be another thread (like the filter thread) with the lock doing
        // a long operation. In one case we had it trying to create a new
        // fresh test and the memory writer could not make any progress
        // because the merge was continuing to dirty the pages.
        //

        // There is no need to take a lock because the partitions are not
        // going away and the operation we are about to do is just turning
        // on a one-way flag.
        //
        {
            PARTITIONID partid = 1;
            CPartition* pPart = _partList.LokGetPartition ( partid );
            pPart->PrepareForCleanup();
        }

        //
        // Stop in progress merges.
        //

        StopMerges();

        //
        // Dismount each of the partitions.
        //
        // ================================================
        {
            CPriLock lock( _mutex );
            Win4Assert( !_isDismounted );

            _isDismounted = TRUE;

            CKeyList * pKeyList = _sKeyList.Acquire();
            delete pKeyList;

            CPartIter iter;
            for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
            {
                CPartition * pPart = iter.LokGet();

                // =======================================================
                CChangeTrans  xact( *this, pPart );

                if ( STATUS_SUCCESS == pPart->LokDismount( xact ) )
                {
                    xact.LokCommit();
                }
                //=========================================================
            }

            if ( _pBackupWorkItem )
                _pBackupWorkItem->GetSaveProgressTracker().SetAbort();
        }
        // ================================================

        _workMan.WaitForDeath();

    }
    CATCH( CException, e )
    {
        status = e.GetErrorCode();
        ciDebugOut(( DEB_ERROR,
            "ContentIndex Dismount Failed. Error Code 0x%X\n", status ));
    }
    END_CATCH

    ciDebugOut(( DEB_ITRACE, "*** CI: Completed DisMount ***\n" ));

    return status;
}

//+---------------------------------------------------------------------------
//
//  Function:   CResManager::Empty, public
//
//  Synopsis:   Releases resources associated with the resman object.  This
//              includes all indexes, the persistent freshlog, the persistant
//              changelog, the freshtest, and the master merge log.
//
//  History:    15-Aug-94   DwightKr    Created
//
//----------------------------------------------------------------------------
void CResManager::Empty()
{
    CPriLock lock( _mutex );

    _isBeingEmptied = TRUE;

    //
    //  Cancel any connections to the CI Filter service.
    //
    _pFilterAgent->LokCancel();

    //
    //  Delete everything from the index table here first.
    //
    _idxTab->LokEmpty();

    //
    //  If anything fails after this point, chkdsk /f or autochk will
    //  release the disk storage associated with the leaked objects
    //  since they are no longer part of the persistent index list.
    //

    //
    //  For each partition, zombify all indexes.
    //
    CPartIter iter;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        CPartition* pPart = iter.LokGet();

        //
        //  Zombify all indexes in this partition, and delete them if their
        //  ref-count is 0.
        //
        CIndex ** aIndex = 0;
        TRY
        {
            unsigned cIndex;
            aIndex = pPart->LokZombify( cIndex );

            ReleaseIndexes( cIndex, aIndex, NULL );
        }
        CATCH ( CException, e )
        {
        }
        END_CATCH

        delete [] aIndex;

        //
        //  Delete the change log associated with this partition.  These
        //  routines do not throw exceptions.
        //
        WORKID widChangeLog = _partList.GetChangeLogObjectId( pPart->GetId() );
        if ( widChangeLog != widInvalid)
            _storage.RemoveObject( widChangeLog );

        WORKID widMMergeLog;
        WORKID widDummy;
        pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy );
        if ( widMMergeLog != widInvalid)
            _storage.RemoveObject( widMMergeLog );
    }

    _fresh.LokEmpty();          // Release storage associated with fresh test

    WORKID widFreshLog  = _storage.GetSpecialItObjectId( itFreshLog );

#ifdef KEYLIST_ENABLED
    WORKID widKeyList   = _storage.GetSpecialItObjectId( itMMKeyList );
#else
    WORKID widKeyList   = widInvalid;
#endif  //

    WORKID widPhrLat    = _storage.GetSpecialItObjectId( itPhraseLat );

    if ( widFreshLog != widInvalid)
        _storage.RemoveObject( widFreshLog );
    if ( widKeyList != widInvalid)
        _storage.RemoveObject( widKeyList );
    if ( widPhrLat != widInvalid)
        _storage.RemoveObject( widPhrLat );
}

//+---------------------------------------------------------------------------
//
//  Function:   StopCurrentMerge
//
//  Synopsis:   Aborts any in-progress merge.
//
//  History:    5-04-94   srikants   Created
//
//----------------------------------------------------------------------------

NTSTATUS CResManager::StopCurrentMerge()
{
    {
        CPriLock lock( _mutex );

        //
        // If we're doing a merge right now then kill it off.
        //

        if ( 0 != _pMerge )
            _pMerge->LokAbort();

    }

    return STATUS_SUCCESS;
}

//+---------------------------------------------------------------------------
//
//  Member:      CResManager::DisableUpdates, public
//
//  Arguments:   [partid] -- partition id
//
//  History:     24-Dec-94  Anonymous   Created
//
//----------------------------------------------------------------------------
void CResManager::DisableUpdates( PARTITIONID partid )
{
    CPriLock lock( _mutex );
    CPartition * pPart = _partList.LokGetPartition( partid );

    Win4Assert( 0 != pPart );
    pPart->LokDisableUpdates();
}


//+---------------------------------------------------------------------------
//
//  Member:      CResManager::EnableUpdates, public
//
//  Arguments:   [partid]    -- partition id
//
//  History:     24-Dec-94  Anonymous   Created
//
//----------------------------------------------------------------------------
void CResManager::EnableUpdates( PARTITIONID partid )
{
    CPriLock lock( _mutex );
    CPartition * pPart = _partList.LokGetPartition( partid );

    Win4Assert( 0 != pPart );
    pPart->LokEnableUpdates( _fFirstTimeUpdatesAreEnabled );

    _fFirstTimeUpdatesAreEnabled = FALSE;
}


//+---------------------------------------------------------------------------
//
//  Function:   LokCheckIfDiskLow
//
//  Synopsis:   Checks if we are running low on free disk space. It has
//              a "LowWaterMark" and a "HighWaterMark".
//
//              If current state is "not full", then it will check to
//              see if the free disk space is < the LowWaterMark.
//
//              if current state is "full", then it will check to see if
//              the free disk space is < the HighWaterMark.
//
//              The HighWaterMark is > the LowWaterMark to prevent hysterisis.
//
//  Arguments:  [resman]         -- Resource manager
//              [storage]        -- Storage object.
//              [fCurrent]       -- Current state.
//              [adviseStatus]   -- reference to ICiCAdviseStatus
//
//  Returns:    TRUE if we are running low on disk space.
//              FALSE otherwise.
//
//  History:    10-11-94   srikants   Created
//              Jan-07-96  mohamedn   CFwEventItem
//
//----------------------------------------------------------------------------

BOOL LokCheckIfDiskLow( CResManager & resman,
                        PStorage & storage,
                        BOOL fIsCurrentFull,
                        ICiCAdviseStatus & adviseStatus )
{
    BOOL fLowOnDisk = fIsCurrentFull;

    TRY
    {
        __int64 diskTotal, diskRemaining;
        storage.GetDiskSpace( diskTotal, diskRemaining );

        if ( !fIsCurrentFull )
            fLowOnDisk = diskRemaining < lowDiskWaterMark;
        else
            fLowOnDisk = diskRemaining < highDiskWaterMark;

        //
        // It is okay to read it without mutex as it is only a heuristic.
        //
        if ( fLowOnDisk && !fIsCurrentFull && !storage.IsReadOnly() )
        {
            ciDebugOut(( DEB_WARN, "****YOU ARE RUNNING LOW ON DISK SPACE****\n"));
            ciDebugOut(( DEB_WARN, "****PLEASE FREE UP SOME SPACE        ****\n"));

            ULONG mbToFree = highDiskWaterMark/(1024*1024);

            ULONG mbIndex;
            resman.IndexSize( mbIndex );
            mbToFree = max( mbToFree, mbIndex );
            mbToFree = min( mbToFree, 50 ); // don't ask for more than 50 MB

            CFwEventItem item(  EVENTLOG_AUDIT_FAILURE,
                                MSG_CI_LOW_DISK_SPACE,
                                2);

            item.AddArg(storage.GetVolumeName() );
                        item.AddArg(mbToFree);

            item.ReportEvent(adviseStatus);
        }
        else if ( fIsCurrentFull && !fLowOnDisk )
        {
            ciDebugOut(( DEB_WARN, "****DISK SPACE FREED UP              ****\n"));
        }
    }
    CATCH( CException,e )
    {
        ciDebugOut(( DEB_ERROR,
                     "Error 0x%X while getting disk space info\n",
                     e.GetErrorCode() ));
    }
    END_CATCH

    return ( fLowOnDisk );
}


//+---------------------------------------------------------------------------
//
//  Function:   NoFailFreeResources
//
//  Synopsis:   deletes a big wordlist and schedules refiltering
//
//  History:    6-Jan-95    BartoszM    Created
//
//----------------------------------------------------------------------------

void CResManager::NoFailFreeResources()
{
    ciDebugOut(( DEB_WARN, "Running out of resources.  " ));
    TRY
    {

        PARTITIONID partId = 1;

        CPriLock   lock(_mutex);
        CPartition * pPart = LokGetPartition( partId );


        //
        // Before calling remove word list, we must append any refiled
        // documents to the doc queue. This is because, the docqueue
        // can handle only one set of refiled documents.
        //
        {
            // =========================================
            CChangeTrans    xact( *this, pPart );
            pPart->LokAppendRefiledDocs( xact );
            xact.LokCommit();
            // =========================================
        }

        LokNoFailRemoveWordlist( pPart );
    }
    CATCH( CException, e )
    {
    }
    END_CATCH
}


//+-------------------------------------------------------------------------
//
//  Member:     CResManager::IsMemoryLow, private
//
//  Returns:    TRUE if we're in a low memory situation
//
//  History:    10-May-93 AmyA      Created from old DoUpdates
//               3-May-96 dlee      #if 0'ed it out because the memory
//                                  load # isn't reliable, and we've
//                                  got LOTS of other allocations anyway.
//              05-Nov-97 KyleP     New approach
//
//--------------------------------------------------------------------------

BOOL CResManager::IsMemoryLow()
{
    SCODE sc = _xLowRes->IsMemoryLow();

    if ( E_NOTIMPL == sc )
        sc = _xLowResDefault->IsMemoryLow();

    return ( S_OK == sc );
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::IsIoHigh, private
//
//  Returns:    TRUE if the system is performing a 'lot' of I/O
//
//  History:    10-Dec-97  KyleP    Created
//
//  Notes:      This call takes time (~ 5 sec) to complete.
//
//--------------------------------------------------------------------------

BOOL CResManager::IsIoHigh()
{
    SCODE sc = _xLowRes->IsIoHigh( &_fStopMerge );

    if ( E_NOTIMPL == sc )
        sc = _xLowResDefault->IsIoHigh( &_fStopMerge );

    return ( S_OK == sc );
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::IsBatteryLow, private
//
//  Returns:    TRUE if the system is running low on battery power.
//
//  History:    16-Jul-98  KyleP    Created
//
//  Notes:      By default, even 100% battery (as opposed to A/C) may be
//              considered low.
//
//--------------------------------------------------------------------------

BOOL CResManager::IsBatteryLow()
{
    SCODE sc = _xLowRes->IsBatteryLow();

    if ( E_NOTIMPL == sc )
        sc = _xLowResDefault->IsBatteryLow();

    return ( S_OK == sc );
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::IsOnBatteryPower, private
//
//  Returns:    TRUE if the system is running on battery power.
//
//  History:    01-Oct-00  dlee    Created
//
//--------------------------------------------------------------------------

BOOL CResManager::IsOnBatteryPower()
{
    SCODE sc = _xLowRes->IsOnBatteryPower();

    if ( E_NOTIMPL == sc )
        sc = _xLowResDefault->IsOnBatteryPower();

    return ( S_OK == sc );
} //IsOnBatteryPower

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::SampleUserActivity, private
//
//  Returns:    - nothing -
//
//  History:    31 Jul 98  AlanW    Created
//
//--------------------------------------------------------------------------

void CResManager::SampleUserActivity()
{
    SCODE sc = _xLowRes->SampleUserActivity();

    if ( E_NOTIMPL == sc )
        _xLowResDefault->SampleUserActivity();

    return;
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::IsUserActive, private
//
//  Arguments:  [fCheckLongTerm] - TRUE if long-term activity should be checked.
//
//  Returns:    TRUE if the interactive user is using the keyboard or mouse
//
//  History:    28 Jul 98  AlanW    Created
//
//--------------------------------------------------------------------------

BOOL CResManager::IsUserActive(BOOL fCheckLongTerm)
{
    SCODE sc = _xLowRes->IsUserActive(fCheckLongTerm);

    if ( E_NOTIMPL == sc )
        sc = _xLowResDefault->IsUserActive(fCheckLongTerm);

    return ( S_OK == sc );
}

//======================
//  STATE AND STATISTICS
//======================

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::CountPendingUpdates, public
//
//  Arguments:  [secCount (output)] -- Count of secondary Q Updates
//
//  Returns:    Count of pending updates.
//
//  History:    14-Dec-92 KyleP     Created
//
//  Notes:      If wordlists are being constructed the count may be low.
//
//--------------------------------------------------------------------------

unsigned CResManager::CountPendingUpdates(unsigned& secCount)
{
    CPriLock lock( _mutex );
    unsigned count = 0;
    CPartIter iter;
    secCount = 0;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        count += iter.LokGet()->LokCountUpdates();

        secCount += iter.LokGet()->LokCountSecUpdates();
    }
    return(count + secCount);
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::UpdateCounters, public
//
//  Returns:    Updates performance counters.
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

void CResManager::LokUpdateCounters()
{
    if ( !_isDismounted )
    {
        _prfCounter.Update( CI_PERF_NUM_WORDLIST, _partList.LokWlCount() );

        unsigned cSecQDocuments;
        unsigned cUnfilteredDocs = CountPendingUpdates( cSecQDocuments );
        _prfCounter.Update( CI_PERF_FILES_TO_BE_FILTERED, cUnfilteredDocs );
        _prfCounter.Update( CI_PERF_DEFERRED_FILTER_FILES, cSecQDocuments );

        _prfCounter.Update( CI_PERF_NUM_UNIQUE_KEY, _sKeyList->MaxKeyIdInUse() );
        _prfCounter.Update( CI_PERF_DOCUMENTS_FILTERED, _cFilteredDocuments );

        ULONG cDocuments;
        GetClientState( cDocuments );
        _prfCounter.Update( CI_PERF_NUM_DOCUMENTS, cDocuments );
    }
}

//====================
// UPDATES AND QUERIES
//====================

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::ReserveUpdate, public
//
//  Arguments:  [wid] -- Used to confirm hint.
//
//  Returns:    Hint to position of reserved slot.
//
//  History:    30-Aug-95   KyleP       Created
//
//----------------------------------------------------------------------------

unsigned CResManager::ReserveUpdate( WORKID wid )
{
    CPriLock lock ( _pendQueue.GetMutex() );

    return _pendQueue.LokPrepare( wid );
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::UpdateDocument, public
//
//  Arguments:  [iHint]    -- Positional hint
//              [wid]      -- wid to add/delete
//              [partid]   -- partition containing this wid
//              [usn]      -- USN associated with change
//              [volumeId] -- Volume Id
//              [action]   -- addition / deletion
//
//  History:    08-Apr-91   BartoszM    Created
//              08-Oct-93   BartoszM    Rewrote to accept single document
//              30-Aug-95   KyleP       Use reserved slots
//
//----------------------------------------------------------------------------

SCODE CResManager::UpdateDocument( unsigned iHint,
                                   WORKID wid,
                                   PARTITIONID partid,
                                   USN usn,
                                   VOLUMEID volumeId,
                                   ULONG action )
{
    if ( !IsIndexingEnabled() )
        return CI_E_FILTERING_DISABLED;

    Win4Assert ( partid == 1 );

    //
    // Try to avoid lock contention
    // with filter agent
    //
    _pFilterAgent->SlowDown();


    CPriLock lock ( _mutex );

    BOOL fComplete;

    {
        CPriLock lock2( _pendQueue.GetMutex() );

        if ( _isBeingEmptied )          // The content index is empty
        {
            //
            // Just get rid of any pending entries in the queue.
            //
            if ( !_pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action ) )
            {
                while ( _pendQueue.LokRemove( wid, usn, volumeId, partid, action ) )
                    ;
            }
            return CI_E_SHUTDOWN;
        }

        //
        // The if clause triggers when this document is the only one
        // on the queue.
        //

        fComplete = _pendQueue.LokComplete( iHint, wid, usn, volumeId, partid, action );
    }

    SCODE sc = S_OK;

    if ( fComplete )
    {
        CPartition* pPart = _partList.LokGetPartition ( partid );

        CChangeTrans xact( *this, pPart );
        sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 );
        xact.LokCommit();

        _pFilterAgent->LokWakeUp();
    }
    else
    {
        BOOL fGotOne = FALSE;

        CPendQueueTrans  pendQueueTrans( _pendQueue );

        while ( TRUE )
        {
            BOOL fGotAnother;

            {
                CLock lock2( _pendQueue.GetMutex() );
                fGotAnother = _pendQueue.LokRemove( wid, usn, volumeId, partid, action );
            }

            if ( fGotAnother )
            {
                fGotOne = TRUE;

                CPartition* pPart = _partList.LokGetPartition ( partid );

                CChangeTrans xact( *this, pPart );
                sc = pPart->LokUpdateDocument ( xact, wid, usn, volumeId, action, 1, 0 );
                xact.LokCommit();
            }
            else
                break;
        }

        pendQueueTrans.Commit();

        if ( fGotOne )
            _pFilterAgent->LokWakeUp();
    }

    return sc;
}


//+---------------------------------------------------------------------------
//
//  Member:     CResManager::FlushUpdates
//
//  Synopsis:   Flushes all update notifications to disk
//
//  History:    27-Jun-97   SitaramR   Created
//
//----------------------------------------------------------------------------

void CResManager::FlushUpdates()
{
    CPriLock lock( _mutex );
    CPartition * pPart = _partList.LokGetPartition( partidDefault );

    Win4Assert( 0 != pPart );

    pPart->LokFlushUpdates();
}


//+---------------------------------------------------------------------------
//
//  Member:     CResManager::QueryIndexes, public
//
//  Arguments:  [cPartitions]     -- count of partitions
//              [aPartID]         -- array of partition id's
//              [freshTest]       -- return arg.: fresh test
//              [cInd]            -- return arg: count of indexes
//              [cPendingUpdates] -- Pending update 'threshold'.  If fewer
//                                   pending updates, the pending wids are
//                                   returned in *pcurPending.
//              [pcurPending]     -- Pending cursors stored here.
//              [pFlags]          -- return arg: status of indexes
//
//  Returns:    Array of pointers to indexes
//
//  History:    08-Oct-91   BartoszM    Created
//
//  Notes:      Called by Query
//              Indexes and fresh test have ref count increased
//              Flags may change to indicate status of indexes
//
//----------------------------------------------------------------------------

CIndex** CResManager::QueryIndexes (
    unsigned cPartitions,
    PARTITIONID aPartID[],
    CFreshTest** freshTest,
    unsigned& cInd,
    ULONG cPendingUpdates,
    CCurStack * pcurPending,
    ULONG* pFlags )
{

    unsigned count = 0;
    CPartition* pPart;

    CPriLock lock ( _mutex );

    if ( _isCorrupt )
        THROW( CException( CI_CORRUPT_DATABASE ) );

    if ( _isBeingEmptied )   // Content index is empty or corrupt
    {
        cInd = 0;
        cPartitions = 0;

        return 0;
    }

    for ( unsigned i = 0; i < cPartitions; i++ )
    {
        pPart = _partList.LokGetPartition( aPartID [i] );
        count += pPart->LokIndexCount();
    }

    XArray<CIndex *> apIndex( count );

#if CIDBG || DBG
    unsigned j = 0;
#endif

    for ( i = 0; i < cPartitions; i++ )
    {
        pPart = _partList.LokGetPartition(aPartID [i]);

#if CIDBG || DBG
        j +=
#endif // CIDBG || DBG
        pPart->LokGetIndexes ( &apIndex[i] );

        if ( LokIsScanNeeded() && !_storage.IsReadOnly() )
            *pFlags |=  CI_NOT_UP_TO_DATE;

        //
        // If we're looking for pending updates, get them.  Otherwise just
        // set the out-of-date flag.
        //

        unsigned cPending = pPart->LokCountUpdates();

        cPending += pPart->LokCountSecUpdates();

        if ( cPending != 0 )
        {
            ULONG fFlags = CI_NOT_UP_TO_DATE;   // assume non-success

            if ( cPendingUpdates > 0 )
            {
                Win4Assert( 0 != pcurPending );

                if ( cPending <= cPendingUpdates )
                {
                    XArray<WORKID> pWid( cPending );

                    BOOL fSucceeded = pPart->LokGetPendingUpdates(
                                                    pWid.GetPointer(),
                                                    cPending );
                    if ( fSucceeded )
                    {
                        Win4Assert( cPending > 0 );

                        XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, cPending ) );
                        pcurPending->Push( xCursor.GetPointer() );
                        xCursor.Acquire();
                        fFlags = 0;
                    }
                }
            }
            *pFlags |= fFlags;
        }
    }

    //
    // Also check to see if we are actually in the process of filtering some
    // documents.
    //
    if ( _docList.Count() > 0 )
    {
        if ( _docList.Count() <= cPendingUpdates )
        {
            Win4Assert( 0 != pcurPending );

            XArray<WORKID> pWid( _docList.Count() );

            ciDebugOut((DEB_FILTERWIDS,
                       "CResManager::QueryIndexes - pending documents: %d %x\n",
                        _docList.Count(), pWid.GetPointer()));

            _docList.LokGetWids( pWid );
            XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, _docList.Count() ) );
            pcurPending->Push( xCursor.GetPointer() );
            xCursor.Acquire();
        }
        else
            *pFlags |= CI_NOT_UP_TO_DATE;
    }

    //
    // Also check documents pending notifcation that may be complete but
    // out-of-order.
    //

    {
        CPriLock lock( _pendQueue.GetMutex() );

        unsigned cwidPending = _pendQueue.LokCountCompleted();

        if ( cwidPending > 0 )
        {
            if ( cwidPending <= cPendingUpdates )
            {
                Win4Assert( 0 != pcurPending );

                XArray<WORKID> pWid( cwidPending );

                _pendQueue.LokGetCompleted( pWid.GetPointer() );

                XPtr<CPendingCursor> xCursor( new CPendingCursor( pWid, cwidPending ) );
                pcurPending->Push( xCursor.GetPointer() );
                xCursor.Acquire();
            }
            else
                *pFlags |= CI_NOT_UP_TO_DATE;
        }
    }

    //
    // Finally, check for pending scans.
    //

    if ( _isOutOfDate )
        *pFlags |= CI_NOT_UP_TO_DATE;

    Win4Assert ( j == count );

    *freshTest = _fresh.LokGetFreshTest();

    cInd = count;

    return apIndex.Acquire();
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::ReleaseIndexes, public
//
//  Synopsis:   Decrements ref counts, deletes if indexes
//              marked to be deleted.
//
//  Arguments:  [cInd] -- count of indexes
//              [apIndex] -- array of indexes
//              [freshTest] -- fresh test
//
//  History:    08-Oct-91   BartoszM    Created
//
//  Notes:      Takes ResMan lock
//
//----------------------------------------------------------------------------

void CResManager::ReleaseIndexes ( unsigned cInd, CIndex** apIndex,
    CFreshTest* pFreshTest )
{
    ciDebugOut (( DEB_ITRACE, "Release Indexes\n" ));

    CPriLock lock ( _mutex );

    for ( unsigned i = 0; i < cInd; i++ )
    {
        if ( apIndex[i] != 0 )
            LokReleaseIndex ( apIndex[i] );
    }

    if ( pFreshTest)
        _fresh.LokReleaseFreshTest (pFreshTest);
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::BackupContentIndexData
//
//  Synopsis:   Private method called by the merge thread to backup the
//              content index data. If successful, it sets the status of
//              the backup work item to indicate success. Otherwise, it
//              must be considered to have failed.
//
//  History:    3-18-97   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::BackupContentIndexData()
{
    Win4Assert( 0 != _pBackupWorkItem );

    ciDebugOut(( DEB_WARN, "Starting Backup of Ci Data\n" ));

    //
    // Create the backup object.
    //
    CPartition *pPartition =_partList.LokGetPartition ( partidDefault );

    CBackupCiPersData   backup( *_pBackupWorkItem,
                                *this,
                                *pPartition );

    // =========================================
    {
        CPriLock    lock(_mutex);
        backup.LokGrabResources();
    }
    // =========================================

    backup.BackupIndexes();

    // =========================================
    {
        CPriLock    lock(_mutex);
        backup.LokBackupMetaInfo();
        ciDebugOut(( DEB_WARN, "Completed Backup of Ci Data\n" ));
        _pBackupWorkItem->LokSetStatus( STATUS_SUCCESS );
    }
    // =========================================
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokUpdateBackupMergeProgress
//
//  Synopsis:   Updates the backup progress during the merge.
//
//  History:    3-20-97   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokUpdateBackupMergeProgress()
{
    Win4Assert( 0 != _pBackupWorkItem );

    CCiFrmPerfCounter counter( _adviseStatus.GetPointer(),
                               CI_PERF_MERGE_PROGRESS );
    DWORD dwMergeProgress = counter.GetCurrValue();

    //
    // Assuming Merge is half of the work, we will assume half
    // progress. So make the denominator 200 instead of 100.
    //
    if ( _pMerge )
    {
        const cShadowMergePart = 5;

        if ( _pBackupWorkItem->IsFullSave() )
        {
            if ( _pMerge->GetMergeType() != mtMaster )
            {
                //
                // The shadow merge preceding a master is approx. 5%
                // say.
                //
                dwMergeProgress = cShadowMergePart;
            }
            else
            {
                dwMergeProgress = min(dwMergeProgress + cShadowMergePart, 100);
            }
        }

        _pBackupWorkItem->LokUpdateMergeProgress( (ULONG) dwMergeProgress, 100 );
    }
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::BackupCIData
//
//  Synopsis:   External method to backup content index data. It creates
//              a backup work item and waits for the backup to complete.
//
//  Arguments:  [storage]         - Destination storage
//              [fFull]           - [in/out] Set to TRUE if a full merge is
//              needed on input; On output will be set to TRUE if a full
//              merge was done.
//              [progressTracker] - Progress tracker.
//
//  History:    3-18-97   srikants   Created
//
//----------------------------------------------------------------------------

SCODE CResManager::BackupCIData( PStorage & storage,
                                 BOOL & fFull,
                                 XInterface<ICiEnumWorkids> & xEnumWorkids,
                                 PSaveProgressTracker & progressTracker )
{
    if ( !IsIndexMigrationEnabled() || !IsIndexingEnabled() )
        return CI_E_INVALID_STATE;

    SCODE sc = S_OK;

    Win4Assert( fFull );

    TRY
    {

        // ===================================================
        {
            CLock   lock(_mutex);
            if ( _isDismounted )
                THROW( CException( CI_E_SHUTDOWN ) );

            // There can be only one operation going on at a time.

            if ( 0 != _pBackupWorkItem )
                THROW( CException( CI_E_INVALID_STATE ) );

            _pBackupWorkItem = new CBackupCiWorkItem( storage,
                                                      fFull,
                                                      progressTracker );

            _eventMerge.Set();
        }
        // ===================================================

        ciDebugOut(( DEB_WARN, "Waiting for backup to complete\n" )) ;
        //
        // Wait for the work-item to be completed.
        //
        while ( TRUE )
        {
            DWORD const dwWaitTime = 1 * 60 * 1000; // 1 minute in millisecs
            _pBackupWorkItem->WaitForCompletion( dwWaitTime );

            //============================================================
            {
                CPriLock    lock(_mutex);
                if ( _pBackupWorkItem->LokIsDone() )
                    break;

                //
                // If there is a merge going on, we must update the progress.
                //
                if ( _pBackupWorkItem->LokIsDoingMerge() )
                    LokUpdateBackupMergeProgress();

                _pBackupWorkItem->LokReset();
            }
            //============================================================

        } // of while


        sc= _pBackupWorkItem->GetStatus();
        ciDebugOut(( DEB_WARN, "Backup completed with code 0x%X\n", sc ));
        if ( S_OK == sc )
        {
            fFull = _pBackupWorkItem->IsFullSave();
            xEnumWorkids.Set( _pBackupWorkItem->AcquireWorkIdEnum() );
        }

        // ====================================
        {
            CLock   lock(_mutex);
            delete _pBackupWorkItem;
            _pBackupWorkItem = 0;
        }
        // ====================================
    }
    CATCH( CException,e )
    {
        sc = HRESULT_FROM_WIN32( e.GetErrorCode() );
        Win4Assert( !"How Did we Come Here" );
    }
    END_CATCH

    return sc;
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::CiState, public
//
//  Arguments:  [state] -- internal state of the CI
//
//  History:    01-Nov-95   DwightKr    Created
//
//
//----------------------------------------------------------------------------

#define setState( field, value ) \
    if ( state.cbStruct >= ( offsetof( CIF_STATE, field) +   \
                             sizeof( state.field ) ) )       \
    {                                                        \
        state.field = ( value );                             \
    }

#define roundup(a, b) ((a%b) ? (a/b + 1) : (a/b))

NTSTATUS CResManager::CiState(CIF_STATE & state)
{
    CPriLock lock( _mutex );

    if ( _isDismounted )
        return CI_E_SHUTDOWN;

    if ( _isCorrupt )
        return CI_CORRUPT_DATABASE;

    setState( cWordList, _partList.LokWlCount() );
    setState( cPersistentIndex, _partList.LokIndexCount() - state.cWordList );

    // Get the perf counter for the # of queries executed

    Win4Assert( !_adviseStatus.IsNull() );
    long cQ;
    _adviseStatus->GetPerfCounterValue( CI_PERF_TOTAL_QUERIES, &cQ );
    setState( cQueries, cQ );

    unsigned cSecQDocuments;
    ULONG cUnfilteredDocs = CountPendingUpdates(cSecQDocuments) + _docList.Count();
    setState( cDocuments, cUnfilteredDocs );
    setState( cSecQDocuments, cSecQDocuments );
    setState( cFreshTest, LokGetFreshCount() );

    CCiFrmPerfCounter counter( _adviseStatus.GetPointer(),
                               CI_PERF_MERGE_PROGRESS );
    setState( dwMergeProgress, (ULONG) counter.GetCurrValue() );

    setState( cFilteredDocuments, _cFilteredDocuments );
    unsigned size = roundup(_partList.LokIndexSize(), ((1024*1024)/CI_PAGE_SIZE));
    setState( dwIndexSize,  size);
    setState( cUniqueKeys, _sKeyList->MaxKeyIdInUse() );

    state.cbStruct = min( state.cbStruct, sizeof(state) );

    DWORD eCiState = _dwFilteringState;

    //
    //  Set each of the state bits independently.
    //

    //
    //  Are we in a shadow or master merge?
    //
    if ( 0 != _pMerge )
    {
        switch (_pMerge->GetMergeType() )
        {
        case mtShadow:
            eCiState |= CIF_STATE_SHADOW_MERGE;
            break;

        case mtAnnealing:
            eCiState |= CIF_STATE_ANNEALING_MERGE;
            break;

        case mtIncrBackup:
            eCiState |= CIF_STATE_INDEX_MIGRATION_MERGE;
            break;

        default:
            eCiState |= CIF_STATE_MASTER_MERGE;
            break;
        }
    }
    else
    {
        CPartition *pPartition = _partList.LokGetPartition ( partidDefault );

        if ( pPartition->InMasterMerge() )
            eCiState |= CIF_STATE_MASTER_MERGE_PAUSED;
    }

    if ( LokIsScanNeeded() )
        eCiState |= CIF_STATE_CONTENT_SCAN_REQUIRED;

    setState( eState, (CIF_STATE_FLAGS) eCiState );

    return STATUS_SUCCESS;
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::IndexSize
//
//  Synopsis:   Computes the size of the indexes in all partitions
//
//  Arguments:  [mbIndex] - On output, will have the size of all indexes
//              in MBs.
//
//  History:    4-15-96   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::IndexSize( ULONG & mbIndex )
{
    CLock   lock(_mutex);
    mbIndex = _partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE);
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokCheckWordlistQuotas, public
//
//  Synopsis:   Determine if we've reached wordlist capacity.
//
//  Returns:    TRUE if we have as much wordlist data in memory as
//              parameters allow.
//
//  History:    14-Jan-1999   KyleP      Created
//
//----------------------------------------------------------------------------

BOOL CResManager::LokCheckWordlistQuotas()
{
    CPartIter iter;

    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList) )
    {
        CPartition* pPart = iter.LokGet();

        if ( pPart->LokCheckWordlistMerge() )
            return TRUE;
    }

    return FALSE;
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::MarkCorruptIndex
//
//  Synopsis:   Marks the index as corrupt, disables updates in all partitions
//              and wakes up the merge thread to notify the framework client
//              about the corruption.
//
//  History:    1-30-97   srikants   Created
//
//----------------------------------------------------------------------------

NTSTATUS CResManager::MarkCorruptIndex()
{
    CPriLock    lock(_mutex);

    //
    // Disable updates in all partitions
    //
    CPartIter iter;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        CPartition * pPart = iter.LokGet();
        pPart->LokDisableUpdates();
    }

    if ( _state.FLokCorruptionNotified() )
        return S_OK;

    _isCorrupt = TRUE;
    StopCurrentMerge();
    _eventMerge.Set();

    return S_OK;
}

//=====================
//             KEYINDEX
//=====================

#ifdef KEYLIST_ENABLED
//+---------------------------------------------------------------------------
//
//  Member:      CResManager::_AddRefKeyList, private
//
//  Synopsis:    Adds to refcount on the keylist
//
//  History:     07-Jul-94   dlee        Created
//
//----------------------------------------------------------------------------

CKeyList * CResManager::_AddRefKeyList()
{
    CPriLock lock( _mutex );
    _sKeyList->Reference();
    return _sKeyList.GetPointer();
}

//+---------------------------------------------------------------------------
//
//  Member:      CResManager::_ReleaseKeyList, private
//
//  Synopsis:    Release a refcount on the keylist
//
//  History:     07-Jul-94   dlee        Created
//
//----------------------------------------------------------------------------

void CResManager::_ReleaseKeyList()
{
    CPriLock lock( _mutex );

    _sKeyList->Release();

    if ( !_sKeyList->InUse() && _sKeyList->IsZombie() )
    {
        ciDebugOut(( DEB_ITRACE,
                     "Keylist %x unreferenced zombie.  Deleting\n",
                     _sKeyList->GetId() ));

        CKeyList * pKeyList = _sKeyList.Acquire();
        pKeyList->Remove();
        delete pKeyList;
    }
}

#endif  // KEYLIST_ENABLED

//+---------------------------------------------------------------------------
//
//  Member:      CResManager::KeyToId, public
//
//  Synopsis:    Maps from a key to an id.
//
//  Arguments:   [pkey] -- pointer to the key to be mapped to ULONG
//
//  Returns:     key id - a ULONG
//
//  History:     03-Nov-93   w-Patg      Created.
//               17-Feb-94   KyleP       Initial version
//
//----------------------------------------------------------------------------

ULONG CResManager::KeyToId( CKey const * pkey )
{

#ifdef KEYLIST_ENABLED
    CKeyList * pKeyList = _AddRefKeyList();

    ULONG kid = pKeyList->KeyToId( pkey );

    _ReleaseKeyList();

    return( kid );
#else
    return widInvalid;
#endif  // KEYLIST_ENABLED

}

//+---------------------------------------------------------------------------
//
//  Member:      CResManager::IdToKey, public
//
//  Synopsis:    Maps from an id to a key.
//
//  Arguments:   [ulKid] -- key id to be mapped to a key
//               [rkey] -- reference to the returned key
//
//  Returns:     void
//
//  History:     03-Nov-93   w-Patg      Created.
//               17-Feb-94   KyleP       Initial version
//
//----------------------------------------------------------------------------
void CResManager::IdToKey( ULONG ulKid, CKey & rkey )
{

#ifdef KEYLIST_ENABLED
    CKeyList * pKeyList = _AddRefKeyList();

    pKeyList->IdToKey( ulKid, rkey );

    _ReleaseKeyList();
#else
    rkey.FillMin();
#endif  // KEYLIST_ENABLED

}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::ComputeRelevantWords, public
//
//  Synopsis:   Computes and returns relevant words for a set of wids
//
//  Arguments:  [cRows]    -- # of rows to compute and in pwid array
//              [cRW]      -- # of rw per wid
//              [pwid]     -- array of wids to work over
//              [partid]   -- partition
//
//  History:    10-May-94 v-dlee  Created
//
//--------------------------------------------------------------------------

CRWStore * CResManager::ComputeRelevantWords(ULONG cRows,ULONG cRW,
                                             WORKID *pwids,
                                             PARTITIONID partid)
{
    CRWStore *prws = 0;

#ifdef KEYLIST_ENABLED

    CPriLock lock( _mutex );

    CPartition *pPart = LokGetPartition(partid);

    if (pPart != 0)
    {
        CPersIndex *pIndex= pPart->GetCurrentMasterIndex();

        if (pIndex != 0)
        {
            prws = pIndex->ComputeRelevantWords(cRows,cRW,pwids,
                                                _sKeyList.GetPointer() );
        }
    }
#endif  // KEYLIST_ENABLED

    return prws;
} //ComputeRelevantWords

//+-------------------------------------------------------------------------
//
//  Member:     CRWStore::RetrieveRelevantWords, public
//
//  Synopsis:   Retrieves relevant words already computed
//
//  Arguments:  [fAcquire] -- TRUE if ownership is transferred.
//              [partid]   -- partition
//
//  History:    10-May-94 v-dlee  Created
//
//--------------------------------------------------------------------------

CRWStore * CResManager::RetrieveRelevantWords(BOOL fAcquire,
                                              PARTITIONID partid)
{
    CRWStore *prws = 0;

#ifdef KEYLIST_ENABLED

    CPriLock lock( _mutex );

    CPartition *pPart = LokGetPartition(partid);

    if (pPart != 0)
    {
        prws = pPart->RetrieveRelevantWords(fAcquire);
    }
#endif  // KEYLIST_ENABLED

    return prws;
} //RetrieveRelevantWords



//================
// PRIVATE METHODS
//================



//+-------------------------------------------------------------------------
//
//  Member:     CResManager::CompactResources
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

void CResManager::CompactResources()
{
    CPriLock lock( _mutex );

    CPartIter iter;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        iter.LokGet()->LokCompact();
    }
}


 //=================================//
 //              MERGES             //
 //=================================//

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::DoMerges, private
//
//  History:    08-Apr-91   BartoszM    Created
//              25-Feb-92   BartoszM    Rewrote to use thread
//              30-Jun-97   KrishnaN    Calling docstore.FlushPropertyStore
//                                      before a shadow merge.
//              03-Mar-98   KitmanH     Don't merge if _storage is read-only
//
//  Notes:      Entry point for captive threads
//
//----------------------------------------------------------------------------

void CResManager::DoMerges()
{
    if ( _storage.IsReadOnly() )
       return;

    CCiFrmPerfCounter pPerIndexCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_PERSISTENT_INDEXES );
    CCiFrmPerfCounter pIndexSizeCounter( _adviseStatus.GetPointer(), CI_PERF_INDEX_SIZE );
    CCiFrmPerfCounter pPendingCounter( _adviseStatus.GetPointer(), CI_PERF_FILES_TO_BE_FILTERED );
    CCiFrmPerfCounter pNumKeysCounter( _adviseStatus.GetPointer(), CI_PERF_NUM_UNIQUE_KEY);
    CCiFrmPerfCounter pWordListCounter(_adviseStatus.GetPointer(), CI_PERF_NUM_WORDLIST);
    CCiFrmPerfCounter pDeferredFilesCounter( _adviseStatus.GetPointer(), CI_PERF_DEFERRED_FILTER_FILES );

    XPtr<CIdleTime> xIdle( new CIdleTime() );

    //
    // Allow mount to complete before checking for merges. O/W we will
    // start a merge before the mount/startup is complete and it might
    // prevent a system from coming up.
    //

    const lowDiskPollTime = 1;  // 1 minute

    DWORD dwWaitTime = _isLowOnDiskSpace ? lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval();

    _eventMerge.Wait(  dwWaitTime * 1000 * 60 );

    BOOL fAnnealing = FALSE;

    while ( !_fStopMerge )
    {
        BOOL fCorrupt = FALSE;

        ciDebugOut (( DEB_ITRACE, "\t|Merge Wakeup!\n" ));

        //
        // Book keeping chores.
        //
        BOOL    fBackupCiData = FALSE;
        BOOL    fBackupStarted = FALSE;

        TRY
        {
            // ===============================================
            CPriLock   lock(_mutex);

            if ( _isCorrupt  )
            {
               if ( !_state.FLokCorruptionNotified() )
                  LokNotifyCorruptionToClient();
            }
            else
            {
               //
               // Use low disk condition plus the current state to decide if
               // updates should be enabled or not
               //

               _isLowOnDiskSpace = LokCheckIfDiskLow( *this,
                                                      _storage,
                                                      _isLowOnDiskSpace,
                                                      _adviseStatus.GetReference() );
               if ( _isLowOnDiskSpace )
               {
                  if ( _state.LokGetState() != eUpdatesToBeDisabled
                       && _state.LokGetState() != eUpdatesDisabled )
                  {
                     _state.LokSetState( eUpdatesToBeDisabled );
                     _state.LokSetUpdateType( eIncremental );
                  }
               }
               else
               {
                  if ( _state.LokGetState() == eUpdatesDisabled )
                     _state.LokSetState( eUpdatesToBeEnabled );
               }

               if ( _state.LokGetState() == eUpdatesToBeDisabled )
                  LokNotifyDisableUpdatesToClient();
               else if ( _state.LokGetState() == eUpdatesToBeEnabled )
                  LokNotifyEnableUpdatesToClient();
            }

            fBackupCiData = 0 != _pBackupWorkItem;
            // ===============================================
        }
        CATCH( CException, e )
        {
            // ignore and try again the next time through the loop.
        }
        END_CATCH

        TRY
        {
            if ( !_isCorrupt && _partidToMerge != partidInvalid )
            {
                ciDebugOut (( DEB_ITRACE, "Forced merge in %ld\n",
                              _partidToMerge ));

                switch ( _mtForceMerge )
                {
                case CI_MASTER_MERGE:
                    MasterMerge ( _partidToMerge );
                    break;

                case CI_SHADOW_MERGE:
                    Merge( mtShadow, _partidToMerge );
                    break;

                case CI_ANNEALING_MERGE:
                    Merge( mtAnnealing, _partidToMerge );
                    break;

                case CI_ANY_MERGE:
                    break;  // Do nothing.  Let normal mechanism work it out.

                default:
                    Win4Assert( !"Invalid ForceMerge type" );
                }
            }

            if ( !_isLowOnDiskSpace && !_isCorrupt )
            {
                //
                // Refile the documents from the secondary queue to the primary
                // queue in pull filtering.
                //
                // ======================= lock ======================
                if ( !_fPushFiltering )
                {
                    CLock   lock(_mutex);
                    LokRefileSecQueueDocs();
                }
                // ======================= unlock ======================

                if ( fBackupCiData )
                {
                    BOOL fIsMasterPresent = FALSE;
                    BOOL fIsMMergeInProgress = FALSE;

                    //================================
                    {
                        CPriLock    lock(_mutex);
                        _pBackupWorkItem->LokSetDoingMerge();
                        fIsMasterPresent = LokIsMasterIndexPresent( partidDefault );
                        fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault );
                    }
                    //================================

                    if ( _pBackupWorkItem->IsFullSave() ||
                         fIsMMergeInProgress ||
                         !fIsMasterPresent )
                    {
                        _pBackupWorkItem->SetDoingFullSave();
                        MasterMerge( partidDefault );
                    }
                    else
                    {
                        Merge( mtIncrBackup, partidDefault );
                    }

                    fBackupStarted = TRUE;
                    BackupContentIndexData();
                }
                else
                {
                    if ( fAnnealing )
                    {
                        fAnnealing = FALSE;

                        CPartIdStack partitionIds;
                        CheckAndDoMerge( mtAnnealing, partitionIds );

                        while ( partitionIds.Count() > 0 )
                           Merge( mtAnnealing, partitionIds.Pop() );
                    }

                    {
                        CPartIdStack partitionIds;
                        CheckAndDoMerge( mtShadow, partitionIds );

                        while ( partitionIds.Count() > 0 )
                        {
                            // Call FlushPropertyStore on DocStore to give it
                            // a chance to persist changes before a shadow merge.
                            _docStore->FlushPropertyStore();
                            Merge(  mtShadow, partitionIds.Pop() );
                        }
                    }

                    {
                        if ( CheckDeleteMerge( _partidToMerge != partidInvalid ) )
                        {
                            // Call FlushPropertyStore on DocStore to give it
                            // a chance to persist changes before a shadow merge.
                            _docStore->FlushPropertyStore();
                            Merge( mtDeletes, 1 );
                        }
                    }

                    BOOL fIsMasterPresent = FALSE;
                    BOOL fIsMMergeInProgress = FALSE;

                    //================================
                    {
                        CPriLock    lock(_mutex);
                        fIsMasterPresent = LokIsMasterIndexPresent( partidDefault );
                        fIsMMergeInProgress = LokIsMasterMergeInProgress( partidDefault );
                    }
                    //================================

                    {
                        CPartIdStack partitionIds;
                        CheckAndDoMerge( mtMaster, partitionIds );

                        while ( partitionIds.Count() > 0 )
                            MasterMerge( partitionIds.Pop() );
                    }
                }

            }
        }
        CATCH ( CException, e )
        {
            ciDebugOut (( DEB_ERROR,
                          "ResMan::DoMerges -- merge failed with 0x%x\n",
                          e.GetErrorCode() ));

            if ( IsDiskFull(e.GetErrorCode()) )
            {
                ciDebugOut(( DEB_ERROR, "***** DISK IS FULL *****\n" ));
                CPriLock   lock(_mutex);
                _isLowOnDiskSpace = LokCheckIfDiskLow( *this,
                                           _storage,
                                           _isLowOnDiskSpace,
                                           _adviseStatus.GetReference() );
            }
            else if ( IsCorrupt( e.GetErrorCode() ) )
            {
                Win4Assert( "!Merge failed, but not for low disk.  Why?" );
                fCorrupt = TRUE;
            }

            //
            // We have to prevent a master merge from being restarted
            // immediately. The reason for failure could be something like
            // out of disk space (typical reason) or log full. In either
            // case we will only be exacerbating the situation if we
            // restart immediately.
            //
            _eventMerge.Reset();

        }
        END_CATCH

        // ==========================================================
        {
            CPriLock lock( _mutex );

            Win4Assert( _partList.LokIndexCount() >= _partList.LokWlCount() );
            pPerIndexCounter.Update(_partList.LokIndexCount()-_partList.LokWlCount());
            pIndexSizeCounter.Update(_partList.LokIndexSize() / ((1024*1024)/CI_PAGE_SIZE) );

            pNumKeysCounter.Update(_sKeyList->MaxKeyIdInUse());

            pWordListCounter.Update(_partList.LokWlCount());

            _partidToMerge = partidInvalid;
            if ( _fStopMerge )
            {
                ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
                break;
            }

            if ( !_isCorrupt && ( _state.LokGetState() == eSteady ) && ( 0 != _pFilterAgent ) )
                _pFilterAgent->LokWakeUp();

            _eventMerge.Reset();   // sleep

            if ( fBackupStarted || _isCorrupt )
            {
                //
                // Set that backup to be complete even if it failed. Once
                // Backup starts, it either succeeds or fails but in either
                // case the operation is considered done.
                //

                if ( 0 != _pBackupWorkItem )
                {
                    if ( _isCorrupt )
                        _pBackupWorkItem->LokSetStatus( CI_CORRUPT_DATABASE );

                    _pBackupWorkItem->LokSetDone();
                }
            }
        }
        // ==========================================================

        BOOL fSteadyState = TRUE;

        if ( fCorrupt )
        {
            CPriLock lock(_mutex);
            _isCorrupt = TRUE;
            fSteadyState = _state.LokGetState() == eSteady;
        }

        unsigned cSecQCount;
        unsigned cPending = CountPendingUpdates( cSecQCount );

        //
        //  Wait for either the merge event to be signaled or the deadman timeout.
        //

        dwWaitTime = ( ( _isLowOnDiskSpace ) ||
                       ( !fSteadyState ) ||
                       ( 0 != cPending ) ) ?
                     lowDiskPollTime : _frmwrkParams.GetMaxMergeInterval();

        NTSTATUS Status = _eventMerge.Wait( dwWaitTime * 60 * 1000 );

        if ( !_isCorrupt )
        {
            unsigned PercentIdle = xIdle->PercentIdle();

            //
            // Are we idle enough to consider an annealing merge?
            //

            if ( STATUS_TIMEOUT == Status && PercentIdle >= _frmwrkParams.GetMinMergeIdleTime() )
            {
                ciDebugOut(( DEB_ITRACE, "Idle time for period: %u percent\n", PercentIdle ));
                fAnnealing = TRUE;
            }
        }

        if ( _fStopMerge )
        {
            ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
            break;
        }

        cPending = CountPendingUpdates( cSecQCount );

        pPendingCounter.Update(cPending);
        pDeferredFilesCounter.Update(cSecQCount);

        ciDebugOut (( DEB_ITRACE | DEB_PENDING,
                      "%d updates pending\n", cPending ));
    }

    // ===========================================
    {
        CPriLock    lock2(_mutex);
        if ( _pBackupWorkItem )
        {
            ciDebugOut(( DEB_WARN, "Forcing the backup to be done\n" ));
            _pBackupWorkItem->LokSetDone();
        }
    }
    // ===========================================
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::StopMerges, private
//
//  Synopsis:   Aborts merge-in-progress (if any) and sets merge flag.
//
//  History:    13-Aug-93 KyleP     Created
//
//--------------------------------------------------------------------------

void CResManager::StopMerges()
{
    {
        CPriLock lock( _mutex );

        _fStopMerge = TRUE;
        _eventMerge.Set(); // wake up

        //
        // If we're doing a merge right now then kill it off.
        //

        if ( 0 != _pMerge )
        {
            _pMerge->LokAbort();
        }
    }

    //
    // Wait for merge thread to die.
    //

    _thrMerge.WaitForDeath();
}


//+---------------------------------------------------------------------------
//
//  Member:     CResManager::CheckAndDoMerge, private
//
//  History:    08-Apr-91   BartoszM    Created
//
//  Notes:      Called by captive thread.
//
//----------------------------------------------------------------------------

void CResManager::CheckAndDoMerge( MergeType mt, CPartIdStack & partitionIds)
{
    if ( !IsBatteryLow() )
    {
        CPriLock lock( _mutex );
        ciDebugOut (( DEB_ITRACE, "check merge\n" ));
        CPartIter iter;

        for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
        {
            CPartition* pPart = iter.LokGet();

            if ( mt == mtMaster )
            {
                __int64 shadowIndexSize = pPart->LokIndexSize();
                CPersIndex * pMasterIndex = pPart->GetCurrentMasterIndex();
                if ( pMasterIndex )
                    shadowIndexSize -= pMasterIndex->Size();
    
                shadowIndexSize *= CI_PAGE_SIZE;
    
                CMasterMergePolicy mergePolicy( _storage, shadowIndexSize,
                                                LokGetFreshCount(),
                                                _mergeTime,
                                                _frmwrkParams,
                                                _adviseStatus.GetReference() );

                if ( mergePolicy.IsTimeToMasterMerge() ||
                     ( mergePolicy.IsMidNightMergeTime() &&
                       !IsOnBatteryPower() ) )
                    partitionIds.Push( pPart->GetId() );
            }
            else
            {
                if ( pPart->LokCheckMerge(mt) ||
                     ( IsMemoryLow() && pPart->LokCheckLowMemoryMerge() ) )
                {
                    #if CIDBG == 1
                    if ( !pPart->LokCheckMerge(mt) )
                    {
                        ciDebugOut(( DEB_WARN, "Merge due to low memory. %u wordlists, %u shadow.\n",
                                     pPart->WordListCount(), pPart->LokIndexCount() ));
                    }
                    #endif
                    partitionIds.Push( pPart->GetId() );
                }
            }
        }
    }
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::CheckDeleteMerge, private
//
//  Synopsis:   Merge check for delete merge
//
//  Arguments:  [fForce] -- If TRUE, even 1 pending delete will force merge.
//
//  Returns:    TRUE if merge should be performed.
//
//  History:    12-Jun-97   KyleP       Created
//
//  Notes:      Called by captive thread.
//
//----------------------------------------------------------------------------

BOOL CResManager::CheckDeleteMerge( BOOL fForce )
{
    CPriLock lock( _mutex );
    ciDebugOut (( DEB_ITRACE, "check delete merge\n" ));

    //
    // A 'delete merge' occurs when enough documents have been deleted, and no
    // documents have been added or modified (which would cause a shadow merge).
    //

    return ( ( _fresh.LokDeleteCount() > _frmwrkParams.GetMaxFreshDeletes() ) ||
             ( fForce && _fresh.LokDeleteCount() > 0 ) );
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::Merge, private
//
//  Synopsis:   Perform a merge in partition
//
//  Arguments:  [partid] -- partition id
//              [mt]     -- Merge type
//
//  Signals:    CExcetpion
//
//  History:    08-Apr-91   BartoszM    Created
//              13-Nov-91   BartoszM    Rewrote using CMerge
//
//----------------------------------------------------------------------------

void CResManager::Merge( MergeType mt, PARTITIONID partid )
{
    ciDebugOut (( DEB_ITRACE, "%s", (mtDeletes == mt) ? "Delete merging: Have a quick stretch" :
                                                        "Shadow merging: Get a cup of coffee\n" ));

    CMerge Merge( *this, partid, mt );

    //
    // Perform resource acquisition under storage transaction.  If
    // aborted all persistent storage will be deleted by the storage
    // transaction.
    //
    {
        CPriLock lock ( _mutex );

        Win4Assert( _pMerge == 0 );

        if (_fStopMerge)
        {
            ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
            return;
        }

        Merge.LokGrabResources();  // may throw an exception

        //
        // Optimization: Don't merge zero indexes.
        //

        if ( 0 != Merge.LokCountOld() )
        {
            _pMerge = &Merge;
        }
        else if ( mtDeletes != mt )
        {
            ciDebugOut(( DEB_ITRACE, "Shadow merge of zero indexes.  Cancelled.\n" ));
            return;
        }

    }

    XPtr<CNotificationTransaction> xNotifyTrans;

    // Perform the merge under simple transaction
    // No lock held.
    {
        TRY
        {
            //
            // Perform merge, if necessary.
            //

            if ( 0 != Merge.LokCountOld() )
            {
                CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS );
                Merge.Do( counter );

            }

            //
            // NotifTrans should be outside resman lock and the resman lock
            // is acquired in notifTran's destructor
            //
            xNotifyTrans.Set( new CNotificationTransaction( this,
                                  _xIndexNotifTable.GetPointer() ) );
        }
        CATCH ( CException, e )
        {
            CPriLock lock ( _mutex );
            Merge.LokRollBack();
            _pMerge = 0;

            // Really bad errors indicate the index is corrupt.

            SCODE scE = e.GetErrorCode();

            if ( STATUS_INTEGER_DIVIDE_BY_ZERO == scE ||
                 STATUS_ACCESS_VIOLATION       == scE ||
                 STATUS_IN_PAGE_ERROR          == scE )
            {
                ciDebugOut(( DEB_ERROR,
                             "Corrupt index, caught 0x%x\n", scE ));
                _storage.ReportCorruptComponent( L"ShadowMerge" );
                THROW( CException( CI_CORRUPT_DATABASE ) );
            }

            RETHROW();
        }
        END_CATCH
    }

    // nb: no failures allowed until CMergeTrans is constructed!

    CDiskIndex* pIndexNew = 0;
    { //=================== begin notification transaction =========================
        {
            CLock lock( _mutex );
            XPtr<CFreshTest> xFreshTestAtMerge;

            {
                _pMerge = 0;

                CMergeTrans Xact( *this, _storage, Merge );

                unsigned cIndOld  = Merge.LokCountOld();
                INDEXID* aIidOld  = Merge.LokGetIidList();
                pIndexNew = Merge.LokGetNewIndex();

                //
                // Write the new fresh test persistently to disk.
                //
                CPersFresh newFreshLog( _storage, _partList );
                CShadowMergeSwapInfo swapInfo;

                swapInfo._widNewFreshLog = _fresh.LokUpdate( Merge,
                                                             Xact,
                                                             newFreshLog,
                                                             pIndexNew ? pIndexNew->GetId() : iidInvalid,
                                                             cIndOld,
                                                             aIidOld,
                                                             xFreshTestAtMerge );

                swapInfo._widOldFreshLog =
                                    _storage.GetSpecialItObjectId( itFreshLog );
                swapInfo._cIndexOld = cIndOld;
                swapInfo._aIidOld = aIidOld;

                _partList.LokSwapIndexes( Xact,
                                          partid,
                                          pIndexNew,
                                          swapInfo );

                //
                // No failures allowed after this point on until the transaction
                // is committed.
                //
                Merge.ReleaseNewIndex();

                ciDebugOut (( DEB_ITRACE, "done merging\n" ));

                Xact.LokCommit();

#if CIDBG == 1
                if ( pIndexNew )
                    pIndexNew->Reference();
#endif
                //==========================================
            }

            ciFAILTEST( STATUS_NO_MEMORY );

            CPartition *pPartition =_partList.LokGetPartition ( partid );

            //
            // Delete the wids in the change log that have made it to
            // the new shadow index created in this merge. The algorithm
            // needs the latest fresh test, fresh test snapshoted at the
            // start of shadow merge and the doc list. If the latest fresh
            // test is the same as the fresh test at merge, then as an
            // optimization, the fresh test at merge is null. So, in the
            // optimized case, we use the latest fresh test as the fresh
            // test at merge.
            //

            CFreshTest *pFreshTestLatest = _fresh.LokGetFreshTest();
            CFreshTestLock freshTestLock( pFreshTestLatest );

            CFreshTest *pFreshTestAtMerge;
            if ( xFreshTestAtMerge.IsNull() )
                pFreshTestAtMerge = pFreshTestLatest;
            else
                pFreshTestAtMerge = xFreshTestAtMerge.GetPointer();

            //==========================================

            CChangeTrans xact( *this, pPartition );
            pPartition->LokDeleteWIDsInPersistentIndexes( xact,
                                                          *pFreshTestLatest,
                                                          *pFreshTestAtMerge,
                                                          _docList,
                                                          xNotifyTrans.GetReference() );
            xact.Commit();
        }

        xNotifyTrans.Free();
    } //=================== end notification transaction =========================

    #if CIDBG == 1
        if ( pIndexNew )
        {
            pIndexNew->VerifyContents();

            {
                CPriLock lock ( _mutex );
                LokReleaseIndex( pIndexNew );
            }
        }
    #endif

    _storage.CheckPoint();
} //Merge

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LogMMergeStartFailure
//
//  Synopsis:   Writes an eventlog message to indicate a MasterMerge could
//              not be started.
//
//  Arguments:  [fRestart]       - Indicates if this is a restarted master merge.
//              [storage]        - Storage reference
//              [error]          - Error code.
//              [adviseStatus]   - reference to ICiCAdviseStatus
//
//  History:    5-27-96   srikants   Created
//              Jan-07-96 mohamedn   CDmFwEventItem
//
//----------------------------------------------------------------------------

void CResManager::LogMMergeStartFailure( BOOL fRestart,
                                         PStorage & storage,
                                         ULONG error,
                                         ICiCAdviseStatus & adviseStatus)
{

    TRY
    {
        DWORD dwMsgCode = fRestart ?
                          MSG_CI_MASTER_MERGE_CANT_RESTART :
                          MSG_CI_MASTER_MERGE_CANT_START;


        CDmFwEventItem Item( EVENTLOG_AUDIT_FAILURE,
                             dwMsgCode,
                             2,
                             sizeof(error),
                             (void *)&error );
        Item.AddArg( storage.GetVolumeName() );
        Item.AddArg( error );

        Item.ReportEvent( adviseStatus );

    }
    CATCH( CException, e )
    {
        ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergeStartFailure\n",
                     e.GetErrorCode() ));
    }
    END_CATCH
}


//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LogMMergePaused
//
//  Synopsis:   Writes an eventlog message that master merge was paused.
//
//  Arguments:  [storage]      - Storage
//              [error]        - Error which caused the merge to be paused.
//              [adviseStatus] - Advise status to use for notifying about the
//              event.
//
//  History:    1-24-97   srikants   Added header
//
//----------------------------------------------------------------------------

void CResManager::LogMMergePaused( PStorage & storage,
                                   ULONG error,
                                   ICiCAdviseStatus &adviseStatus)
{

    if ( IsCorrupt( error ) )
    {
        // don't log "paused" when there is corruption
        return;
    }

    TRY
    {

        CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
                             MSG_CI_MASTER_MERGE_ABORTED,
                             2,
                             sizeof(error),
                             (void *)&error );
        Item.AddArg( storage.GetVolumeName() );
        Item.AddArg( error );

        Item.ReportEvent( adviseStatus );

    }
    CATCH( CException, e )
    {
        ciDebugOut(( DEB_ERROR, "Error 0x%X in LogMMergePaused\n",
                     e.GetErrorCode() ));
    }
    END_CATCH
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::MasterMerge, private
//
//  Synopsis:   Perform a MasterMerge in partition
//
//  Arguments:  [partid] -- partition id
//
//  Signals:    CException
//
//  History:    08-Apr-91   BartoszM    Created
//              13-Nov-91   BartoszM    Rewrote using CMerge
//              17-Feb-94   KyleP       Added keylist merge
//              Jan-07-96   mohamedn    CDmFwEventItem
//
//----------------------------------------------------------------------------

void CResManager::MasterMerge ( PARTITIONID partid )
{
    ciDebugOut (( DEB_ITRACE, "\n===MASTER MERGE===\n" ));

    //
    //  Force a shadow merge if we are not restarting a master merge.  This
    //  will force all word lists into persistent indexes.
    //
    BOOL    fMasterMergePaused = FALSE;

    //
    // Before starting the master merge, we have to swap the deleted index
    // iid. However, if there is a failure before we commit the beginning
    // of the master merge, we have to roll back this change. The
    // CDeletedIIDTrans object does this.
    //
    CDeletedIIDTrans    delIIDTrans( *this );
    INDEXID iidDelPreMerge = iidInvalid;

    {
        CPriLock lock ( _mutex );
        CPartition *pPartition = _partList.LokGetPartition ( partid );
        fMasterMergePaused = pPartition->InMasterMerge();
        if ( !fMasterMergePaused )
        {
            if ( 0 == LokGetFreshCount() )
            {
                //
                // There is no benefit in doing a master merge. Just return
                //
                return;
            }

            iidDelPreMerge = _activeDeletedIndex.GetIndex();
            delIIDTrans.LokLogNewDeletedIid( iidDelPreMerge,
                                             _activeDeletedIndex.GetNewIndex() );
            _activeDeletedIndex.SwapIndex();
        }
    }


    if ( !fMasterMergePaused )
    {
        //
        // First do a delete merge to finish merging all outstanding
        // word-lists into a persistent index.
        //
        // A delete merge will force an optimized null merge if there are
        // outstanding deletes in the fresh test which need to be logged.
        // This is an important case for Push filtering.  Since it doesn't
        // hurt in the other model, just keep the code identical.
        //

        Merge( mtDeletes, partid );
    }
    else
    {
        //
        // This is a restarted master merge. The _activeDeletedIndex contains
        // the "Post" master merge deleted index. The "New" one will be same
        // as the "Prev" one.
        //
        iidDelPreMerge = _activeDeletedIndex.GetNewIndex();

        CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
                             MSG_CI_MASTER_MERGE_RESTARTED,
                             1 );

        Item.AddArg( _storage.GetVolumeName() );
        Item.ReportEvent( _adviseStatus.GetReference() );

    }


    CMasterMerge Merge( *this, partid );

    TRY
    {
        CPriLock lock ( _mutex );

        Win4Assert( _pMerge == 0 );

        if ( _fStopMerge )
        {
            ciDebugOut(( DEB_ITRACE, "Stopping Merge\n" ));
            return;
        }

        ciFAILTEST( STATUS_NO_MEMORY );

        CPartition *pPartition =_partList.LokGetPartition ( partid );

        if ( !pPartition->InMasterMerge() )
        {
            Win4Assert( delIIDTrans.IsTransLogged() );
            Merge.LokGrabResources( delIIDTrans );     // may throw an exception
            if ( 0 == Merge.LokCountOld() )
            {
                ciDebugOut(( DEB_ITRACE, "Master merge of 0 indexes cancelled\n"));
                return;
            }
        }
        else
        {
            Merge.LokLoadRestartResources();
        }

        //
        // The deletionIIDTrans must be either committed or not logged at
        // all.
        //
        Win4Assert( !delIIDTrans.IsRollBackTrans() );

        _pMerge = &Merge;

    }
    CATCH( CException, e )
    {
        ciDebugOut(( DEB_ERROR, "Error 0x%X while starting a MasterMerge\n",
                     e.GetErrorCode() ));

        LogMMergeStartFailure( fMasterMergePaused,
                               _storage,
                               e.GetErrorCode(),
                               _adviseStatus.GetReference() );

        RETHROW();
    }
    END_CATCH


    // Perform the merge under simple transaction
    // No lock held.
    {
        TRY
        {
            // Perform merge
            CCiFrmPerfCounter counter( _adviseStatus.GetPointer(), CI_PERF_MERGE_PROGRESS );
            Merge.Do( counter );
        }
        CATCH ( CException, e )
        {
            CPriLock lock ( _mutex );
            Merge.LokRollBack();
            _pMerge = 0;

            LogMMergePaused( _storage,
                             e.GetErrorCode(),
                             _adviseStatus.GetReference() );
            RETHROW();
        }
        END_CATCH
    }

    CMasterMergeIndex* pMMergeIndex = 0;
    {
        CPriLock lock ( _mutex );
        {
            _pMerge = 0;

            unsigned cIndOld  = Merge.LokCountOld();
            INDEXID* aIidOld  = Merge.LokGetIidList();
            pMMergeIndex = Merge.LokGetMasterMergeIndex();

            CMergeTrans MergeTrans( *this, _storage, Merge );

            ciFAILTEST( STATUS_NO_MEMORY );

            //==========================================
            CMasterMergeTrans xact( *this,
                                    _storage,
                                    MergeTrans,
                                    pMMergeIndex,
                                    Merge );

            CPartition *pPart = LokGetPartition(partid);

            #ifdef KEYLIST_ENABLED
            pPart->SetRelevantWords(pMMergeIndex->AcquireRelevantWords());
            #endif  // KEYLIST_ENABLED

            //
            // Remove old indexes from fresh list after master merge
            //
            CPersFresh newFreshLog( _storage, _partList );

            CMasterMergeSwapInfo SwapInfo;

            SwapInfo._widNewFreshLog = _fresh.LokRemoveIndexes (
                                          MergeTrans,
                                          newFreshLog,
                                          cIndOld, aIidOld, iidDelPreMerge );

            SwapInfo._widOldFreshLog = _storage.GetSpecialItObjectId( itFreshLog );

            SwapInfo._cIndexOld = cIndOld;
            SwapInfo._aIidOld = aIidOld;
            SwapInfo._partid = partid;

            CKeyList const * pOldKeyList = _sKeyList.GetPointer();
            CKeyList const * pNewKeyList = Merge.LokGetNewKeyList();

            _partList.LokSwapIndexes( MergeTrans,
                                      partid,
                                      pMMergeIndex,
                                      SwapInfo,
                                      pOldKeyList,
                                      pNewKeyList );
            //
            // After this point, we must not resume the master merge even
            // if there is a soft failure.
            //
            Merge.ReleaseNewIndex();

#ifdef KEYLIST_ENABLED
            //
            // Get rid of the current KeyList and replace it with the
            // new KeyList
            //
            _storage.SetSpecialItObjectId( itMMKeyList, widInvalid );
            CKeyList * pKeyList = _sKeyList.Acquire();
            Win4Assert( 0 != pKeyList );
            _sKeyList.Set( Merge.LokGetNewKeyList() );
            Merge.LokReleaseNewKeyList();
            pKeyList->Zombify();

            if ( !pKeyList->InUse() )
            {
                pKeyList->Remove();
                delete pKeyList;
            }

#else
            CKeyList * pKeyList = _sKeyList.Acquire();
            Win4Assert( 0 != pKeyList );
            _sKeyList.Set( Merge.LokGetNewKeyList() );
            Merge.LokReleaseNewKeyList();
            delete pKeyList;
#endif  // KEYLIST_ENABLED

#if CIDBG == 1
            pMMergeIndex->Reference();
#endif
            xact.LokCommit();
            //==========================================

            ciDebugOut (( DEB_ITRACE, "done merging\n" ));
        }

        //
        // There is no need to compact the change log here because the
        // participants in the master merge are persistent indexes and
        // there will be no wids to be deleted from the change log when
        // persistent indexes are merged into another persistent index.
        //
    }

#if CIDBG == 1
    pMMergeIndex->VerifyContents();
    {
        CPriLock lock ( _mutex );
        LokReleaseIndex( pMMergeIndex );
    }
#endif

    _storage.CheckPoint();

    CDmFwEventItem Item( EVENTLOG_INFORMATION_TYPE,
                         MSG_CI_MASTER_MERGE_COMPLETED,
                         1 );
    Item.AddArg( _storage.GetVolumeName() );
    Item.ReportEvent( _adviseStatus.GetReference() );
}

//+---------------------------------------------------------------------------
//
//  Class:      CReleaseMMergeIndex
//
//  Purpose:    Class to acquire the ownernship of the target and current
//              master indexes during the destruction of a mastermerge index
//              and release the target and current indexes.
//
//  History:    9-29-94   srikants   Created
//
//----------------------------------------------------------------------------

class CReleaseMMergeIndex
{

public:

    CReleaseMMergeIndex( CResManager & resman, CMasterMergeIndex * mmIndex );

    ~CReleaseMMergeIndex();

private:

    CResManager &       _resman;
    CMasterMergeIndex * _pmmIndex;
    CPersIndex *        _pTargetMaster;
    CPersIndex *        _pCurrentMaster;
};

CReleaseMMergeIndex::CReleaseMMergeIndex( CResManager & resman,
                     CMasterMergeIndex * mmIndex ) :
                            _resman(resman),
                            _pmmIndex(mmIndex),
                            _pTargetMaster(0),
                            _pCurrentMaster(0)
{
    Win4Assert( _pmmIndex );
    Win4Assert( _pmmIndex->IsZombie() && !_pmmIndex->InUse() );
    _pmmIndex->AcquireCurrentAndTarget( &_pCurrentMaster, &_pTargetMaster );
    Win4Assert( 0 == _pCurrentMaster || _pCurrentMaster->IsZombie() );

    if ( 0 != _pCurrentMaster )
    {
        _pCurrentMaster->Reference();
    }

    Win4Assert( 0 != _pTargetMaster );
    _pTargetMaster->Reference();

}

CReleaseMMergeIndex::~CReleaseMMergeIndex()
{
    delete _pmmIndex;

    if ( 0 != _pCurrentMaster )
        _resman.LokReleaseIndex( _pCurrentMaster );

    _resman.LokReleaseIndex( _pTargetMaster );
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokReleaseIndex, private
//
//  Synopsis:   Decrements ref counts, deletes if index
//              marked to be deleted.
//
//  Arguments:  [pIndex] -- index to be released
//
//  History:    08-Oct-91   BartoszM    Created
//
//  Notes:      ResMan LOCKED
//
//----------------------------------------------------------------------------

void CResManager::LokReleaseIndex ( CIndex* pIndex )
{
    pIndex->Release();

    if ( pIndex->IsZombie() && !pIndex->InUse() )
    {
        CIndexId iid = pIndex->GetId();

        //
        //  If it is not a master index, it has to be removed from the
        //  Partition object. A master index would have been already removed
        //  when merge completed.
        //
        //  If we are in the Empty path, then we want to force all indexes
        //  to be deleted if their ref-count is 0.
        //

        if ( !pIndex->IsMaster() || _isBeingEmptied )
        {
            if ( !_isDismounted )
            {
                TRY
                {
                    _partList.LokRemoveIndex ( iid );
                }
                CATCH( CException, e )
                {
                    ciDebugOut(( DEB_ERROR,
                        "Index 0x%08x could not be deleted from index table\n",
                        iid ));
                }
                END_CATCH

                TRY
                {
                    // Must impersonate as system since query threads
                    // impersonating as a query user can't delete the
                    // files sometimes deleted when an index is released
                    // after a merge.  Impersonation can fail, which will
                    // leak the file on disk.

                    CImpersonateSystem impersonate;

                    if ( iid.IsPersistent() )
                        pIndex->Remove();
                }
                CATCH( CException, e )
                {
                    ciDebugOut(( DEB_ERROR,
                        "Index 0x%08x could not be deleted from disk, %#x\n",
                        iid, e.GetErrorCode() ));
                }
                END_CATCH
            }
            else
            {
                //
                // Zombie indexes are cleaned up after reboot.
                //
                //
                // After a dismount, we should not have any dirty streams
                // and releasing a persistent zombie index will update the
                // index table on disk. That is not allowed. This situation
                // happens only when queries have not been released and
                // shutdown is happening.
                //
                //
                ciDebugOut(( DEB_WARN,
                    "Index with iid 0x%X being released AFTER CI dismount\n",
                             iid ));
            }

            delete pIndex;
        }
        else
        {
            //
            // This must be a master merge index as no other master index
            // can be a zombie.
            //
            Win4Assert( pIndex->IsMasterMergeIndex() );
            CReleaseMMergeIndex mmIndexRel( *this,
                                            (CMasterMergeIndex *)pIndex );
        }
    }
}

//+---------------------------------------------------------------------------
//
//  Function:   LokReplaceMasterIndex
//
//  Synopsis:   Replaces the CMasterMergeIndex with the CPersIndex incarnation
//              of the target master index in the index list. Also prepares
//              the CMasterMergeIndex for removal from the in-memory index
//              list by zombifying it.
//
//  Arguments:  [pMMergeIndex] --  The target index in the CMasterMergeIndex
//              form.
//              [pNewMaster]   --  The CPersIndex form of the target index.
//
//  History:    6-30-94   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokReplaceMasterIndex ( CMasterMergeIndex* pMMergeIndex )
{
    Win4Assert( pMMergeIndex->IsMaster() && pMMergeIndex->IsZombie() );

    CPersIndex * pNewMaster = pMMergeIndex->GetTargetMaster();
    Win4Assert( pNewMaster->IsMaster() && !pNewMaster->IsZombie() );

    //
    // Delete the master merge index from the partition.
    //
    CIndexId iid = pMMergeIndex->GetId();
    CPartition * pPart = _partList.LokGetPartition( iid.PartId() );
    pPart->LokRemoveIndex ( iid );

    //
    // Check if there is a query in progress. If not, we can
    // go ahead and delete it. If a query is in progress, the
    // indexsnapshot of the query will delete it.
    //
    if ( !pMMergeIndex->InUse() )
    {
        Win4Assert( !pNewMaster->InUse() );
        pMMergeIndex->ReleaseTargetAndCurrent();
        delete pMMergeIndex;
    }

    //
    // Add the new master index to the partition object. This will
    // now be the only master accessible to new queries.
    //
    pPart->AddIndex( pNewMaster );

}


//
//+---------------------------------------------------------------------------
//
//  Function:   RollBackDeletedIIDChange
//
//  Synopsis:   Rolls back the changed made to the deleted iid during the
//              beginning of a master merge.
//
//  Arguments:  [xact] -  The transaction object which is rolling back the
//              change to the deleted iid.
//
//  History:    7-17-95   srikants   Created
//
//  Notes:      Consider the following sequence of events:
//
//  1. Pre-MasterMerge preparations : Change Deleleted IID to iidDeleted2 from
//     iidDeleted1.
//
//  2. Start a shadow merge.
//
//  3. Commit the beginning of a new MasterMerge
//
//  4. Complete the master merge .....
//
//  If there is a failure between steps 2 and 3, we just revert the deleted
//  IID back to iidDeleted2. Even if some entries were created in the fresh
//  list between steps 2 & 3 of the form, WID->iidDeleted2, it is okay. They
//  will get deleted from the fresh list on the second MasterMerge.
//
//----------------------------------------------------------------------------

void CResManager::RollBackDeletedIIDChange( CDeletedIIDTrans & xact )
{
    CPriLock   lock(_mutex);

    Win4Assert( _activeDeletedIndex.GetIndex() == xact.GetNewDelIID() );
    Win4Assert( _activeDeletedIndex.GetNewIndex() == xact.GetOldDelIID() );

    _activeDeletedIndex.SwapIndex();
}

//========================
//  FILTER RELATED METHODS
//========================



//+---------------------------------------------------------------------------
//
//  Function:   LokGetFilterDocs
//
//  Synopsis:   Gets documents from the change log for filtering. _docList
//              will be filled with the documents.
//
//  Arguments:  [cMaxDocs]        -- [in] Maximum number of docs to retrieve
//              [cDocs]           -- [out] Number of documents retrieved
//              [state]           -- [in/out] State of the document retrieval
//                                   progress.
//
//  Returns:    Number of documents in the docBuffer.
//
//  History:    6-16-94   srikants   Separated for making kernel mode call
//                                   Asynchronous.
//
//  Notes:      This function has two goals:
//
//              1. Select documents from different partitions in a round
//                 robin fashion.
//
//----------------------------------------------------------------------------

BOOL CResManager::LokGetFilterDocs ( ULONG cMaxDocs,
                                     ULONG & cDocs,
                                     CGetFilterDocsState & state )
{
    ciDebugOut(( DEB_ITRACE, "# CResManager::LokGetFilterDocs.\n" ));
    Win4Assert( state.GetTriesLeft() > 0 && state.GetTriesLeft() <=2 );

    cDocs = 0;
    Win4Assert ( !_isBeingEmptied );

    unsigned maxDocs = min( cMaxDocs, CI_MAX_DOCS_IN_WORDLIST);

    if ( _partIter.LokAtEnd() )
        return FALSE;      // There are no valid partitions

    // we are sure that the partition is valid
    CPartition* pPart = _partIter.LokGet();
    Win4Assert ( pPart != 0 );

    //
    // Don't bother looking for more documents to filter if there
    // are too many wordlists in memory -- wait until they are
    // merged.
    //
    if ( pPart->WordListCount() < _frmwrkParams.GetMaxWordlists() )
    {
        CChangeTrans  xact( *this, pPart );
        pPart->LokQueryPendingUpdates ( xact, maxDocs, _docList );
        xact.LokCommit();
    }
    else
    {
        ciDebugOut(( DEB_ITRACE,
                     "Too many wordlists.  Waking merge\n" ));
        _eventMerge.Set();
    }

    _docList.SetPartId ( pPart->GetId() );
    //==================================
    //
    // If there is a failure after this point, we have to re-add
    // the documents to the change log.
    //
    cDocs = _docList.Count();

    for (unsigned i = 0; i < cDocs; i++)
    {
        if (_docList.Status(i) != DELETED && _docList.Status(i) != WL_NULL)
            break;
    }

    BOOL fRetry = FALSE;

    if ( i == cDocs && i != 0 )
    {
        CIndexTrans  xact( *this );
        _fresh.LokDeleteDocuments ( xact, _docList, _activeDeletedIndex.GetIndex() );
        xact.LokCommit();
        cDocs = 0;
        _docList.LokClear();
        state.Reset();

        //
        // If we are doing a bunch of deletions in a "delnode" case,
        // we should see if the number of changed documents has exceeded
        // the max fresh test count and wake up the merge thread. Note
        // that we are not starting a merge here - the merge thread will
        // deal with the policy issues.
        //
        if ( LokGetFreshCount() > _frmwrkParams.GetMaxFreshCount() ||
             CheckDeleteMerge( FALSE ) )
        {
            _eventMerge.Set();
        }

        LokUpdateCounters();
    }

    _partIter.LokAdvance( _partList );
    if ( _partIter.LokAtEnd() )
    {
        _partIter.LokInit( _partList );
        state.DecrementTries();
        fRetry = (0 == cDocs) && (state.GetTriesLeft() > 0);
    }
    else
    {
        fRetry = ( 0 != cDocs );
    }

    Win4Assert( cDocs == _docList.Count() );

    return fRetry;
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::FillDocBuffer, public
//
//  Synopsis:   Fills buffer with strings for paths and properties of docs
//              to be filtered
//
//  Arguments:  [docBuffer] -- (in, out) buffer to be filled with document
//                             names.
//              [cb]        -- (in, out) count of bytes in docBuffer
//              [cDocs]     -- (out) count of # documents to filter
//              [state]     -- [in/out] State of the document retrieval
//                             progress.
//
//  History:    08-Apr-1991   BartoszM    Created
//              24-Nov-1992   KyleP       Retry pending documents
//              22-Mar-1993   AmyA        Split into several functions
//              22-Jan-1997   SrikantS    Changed doc. names to bytes for frame
//                                        work.
//              18-May-2000   KitmanH     Changed to push retried docs to filter
//                                        one by one
//
//----------------------------------------------------------------------------

void CResManager::FillDocBuffer( BYTE * docBuffer,
                                 ULONG & cb, ULONG & cDocs,
                                 CGetFilterDocsState & state )
{
    Win4Assert( docBuffer != 0 );

    // =========================================
    {
        CPriLock    lock(_mutex);
        _docList.LokSortOnWid();
    }
    // ==========================================

    unsigned cDocuments = _docList.Count();

    //
    // Format of the data
    // 4 Bytes remaining files count, 2 Bytes number of files. Following
    // that is (2 Bytes of Name Length in bytes, Name)*
    //

    if ( _fPushFiltering )
    {
        //
        // We can't take an exception while growing the aborted wids
        // array while doing exception processing in the filter manager.
        // So, pre-allocate the array now to insure we won't run out of
        // room later.  Now is a good time to die; later isn't.
        //

        CPriLock lock( _mutex );

        _aAbortedWids.LokReserve( cDocuments );

        //
        // In push filtering there are no refiles, and hence the buffer
        // cannot overflow. Check that the buffer can fit 16 documents.
        // A document name is the serialized form of the wid, i.e. it's
        // 4 bytes long. The buffer size is 4K.
        //

        Win4Assert( cb > 4 + 2 + 16 * (2 + 4) );
    }

    const cbLenField = sizeof(ULONG);

    Win4Assert( cb >= cbLenField );
    Win4Assert( sizeof(ULONG) == 4 );

    if ( cb < cbLenField )
        THROW( CException( STATUS_INVALID_PARAMETER ) );

    //
    // Fill the number of documents left in the changelog in the first
    // 4 bytes of the buffer.
    //
    // =========================================================
    {
        CPriLock lock( _mutex );
        ULONG partid = 1;  
        CPartition * pPart = _partList.LokGetPartition( partid );
        ULONG cDocsLeft = pPart->LokCountUpdates();
        RtlCopyMemory( docBuffer, &cDocsLeft, sizeof(ULONG) );
    }
    // =========================================================

    RtlZeroMemory( docBuffer + cbLenField, sizeof USHORT );

    const cbUSHORT = sizeof USHORT;

    BYTE * pCount = docBuffer + cbLenField;
    *pCount = 0;

    BYTE * pBuf = pCount + cbUSHORT;
    BYTE * pEnd = docBuffer + cb;

    cDocs = 0;

    // copy path/property pairs into buffer until we run out of either
    // buffer space or pairs. We stop filling the buffer if we encounter a
    // document that has a retry count > 1, we will fill the docBuffer with
    // these docs one at a time 

    ciDebugOut((DEB_FILTERWIDS | DEB_ITRACE,
               "CResManager::FillDocBuffer..."));

    unsigned cbExtraSpaceNeeded = 0;

    for (unsigned cSrcDocs = 0; cSrcDocs < cDocuments; cSrcDocs++)
    {
        //
        // Leave the first 2 bytes for the length field
        //
        unsigned cbPath = (unsigned)(pEnd - pBuf);

        if ( cbPath <= cbUSHORT )
        {
            // We would definitely need more space than a USHORT, but
            // we don't know how much right now. But since the filter
            // increases space by a PAGE, I guess we should do fine.
            cbExtraSpaceNeeded = cbUSHORT - cbPath + 1;
            break;
        }

        if (_docList.Status(cSrcDocs) == DELETED)
        {
            cbPath = 0;
            ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %dX",
                         _docList.Wid(cSrcDocs) ));
        }
        else
        {          
            if ( _docList.Retries(cSrcDocs) > 1 )
            {
                ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, 
                             "************FillDocBuffer: Retry count is %d. Wid is %d\n", 
                             _docList.Retries(cSrcDocs), 
                             _docList.Wid(cSrcDocs) ));
                ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "cDocs is %d\n", cDocs ));

                if ( cDocs >  0 )
                {
                    //
                    // There is at least one document in the docBuffer. Stop
                    // now so we can file the doc(s) being retried one by one.
                    //
                    break;
                }
                else
                {
                    //
                    // Add the currenct doc. Stop if the current add is successful
                    //

                    //
                    // Make room for the length field at the beginning.
                    //
                    cbPath -= cbUSHORT;

                    ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d",
                                 _docList.Wid(cSrcDocs) ));

                    unsigned cbPathIn = cbPath;

                    //
                    // If this is not the first try, then attempt to get a more
                    // accurate path.  We may be suffering from NTBUG: 270566
                    //

                    // changes cbPath.
                    if ( !WorkidToDocName( _docList.Wid(cSrcDocs),
                                           pBuf+cbUSHORT,
                                           cbPath,
                                           TRUE ) )
                    {
                        if ( cbPathIn < cbPath)
                        {
                            cbExtraSpaceNeeded = cbPath - cbPathIn;
                        }
                        break;
                    }

                    // cbPath = 0 if object has been deleted (not added to docBuffer)
                    if ( cbPath == 0 )
                    {
                        _docList.SetStatus( cSrcDocs, DELETED );
                        ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" ));
                    }
                    else
                    {
                        ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) );
                        cSrcDocs++;
                        cDocs++;
                        USHORT uscb = (USHORT) cbPath;
                        RtlCopyMemory( pBuf, &uscb, cbUSHORT );
                        pBuf += (cbPath+cbUSHORT);
                        break;
                    }    
                }
            }
            else  // not retries
            {
                ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, 
                             "FillDocBuffer: Not retry for wid %d\n", 
                             _docList.Wid(cSrcDocs) ));

                //
                // Make room for the length field at the beginning.
                //
                cbPath -= cbUSHORT;

                ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, " %d",
                             _docList.Wid(cSrcDocs) ));

                unsigned cbPathIn = cbPath;

                // changes cbPath.
                if ( !WorkidToDocName( _docList.Wid(cSrcDocs),
                                       pBuf+cbUSHORT,
                                       cbPath,
                                       FALSE ) )
                {
                    if ( cbPathIn < cbPath)
                    {
                        cbExtraSpaceNeeded = cbPath - cbPathIn;
                    }
                    break;
                }

                // cbPath = 0 if object has been deleted
                if ( cbPath == 0 )
                {
                    _docList.SetStatus( cSrcDocs, DELETED );
                    ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "X" ));
                }
                else
                {
                    cDocs++;
                    ClearNonStoragePropertiesForWid( _docList.Wid(cSrcDocs) );
                }
            }
        }

        USHORT uscb = (USHORT) cbPath;
        RtlCopyMemory( pBuf, &uscb, cbUSHORT );

        pBuf += (cbPath+cbUSHORT);
    }
    ciDebugOut (( DEB_FILTERWIDS | DEB_NOCOMPNAME, "\n" ));

    //
    // Fill in the count of the number of documents.
    //
    USHORT usFileCount = (USHORT) cSrcDocs;
    RtlCopyMemory( pCount, &usFileCount, cbUSHORT );


    if ( cSrcDocs < cDocuments )
    {
        if ( 0 == cSrcDocs && cbExtraSpaceNeeded > 0 )
        {
            // We need a larger buffer
            cb += cbExtraSpaceNeeded;
        }

        //
        // No refiling in push filtering
        //
        Win4Assert( !_fPushFiltering );

        ciDebugOut (( DEB_ITRACE, "Putting back documents\n" ));
        CDocList docListExtra;

        docListExtra.SetPartId( _docList.PartId() );

        for ( unsigned i = 0, j = cSrcDocs; j < cDocuments; i++, j++ )
        {
            ciDebugOut(( DEB_ITRACE | DEB_NOCOMPNAME, " %d", _docList.Wid(j) ));
            ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, "Refiling Wid %d\n", _docList.Wid(j) ));
            docListExtra.Set( i,
                              _docList.Wid(j),
                              _docList.Usn(j),
                              _docList.VolumeId(j) );
        }

        ciDebugOut (( DEB_ITRACE | DEB_NOCOMPNAME, "\n" ));

        CPriLock   lock2(_mutex);

        docListExtra.LokSetCount(i);
        _docList.LokSetCount( cSrcDocs );

        ReFileCommit ( docListExtra );
    }

    //
    //  If we were unable to convert any of the wids to paths, then all of
    //  them have been deleted.
    //
    if ( 0 == cDocs )
    {
        ciDebugOut (( DEB_ITRACE, "All docs were deleted\n" ));

        CPriLock    lock3(_mutex);

        _docList.LokClear();
        state.Reset();
    }
    else
    {
        cb = (ULONG)(pBuf - docBuffer);
    }
}

//+---------------------------------------------------------------------------
//
//  Function:   NoFailReFile
//
//  Synopsis:   re-adds the documents to the change log witout committing
//
//  History:    6-16-94   srikants   Separated from FilterReady for making
//                                   kernel mode call Asynchronous.
//
//----------------------------------------------------------------------------

BOOL CResManager::NoFailReFile ( CDocList& docList )
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    CPriLock lock ( _mutex );

    if ( _isBeingEmptied )
        return FALSE;

    LokNoFailReFileChanges( docList );
    docList.LokClear();

    return TRUE;
}

//+---------------------------------------------------------------------------
//
//  Function:   ReFileCommit
//
//  Synopsis:   re-adds the documents to the change log and commits
//              them to the change list.
//
//  History:    6-16-94   srikants   Separated from FilterReady for making
//                                   kernel mode call Asynchronous.
//
//----------------------------------------------------------------------------

BOOL CResManager::ReFileCommit ( CDocList& docList )
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    CPriLock lock ( _mutex );

    if ( _isBeingEmptied )
        return FALSE;

    LokNoFailReFileChanges( docList );
    docList.LokClear();

    //
    // The docqueue can handle only one set of refiled documents.
    // we MUST complete processing of the
    // refiled documents.
    //
    PARTITIONID partId = 1;    

    CPartition * pPart = LokGetPartition( partId );

    {
        // =========================================
        CChangeTrans    xact( *this, pPart );
        pPart->LokAppendRefiledDocs( xact );
        xact.LokCommit();
        // =========================================
    }
    return TRUE;
}

//+---------------------------------------------------------------------------
//
//  Function:   LokNoFailReFileChanges
//
//  Synopsis:   Refiles the updates with the change log.
//
//  Arguments:  [docList] --  contains the documents to be added back to the
//              change log.
//
//  History:    5-16-94   srikants   Modified to deal with failures during
//                                   append of docs to change log.
//
//  Notes:      This CANNOT throw
//
//----------------------------------------------------------------------------

void CResManager::LokNoFailReFileChanges( CDocList& docList )
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    Win4Assert ( docList.PartId() == 1 );

    CPartition* pPart = _partList.LokGetPartition ( docList.PartId() );

    pPart->LokRefileDocs( docList );
    _pFilterAgent->LokWakeUp ();
}

//+-------------------------------------------------------------------------
//
//  Member:     CResManager::LokReFileDocument, public
//
//  Returns:    Put a document back on the quueue to be refiltered
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

void CResManager::LokReFileDocument ( PARTITIONID partid,
                                      WORKID wid,
                                      USN usn,
                                      VOLUMEID volumeId,
                                      ULONG retries,
                                      ULONG secQRetries )
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    CPartition* pPart = _partList.LokGetPartition(partid);

    //==========================================
    CChangeTrans xact( *this, pPart );

    //
    // A usn of 0 is used for all refiled documents
    //
    pPart->LokUpdateDocument (xact, wid, 0, volumeId, CI_UPDATE_OBJ, retries, secQRetries);
    xact.LokCommit();
    //==========================================
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokAddToSecQueue
//
//  Synopsis:   Adds the given document to the secondary change log of the
//              partition.
//
//  Arguments:  [partid]       - PartitionId
//              [wid]          - WorkId of the document
//              [volumeId]     - Volume id
//              [cSecQRetries] - Sec Q Retry Count
//
//  History:    5-09-96   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokAddToSecQueue( PARTITIONID partid,
                                    WORKID wid,
                                    VOLUMEID volumeId,
                                    ULONG cSecQRetries )
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    CPartition* pPart = _partList.LokGetPartition(partid);

    //==========================================
    CChangeTrans xact( *this, pPart );
    pPart->LokAddToSecQueue (xact, wid, volumeId, cSecQRetries);
    xact.LokCommit();
    //==========================================
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokRefileSecQueueDocs
//
//  Synopsis:   Transfers the documents from the secondary queue to the primary
//              change queue.
//
//  History:    5-09-96   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokRefileSecQueueDocs()
{
    //
    // No refiling in push filtering
    //
    Win4Assert( !_fPushFiltering );

    CPartIter iter;
    for ( iter.LokInit(_partList); !iter.LokAtEnd(); iter.LokAdvance(_partList))
    {
        CPartition * pPart = iter.LokGet();

        //==========================================
        CChangeTrans    xact( *this, pPart );
        pPart->LokRefileSecQueue( xact );
        xact.LokCommit();
        //==========================================
    }
}


//+-------------------------------------------------------------------------
//
//  Member:     CResManager::LokTransferWordlist, public
//
//  Returns:    Transfer a wordlist from the filter manager
//              to the content index
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

BOOL CResManager::LokTransferWordlist ( PWordList& pWordList )
{
    BOOL fSuccess = TRUE;
    PARTITIONID partid = _docList.PartId();
    CPartition* pPart = _partList.LokGetPartition(partid);
    INDEXID iid = pWordList->GetId();


    TRY
    {
        //==========================================
        CIndexTrans  xact( *this );

        //
        // LokDone can fail while trying to add unfinished docs to the
        // change log. In that case we must disable further USN updates.
        // We should commit the change transaction immediately after the
        // LokDone succeeds.
        //
        CChangeTrans changeXact( *this, pPart );
        pPart->LokDone ( changeXact, iid, _docList );
        ciFAILTEST( STATUS_NO_MEMORY );
        changeXact.LokCommit();

        _fresh.LokAddIndex ( xact, iid, _activeDeletedIndex.GetIndex(),
                             _docList, *pWordList );
        ciDebugOut (( DEB_ITRACE, "CFilterManager::FilterDone "
                    "Success: transfering wordlist\n" ));

        pWordList->Done();

        //
        // This may be an empty wordlist: all documents were deleted or in error.
        // If so, just get rid of the wordlist.
        //

        if ( pWordList->IsEmpty() )
        {
#if CIDBG == 1
            unsigned cShouldHaveData = 0;
            unsigned cDocuments = _docList.Count();

            for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ )
            {
                switch ( _docList.Status(iDoc) )
                {
                case TOO_MANY_RETRIES:
                case SUCCESS:
                case TOO_MANY_BLOCKS_TO_FILTER:
                    cShouldHaveData++;
                    break;

                default:
                    break;
                }
            }

            if ( cShouldHaveData != 0 )
            {
                for ( unsigned iDoc = 0; iDoc < cDocuments; iDoc++ )
                {
                    ciDebugOut(( DEB_ERROR, "WID: 0x%x   STATUS: %d\n",
                                 _docList.Wid(iDoc), _docList.Status(iDoc) ));
                }
            }

            Win4Assert( cShouldHaveData == 0 );
#endif
            pWordList.Delete();
        }
        else
            pWordList.Transfer ( pPart );

        xact.LokCommit();

    }
    CATCH ( CException, e )
    {
        ciDebugOut(( DEB_ERROR,
                     "Exception 0x%x caught commiting wordlist.\n"
                     "delete wordlist and clear doclist\n",
                     e.GetErrorCode() ));
        fSuccess = FALSE;
    }
    END_CATCH

    _eventMerge.Set();  // wake up
    return fSuccess;
}


//+---------------------------------------------------------------------------
//
//  Function:   LokNoFailRemoveWordlist
//
//  Synopsis:   Removes the biggest wordlist from memory and adds the
//              documents back to the change log.
//
//  Requires:   _docList.Count() == 0  when this method is called. This
//              relies on the fact that there are no other outstanding
//              wordlists being constructed.
//
//  History:    10-20-94   srikants   Added the header section and modified
//                                    to deal with the change in refiling
//                                    documents.
//
//----------------------------------------------------------------------------

void CResManager::LokNoFailRemoveWordlist( CPartition * pPart )
{
    //
    // In push filtering, we don't delete completed wordlist to simplify
    // the logic of aborting the notification transaction on the client
    //
    if ( _isBeingEmptied || _fPushFiltering )
        return;

    //
    // Look for biggest wordlist
    //

    unsigned cInd;
    CIndex ** apIndex = pPart->LokQueryMergeIndexes( cInd, mtWordlist );
    XPtrST<CIndex *> xIndex( apIndex );

    unsigned size = 0;
    CWordList * pWordList = 0;

    for ( unsigned i = 0; i < cInd; i++ )
    {
        apIndex[i]->Release();

        if ( apIndex[i]->Size() > size )
        {
            Win4Assert( apIndex[i]->IsWordlist() );

            pWordList = (CWordList *)apIndex[i];
            size = pWordList->Size();
        }
    }


    if ( pWordList )
    {
        //
        // Reschedule contents for later filtering.
        //

        INDEXID iid = pWordList->GetId();
        ciDebugOut(( DEB_ITRACE,
                 "Removing wordlist %x to reduce resource consumption.\n",
                  iid ));

        CDocList _docList;
        _docList.SetPartId ( pPart->GetId() );

        pWordList->GetDocuments( _docList );

#if CIDBG == 1
        for ( unsigned cWid = 0; cWid < _docList.Count(); cWid++ )
        {
            ciDebugOut(( DEB_ITRACE, "Reschedule filtering of wid %d\n",
                         _docList.Wid( cWid ) ));
        }
#endif // CIDBG == 1

        LokNoFailReFileChanges ( _docList );

        //
        // Remove it
        //

        ciDebugOut(( DEB_WARN, "Deleting wordlist with USN (%x, %x).\n",
                 lltoHighPart(pWordList->Usn()), (ULONG) pWordList->Usn() ));

        pPart->LokRemoveIndex( iid );

        if ( !pWordList->InUse() )
        {
            _partList.LokRemoveIndex ( iid );
            delete pWordList;
        }
        else
        {
            pWordList->Zombify();
        }
    }
}


//+-------------------------------------------------------------------------
//
//  Member:     CResManager::LokMakeWordlistId, public
//
//  Returns:    Makes unique wordlist id
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

INDEXID CResManager::LokMakeWordlistId (PARTITIONID partid)
{
    CPartition* pPart = _partList.LokGetPartition(partid);

    if ( pPart == 0 )
        return iidInvalid;

    // Get unique index id
    INDEXID iid = pPart->LokMakeWlstId ();
    ciDebugOut((DEB_FILTERWIDS,
                "CResManager::LokMakeWorkListId - new wordlist iid: %x\n", iid));

    return iid;
}


//+-------------------------------------------------------------------------
//
//  Member:      CResManager::ReportFilterFailure, public
//
//  Synopsis:    Write to event log about unsuccessful filtering
//
//  Arguments:   [wid] -- workid of the file that couldn't be filtered
//
//  History:    05-Jan-95 BartoszM     Created
//
//--------------------------------------------------------------------------

void CResManager::ReportFilterFailure (WORKID wid)
{
    PROPVARIANT var;
    var.vt = VT_UI4;
    var.ulVal = wid;

    _adviseStatus->NotifyStatus( CI_NOTIFY_FILTERING_FAILURE,
                                 1,
                                 &var );
}


//=========
// SCANNING
//=========


//+---------------------------------------------------------------------------
//
//  Member:     CResManager::SetUpdateLost
//
//  Synopsis:   Sets that an update got lost and wakes up the merge thread
//              to notify the client about it.
//
//  Arguments:  [partId] - Partition id
//
//  History:    1-24-97   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::SetUpdatesLost( PARTITIONID partId )
{
    CPriLock    lock(_mutex);
    if ( _state.LokGetState() != eUpdatesToBeDisabled )
    {
       _state.LokSetState( eUpdatesToBeDisabled );
       _state.LokSetUpdateType( eIncremental );
       StopCurrentMerge();
       _eventMerge.Set();
    }

}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokCleanupForFullCiScan, public
//
//  History:    10-Nov-94   DwightKr    Created
//
//  Notes:      This routine performs the disk I/O and other operations
//              which does the cleanup after a corruption is detected.
//
//              This routine is similar to the Empty() method, except
//              that we do not delete the change log object or the
//              freshlog object, and the index list must contain
//              the change log and fresh log after the cleanup is complete.
//
//----------------------------------------------------------------------------
void CResManager::LokCleanupForFullCiScan()
{
    if ( 0 != _pMerge )                     // Stop in progress merges.
        _pMerge->LokAbort();

    //
    //  Delete everything from the index table here first.
    //
    _idxTab->LokEmpty();

    //
    //  If anything fails after this point, chkdsk /f or autochk will
    //  release the disk storage associated with the leaked objects
    //  since they are no longer part of the persistent index list.
    //

    //
    //  For each partition, zombify all indexes, and re-add the change
    //  log to the index table on disk.
    //
    CPartIter iter;
    for ( iter.LokInit(_partList);
         !iter.LokAtEnd();
          iter.LokAdvance(_partList))
    {
        CPartition* pPart = iter.LokGet();
        Win4Assert( pPart != NULL );

        pPart->LokDisableUpdates();

        //
        //  Zombify all indexes in this partition, and delete them if
        //  their ref-count is 0.
        //
        unsigned cIndex;
        CIndex ** aIndex = pPart->LokZombify( cIndex );
        ReleaseIndexes( cIndex, aIndex, NULL );

        delete [] aIndex;

        WORKID widMMergeLog;
        WORKID widDummy;
        pPart->GetMMergeObjectIds( widMMergeLog, widDummy, widDummy );
        if ( widMMergeLog != widInvalid)
            _storage.RemoveObject( widMMergeLog );

        //
        //  Empty the fresh test and the fresh log, and add it to the
        //  index table on disk.
        //
        pPart->LokEmpty();          // Release change list/log storage

        WORKID oidChangeLog = pPart->GetChangeLogObjectId();
        Win4Assert( oidChangeLog != widInvalid );
       _partList.LokAddIt( oidChangeLog, itChangeLog, pPart->GetId() );
    }

    //
    //  Empty the fresh test, fresh log, and add the fresh log to
    //  the index table on disk.
    //
    _fresh.LokInit();               // Release fresh test/log storage
    WORKID oidFreshLog = _storage.GetSpecialItObjectId(itFreshLog);
    Win4Assert( oidFreshLog != widInvalid );
    _partList.LokAddIt( oidFreshLog, itFreshLog, partidFresh1 );

#ifdef KEYLIST_ENABLED
    WORKID widKeyList = _storage.GetSpecialItObjectId( itMMKeyList );
#else
    WORKID widKeyList = widInvalid;
#endif  // KEYLIST_ENABLED

    WORKID widPhrLat  = _storage.GetSpecialItObjectId( itPhraseLat );

    if ( widKeyList != widInvalid)
    {
        _storage.RemoveObject( widKeyList );
        _storage.SetSpecialItObjectId( itMMKeyList, widInvalid );
    }

    if ( widPhrLat != widInvalid)
    {
        _storage.RemoveObject( widPhrLat );
        _storage.SetSpecialItObjectId( itPhraseLat, widInvalid );
    }

    _isBeingEmptied = FALSE;                // Allow new queries
}

// ===========================================================
// Content Index Frame Work Interfacing.
// ===========================================================

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::WorkidToDocName
//
//  Synopsis:   Converts the given workid to path and fill it in the buffer
//              given.
//
//  Arguments:  [wid]       -- Workid to convert.
//              [pbBuf]     --  Destination buffer;
//              [cb]        -- On input the total number of bytes that can be
//                             put. On output, the actual number of bytes copied.
//                             For a deleted file, the cb will be 0.
//              [fAccurate] -- TRUE --> Attempt to get a more accurate
//                             workid-to-path translation.
//
//  Returns:    TRUE if successfully converted; FALSE means the buffer was
//              not big enough.
//
//  History:    12-04-96   srikants   Created
//
//  Notes:      WorkidToDocName is called outside the resman lock. It is a
//              potentially long operation and taking resman lock is not
//              advisable. Also, creation of a CWorkidToDocName object once
//              for each wid->docName conversion is expensive. So, we create
//              one in resman constructor and protect it with a separate
//              lock.
//
//----------------------------------------------------------------------------

BOOL CResManager::WorkidToDocName( WORKID wid,
                                   BYTE * pbBuf,
                                   unsigned & cb,
                                   BOOL fAccurate )
{

    CLock   lock(_workidToDocNameMutex);

    ULONG cbDocName;

    BYTE const * pbDocName = _xWorkidToDocName->GetDocName( wid, cbDocName, fAccurate );

    BOOL fStatus = TRUE;

    if ( pbDocName )
    {
        ciDebugOut(( DEB_FILTERWIDS | DEB_NOCOMPNAME, 
                     "WorkidToDocName: wid is %d and DocName is %ws\n",
                     wid, 
                     pbDocName ));

        if ( cbDocName > cb )
            fStatus = FALSE;
        else
            RtlCopyMemory( pbBuf, pbDocName, cbDocName );

        cb = cbDocName;
    }
    else
    {
        cb = 0;
    }

    return fStatus;
} // WorkidToDocName

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::GetClientState
//
//  Synopsis:   Gets the client specific status information like total
//              number of documents.
//
//  Arguments:  [cDocuments]    - [out] Total # of documents
//
//  History:    12-04-96   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::GetClientState( ULONG & cDocuments )
{
    CI_CLIENT_STATUS    status;
    SCODE sc = _docStore->GetClientStatus( &status );
    if ( S_OK == sc )
    {
        cDocuments = status.cDocuments;
    }
    else
    {
        cDocuments = 0;
    }
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokNotifyCorruptionToClient
//
//  Synopsis:   Informs the client that the catalog is corrupt
//
//  History:    12-04-96   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokNotifyCorruptionToClient()
{
    Win4Assert( _isCorrupt );

    DisableUpdates( partidDefault );

    SCODE sc = _docStore->DisableUpdates( FALSE, CI_CORRUPT_INDEX );
    if ( SUCCEEDED( sc ) )
        _state.LokSetCorruptionNotified();

    //
    // Disabled following assert so that the general NT 5 user is not
    // bothered by this assert.
    //

    //Win4Assert( ! "We must empty the Content Index" );
    // Empty();
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokNotifyDisableUpdatesToClient
//
//  Synopsis:   Informs the client that updates should be disabled
//
//  History:    07-05-97   SitaramR   Created
//
//----------------------------------------------------------------------------

void CResManager::LokNotifyDisableUpdatesToClient()
{
    Win4Assert( _state.LokGetState() == eUpdatesToBeDisabled );

    DisableUpdates( partidDefault );
    BOOL fIncremental = _state.LokGetUpdateType() == eIncremental;

    SCODE sc = _docStore->DisableUpdates( fIncremental, CI_LOST_UPDATE );
    if ( SUCCEEDED(sc) )
    {
        ciDebugOut(( DEB_WARN, "Notified disabled updates to DocStore\n" ));
        _state.LokSetState( eUpdatesDisabled );
    }
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::LokNotifyEnableUpdatesToClient
//
//  Synopsis:   Informs the client that updates should be enabled
//
//  History:    07-05-97   SitaramR   Created
//
//----------------------------------------------------------------------------

void CResManager::LokNotifyEnableUpdatesToClient()
{
   Win4Assert( _state.LokGetState() == eUpdatesToBeEnabled );
   EnableUpdates( partidDefault );

   SCODE sc = _docStore->EnableUpdates();
   if ( SUCCEEDED(sc) )
   {
      ciDebugOut(( DEB_WARN, "Notified enable updates to DocStore\n" ));
      _state.LokSetState( eSteady );
   }
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::StoreValue
//
//  Synopsis:   Stores the given value in the document store
//
//  Arguments:  [wid] - WorkId of the document
//              [ps]  - The FULLPROPSPEC of the property
//              [var] - Value of the property
//              [fInSchema] - Returns TRUE if the property is in the schema
//
//  Returns:    SCODE result
//
//  History:    12-18-96   srikants   Created
//
//----------------------------------------------------------------------------

SCODE CResManager::StoreValue( WORKID wid,
                               CFullPropSpec const & ps,
                               CStorageVariant const & var,
                               BOOL & fInSchema )
{
    FULLPROPSPEC const * fps = ps.CastToStruct();
    PROPVARIANT const * pVar = ConvertToPropVariant( &var );

    fInSchema = TRUE;

    SCODE sc = _propStore->StoreProperty( wid, fps, pVar );

    if ( CI_E_PROPERTY_NOT_CACHED == sc )
        fInSchema = FALSE;
    else if ( FAILED( sc ) )
        return sc;

    return S_OK;
}

BOOL CResManager::StoreSecurity( WORKID wid,
                                 PSECURITY_DESCRIPTOR pSD,
                                 ULONG cbSD )
{
    return S_OK == _docStore->StoreSecurity( wid, (BYTE *) pSD, cbSD );
}

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::FPSToPROPID, public
//
//  Synopsis:   Converts FULLPROPSPEC property to PROPID
//
//  Arguments:  [fps] -- FULLPROPSPEC representation of property
//              [pid] -- PROPID written here on success
//
//  Returns:    S_OK on success
//
//  History:    29-Dec-1997  KyleP      Created.
//
//----------------------------------------------------------------------------

SCODE CResManager::FPSToPROPID( CFullPropSpec const & fps, PROPID & pid )
{
    return _mapper->PropertyToPropid( fps.CastToStruct(), TRUE, &pid );
}

//
// Flush notify support
//

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::NotifyFlush
//
//  Synopsis:   Notifies the client that changes got flushed and gives the
//              flush information.
//
//  Arguments:  [ft]       - FileTime of the last successful flush.
//              [cEntries] - Number of entries in the aInfo
//              [aInfo]    - Array of USN information for the flush.
//
//  History:    1-27-97   srikants   Created
//
//  Note:       We should not call directly into framework client because
//              it violates resman->client lock hierarchy. We should not call
//              into client with resman lock held.
//
//----------------------------------------------------------------------------

void CResManager::NotifyFlush( FILETIME const & ft,
                               ULONG cEntries,
                               USN_FLUSH_INFO ** ppInfo )
{
    if ( !_fPushFiltering )
    {
       //
       // Notification of flushes are sent in pull filtering only.
       // For push filtering, the ICiCIndexNotificationStatus
       // interface is used to notify clients.
       //

       CPriLock   lock(_mutex);

       CChangesFlushNotifyItem * pItem =
               new CChangesFlushNotifyItem( ft, cEntries, ppInfo );
       _flushList.Queue( pItem );

       if (! _fFlushWorkerActive)
       {
           CFwFlushNotify * pWorkItem = new CFwFlushNotify( _workMan, *this );
           _workMan.AddToWorkList( pWorkItem );

           pWorkItem->AddToWorkQueue();
           pWorkItem->Release();
        }
    }
} //NotifyFlush

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::ProcessFlushNotifies
//
//  Synopsis:   Processes the queued flush notifies in a FIFO order. This
//              method is called only by the asynchronous worker thread
//              responsible for notifying the client about flushes.
//
//  History:    1-27-97   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::ProcessFlushNotifies()
{

   //
   // Only one worker thread allowed to operate on the list at a time.
   //
   CLock lock (_mtxChangeFlush );
   CRevertBoolValue RevertFlushActive( _fFlushWorkerActive, TRUE );

   while ( TRUE )
   {
      CChangesFlushNotifyItem * pItem = 0;

      //
      // Remove the first flush item from the list under resman lock.
      //

      //===========================================
      {
         CPriLock   lock(_mutex);
         pItem = _flushList.GetFirst();
         if ( pItem )
            _flushList.Pop();

         if (_flushList.IsEmpty())
             RevertFlushActive.Revert();
      }
      //===========================================

      if ( pItem )
      {

         SCODE sc = _docStore->CheckPointChangesFlushed(
                         pItem->GetFlushTime(),
                         pItem->GetCount(),
                         pItem->GetFlushInfo() );

         if ( FAILED(sc) )
         {
             //
             // We don't have to requeue this item. The client's restart
             // work will just increase.
             //
             ciDebugOut(( DEB_ERROR,
                          "CheckPointChangesFlushed failed. Error (0x%X)\n",
                          sc ));
         }

         pItem->Close();
         delete pItem;
      }
      else
      {
         break;
      }
   } // While (TRUE)
} //ProcessFlushNotifies

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::_LokDeleteFlushWorkItems
//
//  Synopsis:   Deletes the flush notifies in the queue. This is called
//              during shutdown (dismount) to clean up any asynchronous
//              worker threads referring to resman.
//
//  History:    1-27-97   srikants   Created
//
//----------------------------------------------------------------------------

void CResManager::LokDeleteFlushWorkItems()
{
    for ( CChangesFlushNotifyItem * pItem = _flushList.Pop();
          0 != pItem;
          pItem = _flushList.Pop() )
    {
        pItem->Close();
        delete pItem;
    }
} //LokDeleteFlushWorkItems

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::NoFailAbortWidsInDocList
//
//  Synopsis:   Used in push filtering to abort all wids in the current
//              doclist
//
//  History:    24-Feb-97   SitaramR   Created
//
//----------------------------------------------------------------------------

void CResManager::NoFailAbortWidsInDocList()
{
   CPriLock lock ( _mutex );

   if ( _isBeingEmptied )
       return ;

   for ( unsigned i=0; i<_docList.Count(); i++ )
   {
       WORKID wid = _docList.Wid(i);
       USN usn = _docList.Usn(i);

       //
       // Abort the client transaction
       //
       _xIndexNotifTable->AbortWid( wid, usn );

       //
       // Keep track of aborted wids for use during
       // changlog shrink from front.
       //
       _aAbortedWids.NoFailLokAddWid( wid, usn );
   }

   _docList.LokClear();
} //NoFailAbortWidsInDocList

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::NoFailAbortWid
//
//  Synopsis:   Used in push filtering to abort a wid
//
//  Arguments:  [wid]  -- Workid
//              [usn]  -- Usn
//
//  History:    24-Feb-97   SitaramR   Created
//
//----------------------------------------------------------------------------

void CResManager::NoFailAbortWid( WORKID wid, USN usn )
{
    CPriLock lock ( _mutex );

    if ( _isBeingEmptied )
        return ;

    //
    // Abort the client transaction
    //
    _xIndexNotifTable->AbortWid( wid, usn );

    //
    // Keep track of aborted wid for use during
    // changlog shrink from front.
    //
    _aAbortedWids.NoFailLokAddWid( wid, usn );
} //NoFailAbortWid

//+---------------------------------------------------------------------------
//
//  Member:     CResManager::ClearNonStoragePropertiesForWid
//
//  Synopsis:   Clear non-storage properties from the property storage for wid
//
//  Arguments:  [wid]  -- Workid
//
//  History:    10-Oct-2000   KitmanH   Created
//
//----------------------------------------------------------------------------

SCODE CResManager::ClearNonStoragePropertiesForWid( WORKID wid )
{
    return _propStore->ClearNonStoragePropertiesForWid( wid );
}

//====================
// MASTER MERGE POLICY
//====================

//+---------------------------------------------------------------------------
//
//  Member:     CMasterMergePolicy::IsTimeToMasterMerge, public
//
//  Synopsis:   Determines if the system should start a master merge
//              immediately, and not wait until the master merge time.
//
//  History:    4-Aug-94    DwightKr        Created
//
//  Notes:      If new tests are added simply add them here.
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsTimeToMasterMerge() const
{
    return IsMaxShadowIndexSize()    ||
           IsMaxFreshListCount()     ||
           IsMinDiskFreeForceMerge();
}

//+---------------------------------------------------------------------------
//
//  Member:     CMasterMergePolicy::IsMinDiskFreeForceMerge, private
//
//  Synopsis:   If the amount of free disk space is lower than a threshold,
//              then we should master merge, since it is likely that disk
//              space will be released.
//
//  History:    4-Aug-94    DwightKr        Created
//              Jan-07-96   mohamedn        CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMinDiskFreeForceMerge() const
{
    //
    //  If we've exceeded the MIN_DISKFREE_FORCE_MERGE space used on
    //  the system, and a master merge has the potential to make some
    //  useful progress, then it's time to force a master merge.  The
    //  values of MIN_DISKFREE_FORCE_MERGE and MAX_SHADOW_FREESPACE
    //  are expressed as a percentage.
    //
    __int64 diskTotal, diskRemaining;
    _storage.GetDiskSpace( diskTotal, diskRemaining );

    if ( diskRemaining < minDiskFreeForMerge )
        return FALSE;

    if (diskRemaining*100/diskTotal < _frmwrkParams.GetMinDiskFreeForceMerge() &&
        _shadowIndexSize*100/diskRemaining > _frmwrkParams.GetMaxShadowFreeForceMerge() )
    {
       CDmFwEventItem item(  EVENTLOG_INFORMATION_TYPE,
                             MSG_CI_MASTER_MERGE_STARTED_TOTAL_DISK_SPACE_USED,
                             2);

       item.AddArg( _storage.GetVolumeName() );
       item.AddArg( _frmwrkParams.GetMinDiskFreeForceMerge() );

       item.ReportEvent(_adviseStatus);

       ciDebugOut(( DEB_ITRACE | DEB_PENDING,
                    "Master merge, reason: Low on disk space\n" ));

       return TRUE;
    }

    return FALSE;
}

//+---------------------------------------------------------------------------
//
//  Member:     CMasterMergePolicy::IsMaxFreshListCount, private
//
//  Synopsis:   If the number of entries in the fresh list is great than
//              a threshold, we should master merge, and free up memory
//              occupied by the fresh hash.
//
//  History:    4-Aug-94    DwightKr        Created
//              Jan-07-96   mohamedn        CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMaxFreshListCount() const
{
    if ( _freshCount > _frmwrkParams.GetMaxFreshCount() )
    {

       CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE,
                            MSG_CI_MASTER_MERGE_STARTED_FRESH_LIST_COUNT,
                            2);

       item.AddArg( _storage.GetVolumeName() );
       item.AddArg( _frmwrkParams.GetMaxFreshCount() );

       item.ReportEvent(_adviseStatus);

       ciDebugOut(( DEB_ITRACE | DEB_PENDING,
                    "Master merge, reason: More than %d entries in freshhash\n",
                     _frmwrkParams.GetMaxFreshCount() ));

       return TRUE;
    }

    return FALSE;
}

//+---------------------------------------------------------------------------
//
//  Member:     CMasterMergePolicy::IsMaxShadowIndexSize, private
//
//  Synopsis:   If the combined size of all shadow indexes is greater than
//              a threahold, we should master merge, and free up disk space.
//              The content index should not take up more than a given
//              percentage of disk space.
//
//  History:    4-Aug-94    DwightKr        Created
//              Jan-07-96   mohamedn        CDmFwEventItem
//
//----------------------------------------------------------------------------
BOOL CMasterMergePolicy::IsMaxShadowIndexSize() const
{
    __int64 diskTotal, diskRemaining;
    _storage.GetDiskSpace( diskTotal, diskRemaining );

    if ( diskRemaining < minDiskFreeForMerge )
        return FALSE;

    if ( _shadowIndexSize*100/diskTotal > _frmwrkParams.GetMaxShadowIndexSize() )
    {

       CDmFwEventItem item( EVENTLOG_INFORMATION_TYPE,
                            MSG_CI_MASTER_MERGE_STARTED_SHADOW_INDEX_SIZE,
                            2);

       item.AddArg( _storage.GetVolumeName() );
       item.AddArg( _frmwrkParams.GetMaxShadowIndexSize() );

       item.ReportEvent(_adviseStatus);

       ciDebugOut(( DEB_ITRACE | DEB_PENDING,
                    "Master merge, reason: Combined shadow indexes > %d%% of disk\n",
                     _frmwrkParams.GetMaxShadowIndexSize() ));

       return TRUE;
    }

    return FALSE;
}

//+---------------------------------------------------------------------------
//
//  Method:     CMasterMergePolicy::ComputeMidNightMergeTime
//
//  Synopsis:   Function to compute the time (in hundreds of nanoseconds
//              from midnight Dec 31, 1969) for the next master merge based on
//              the value given from registry.
//
//  Arguments:  [mergeTime] -  The time read from the registry control params.
//                             It's an offset in minutes after midnight.
//
//  Returns:    LONG - time for next merge
//
//  History:    19-Feb-1996  srikants   Moved the code from svcfilt.cxx
//              17-Nov-1999  KyleP      Stop using CRT
//
//  Notes:      If the time is before January 1, 1970 it will return -1.
//
//----------------------------------------------------------------------------

ULONGLONG CMasterMergePolicy::ComputeMidNightMergeTime( LONG mergeTime )
{
    Win4Assert( mergeTime < 24 * 60 );
    mergeTime *= 60;

    //
    // What time is it here?
    //

    SYSTEMTIME stLocal;
    GetLocalTime( &stLocal );

    LONG lNow = stLocal.wHour*60*60 + stLocal.wMinute*60 + stLocal.wSecond;

    //
    // Have we passed the daily merge time?
    //

    LONGLONG llHNSTilMerge;   // Hundred-Nano-Second-Til-Merge
    LONGLONG const llHNSOneDay = 24*60*60*10000000i64;

    if ( lNow < mergeTime )
        llHNSTilMerge = mergeTime * 10000000i64;
    else
        llHNSTilMerge = mergeTime * 10000000i64 + llHNSOneDay;

    //
    // Create the *local* filetime for the next merge
    //

    stLocal.wHour = 0;
    stLocal.wMinute = 0;
    stLocal.wSecond = 0;
    stLocal.wMilliseconds = 1;

    FILETIME ftLocal;
    if ( !SystemTimeToFileTime( &stLocal, &ftLocal ) )
        return -1;

    ((ULARGE_INTEGER *)&ftLocal)->QuadPart += llHNSTilMerge;

    //
    // Now, adjust to UTC and return
    //

    FILETIME ft;

    if ( !LocalFileTimeToFileTime( &ftLocal, &ft ) )
        return -1;

#if CIDBG == 1
    FILETIME ft2;

    GetSystemTimeAsFileTime( &ft2 );

    Win4Assert( ((ULARGE_INTEGER *)&ft2)->QuadPart < ((ULARGE_INTEGER *)&ft)->QuadPart );
#endif // CIDBG

    return ((ULARGE_INTEGER *)&ft)->QuadPart;
} //ComputeMidNightMergeTime

//+---------------------------------------------------------------------------
//
//  Member:     CMasterMergePolicy::IsMidNightMergeTime, public
//
//  Synopsis:   Determines if it's been 24 hours since the last merge time
//
//  History:     4-Aug-1994  DwightKr   Created
//              17-Nov-1999  KyleP      Stop using CRT
//
//----------------------------------------------------------------------------

BOOL CMasterMergePolicy::IsMidNightMergeTime()
{
    FILETIME ft;
    GetSystemTimeAsFileTime( &ft );

    if ( ((ULARGE_INTEGER *)&ft)->QuadPart > _mergeTime )
    {
        ULONG mergeTime = _frmwrkParams.GetMasterMergeTime();

        ULONGLONG oldMergeTime = _mergeTime;

        _mergeTime = ComputeMidNightMergeTime( mergeTime );

        //
        // If it's been more than 30 minutes since the master merge time, don't
        // do it.  This can happen if we were doing a master merge as part of
        // indexing documents or if you turn on a machine in the morning and it
        // was off at the master merge time.
        //

        LONGLONG const llHNSHalfHour = 30*60*10000000i64;

        if ( ((ULARGE_INTEGER *)&ft)->QuadPart > ( oldMergeTime + llHNSHalfHour ) )
            return FALSE;

        return TRUE;
    }

    return FALSE;
} //IsMidNightMergeTime