windows-server-2003/inetsrv/query/ntciutil/phystr.cxx

//+---------------------------------------------------------------------------
//
//  Microsoft Corporation
//  Copyright (C) Microsoft Corporation, 1992 - 1998.
//
//  File:       PhysIdx.CXX
//
//  Contents:   FAT Buffer/Index package
//
//  Classes:    CPhysBuffer -- Buffer
//
//  History:    05-Mar-92   KyleP       Created
//              07-Aug-92   KyleP       Kernel implementation
//              21-Apr-93   BartoszM    Rewrote to use memory mapped files
//
//----------------------------------------------------------------------------

#include <pch.cxx>
#pragma hdrstop

#include <pstore.hxx>
#include <phystr.hxx>
#include <eventlog.hxx>
#include <psavtrak.hxx>

//+-------------------------------------------------------------------------
//
//  Function:   PageToLow
//
//  Returns:    Low part of byte count in cPage pages
//
//  History:    21-Apr-93       BartoszM        Created
//
//--------------------------------------------------------------------------

inline ULONG PageToLow(ULONG cPage)
{
    return(cPage << CI_PAGE_SHIFT);
}

//+-------------------------------------------------------------------------
//
//  Function:   PageToHigh
//
//  Returns:    High part of byte count in cPage pages
//
//  History:    21-Apr-93       BartoszM        Created
//
//--------------------------------------------------------------------------

inline ULONG PageToHigh(ULONG cPage)
{
    return(cPage >> (ULONG_BITS - CI_PAGE_SHIFT));
}

//+-------------------------------------------------------------------------
//
//  Function:   ToPages
//
//  Returns:    Number of pages equivalent to (cbLow, cbHigh) bytes
//
//  History:    21-Apr-93       BartoszM        Created
//
//--------------------------------------------------------------------------

inline ULONG ToPages ( ULONG cbLow, ULONG cbHigh )
{
    Win4Assert ( cbHigh >> CI_PAGE_SHIFT == 0 );
    return (cbLow >> CI_PAGE_SHIFT) + (cbHigh << (ULONG_BITS - CI_PAGE_SHIFT));
}

//+-------------------------------------------------------------------------
//
//  Function:   PgCommonPgTrunc
//
//  Returns:    Number of pages truncated to multiple of largest common page
//
//  History:    21-Apr-93       BartoszM        Created
//
//--------------------------------------------------------------------------

inline ULONG PgCommonPgTrunc(ULONG nPage)
{
    return nPage & ~(PAGES_PER_COMMON_PAGE - 1);
}

inline ULONGLONG MAKEULONGLONG( ULONG l, ULONG h )
{
    return ( (ULONGLONG) l | ( (ULONGLONG) h << 32 ) );
}

//+-------------------------------------------------------------------------
//
//  Function:   PgCommonPgRound
//
//  Returns:    Number of pages rounded to multiple of largest common page
//
//  History:    21-Apr-93       BartoszM        Created
//
//--------------------------------------------------------------------------

inline ULONG PgCommonPgRound(ULONG nPage)
{
    return (nPage + PAGES_PER_COMMON_PAGE - 1) & ~(PAGES_PER_COMMON_PAGE - 1);
}

//+-------------------------------------------------------------------------
//
//  Member:     CPhysBuffer::~CPhysBuffer
//
//  History:    07-Dec-93   BartoszM    Created
//
//--------------------------------------------------------------------------

CPhysBuffer::~CPhysBuffer()
{
    Win4Assert( !_fNeedToFlush );

#if CIDBG == 1
    Flush( FALSE );
#endif
}

//+-------------------------------------------------------------------------
//
//  Member:     CPhysBuffer::CPhysBuffer, public
//
//  Synopsis:   Allocates virtual memory
//
//  Arguments:  [stream]         -- stream to use
//              [PageNo]         -- page number
//              [fWrite]         -- for writable memory
//              [fIntentToWrite] -- When fWrite is TRUE, whether the
//                                  client intends to write to the memory so
//                                  it needs to be flushed.
//
//  History:    10-Mar-92 KyleP     Created
//              21-Apr-93 BartoszM  Rewrote to use memory mapped files
//
//--------------------------------------------------------------------------

CPhysBuffer::CPhysBuffer(
    PMmStream& stream,
    ULONG      PageNo,
    BOOL       fWrite,
    BOOL       fIntentToWrite,
    BOOL       fMap ) :
    _cRef( 0 ),
    _pphysNext( 0 ),
    _fNeedToFlush( fIntentToWrite ),
    _fWrite( fWrite ),
    _fMap( fMap ),
    _stream( stream )
{
    _PageNo = PgCommonPgTrunc( PageNo );

    if ( _fMap )
        stream.Map( _strmBuf,
                    COMMON_PAGE_SIZE,
                    PageToLow( _PageNo ),
                    PageToHigh( _PageNo ),
                    fWrite );
    else
    {
        DWORD cbRead;

        _xBuffer.Init( COMMON_PAGE_SIZE );
        _stream.Read( _xBuffer.Get(),
                      MAKEULONGLONG( PageToLow( _PageNo ),
                                     PageToHigh( _PageNo ) ),
                      COMMON_PAGE_SIZE,
                      cbRead );
    }
} //CPhysBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysBuffer::Flush, public
//
//  Synopsis:   Flushes the buffer to disk
//
//  Arguments:  [fFailFlush]     -- If TRUE and flush fails, an exception is
//                                  raised.  Otherwise failures to flush are
//                                  ignored.
//
//  History:    10/30/98    dlee    Moved out of header
//
//--------------------------------------------------------------------------

void CPhysBuffer::Flush( BOOL fFailFlush )
{
    if ( _fNeedToFlush )
    {
        Win4Assert( _fWrite );

        //
        // Reset the flag first, so subsequent attempts to flush won't
        // fail if first attempt fails.  This is so we don't throw
        // in destructors.  If we can't tolerate failures to flush, don't
        // even bother trying to write the data if we can't fail.
        //

        _fNeedToFlush = FALSE;

        if ( fFailFlush )
        {
            if ( _fMap )
            {
                // Flush the buffer and metadata in one call

                _strmBuf.Flush( TRUE );
            }
            else
            {
                _stream.Write( _xBuffer.Get(),
                               MAKEULONGLONG( PageToLow( _PageNo ),
                                              PageToHigh( _PageNo ) ),
                               COMMON_PAGE_SIZE );
            }
        }
        else
        {
            ciDebugOut(( DEB_WARN, "not flushing at %#p, in failure path?\n",
                         this ));
        }
    }

#if CIDBG == 1

    //
    // Make sure the bits on disk are the same as those in memory, since
    // the buffer is marked as not needing to be flushed.  Only do this if
    // using read/write instead of mapping.
    //

    else if ( !_fMap && fFailFlush )
    {
        TRY
        {
            XArrayVirtual<BYTE> xTmp( COMMON_PAGE_SIZE );
    
            DWORD cbRead;
            _stream.Read( xTmp.Get(),
                          MAKEULONGLONG( PageToLow( _PageNo ),
                                         PageToHigh( _PageNo ) ),
                          COMMON_PAGE_SIZE,
                          cbRead );
    
            if ( 0 != memcmp( xTmp.Get(), _xBuffer.Get(), COMMON_PAGE_SIZE ) )
            {
                ciDebugOut(( DEB_ERROR, "read: %p, have: %p\n",
                             xTmp.Get(), _xBuffer.Get() ));
                Win4Assert( !"writable non-flushed buffer was modified!" );
            }
        }
        CATCH( CException, e )
        {
            // Ignore -- it's just an assert.
        }
        END_CATCH;
    }
#endif // CIDBG == 1

} //Flush
    
//+-------------------------------------------------------------------------
//
//  Member:     CBufferCacheIter::CBufferCacheIter, public
//
//  Synopsis:   Constructor
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

CBufferCacheIter::CBufferCacheIter ( CBufferCache& cache )
: _pPhysBuf(cache._aPhysBuf.GetPointer()), _idx(0), _cHash( cache._cHash )
{
    do
    {
        _pBuf = _pPhysBuf[_idx];
        if (_pBuf != 0)
            break;
        _idx++;
    } while ( _idx < _cHash );
} //CBufferCacheIter

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCacheIter::operator++, public
//
//  Synopsis:   Increments the iterator
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

void CBufferCacheIter::operator++()
{
    Win4Assert(_pBuf != 0);
    _pBuf = _pBuf->Next();
    while (_pBuf == 0 && ++_idx < _cHash )
        _pBuf = _pPhysBuf[_idx];
}

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::CBufferCache, public
//
//  Synopsis:   Constructor
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

CBufferCache::CBufferCache( unsigned cHash )
    : _cBuffers( 0 ),
      _cHash( cHash ),
      _aPhysBuf( cHash )
    #if CIDBG == 1
        , _cCacheHits( 0 ), _cCacheMisses( 0 )
    #endif

{
    RtlZeroMemory( _aPhysBuf.GetPointer(), _aPhysBuf.SizeOf() );
}

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::~CBufferCache, public
//
//  Synopsis:   Destructor
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

CBufferCache::~CBufferCache()
{
    Free( FALSE );
}

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::Free, public
//
//  Synopsis:   Deletes all buffers
//
//  Arguments:  [fFailFlush]  -- If TRUE, this method will THROW if the flush
//                               fails.  If FALSE, flush failures are
//                               ignored.
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

void CBufferCache::Free( BOOL fFailFlush )
{
    if ( 0 != _cBuffers )
    {
        for ( unsigned i = 0; i < _cHash; i++ )
        {
            // Keep the cache consistent in case we fail flushing.

            while ( 0 != _aPhysBuf[i] )
            {
                CPhysBuffer * pTemp = _aPhysBuf[i];

                pTemp->Flush( fFailFlush );

#if CIDBG == 1
                if ( pTemp->IsReferenced() )
                    ciDebugOut(( DEB_WARN,
                                 "Buffer for page %ld still referenced.\n",
                                 pTemp->PageNumber() ));
#endif // CIDBG == 1

                _aPhysBuf[i] = pTemp->Next();
                delete pTemp;
                _cBuffers--;
            }

            Win4Assert( 0 == _aPhysBuf[i] );
        }

        Win4Assert( 0 == _cBuffers );
    }
} //Free

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::TryToRemoveBuffers, public
//
//  Synopsis:   Tries to remove unreferenced buffers from the cache.
//              If one is not found, it's not a problem.
//
//  Arguments:  [cMax] -- Recommended maximum in the cache
//
//  History:    15-March-96 dlee     Created
//
//--------------------------------------------------------------------------

void CBufferCache::TryToRemoveBuffers( unsigned cMax )
{
    for ( unsigned i = 0; i < 3 && ( Count() > cMax ); i++ )
    {
        // find an unreferenced page with the lowest (oldest) usn.
    
        ULONG ulPage = ULONG_MAX;
        ULONG ulUSN = ULONG_MAX;
    
        for ( CBufferCacheIter iter(*this); !iter.AtEnd(); ++iter )
        {
            CPhysBuffer * pPhys = iter.Get();
    
            Win4Assert( 0 != pPhys );
    
            if ( ( ulUSN > pPhys->GetUSN() ) &&
                 ( !pPhys->IsReferenced() ) )
            {
                ulUSN = pPhys->GetUSN();
                ulPage = pPhys->PageNumber();
            }
        }
    
        if ( ULONG_MAX == ulPage )
            break;
        else
            Destroy( ulPage );
    }
} //TryToRemoveBuffers

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::Search, public
//
//  Synopsis:   Searches the buffer hash for the requested buffer.
//
//  Arguments:  [ulPage] -- Page number of page to be found.
//
//  Returns:    0 if page not currently buffered, or the pointer
//              to the (physical) buffer.
//
//  History:    09-Mar-92 KyleP     Created
//              17-Mar-93 BartoszM  Rewrote
//
//--------------------------------------------------------------------------

CPhysBuffer * CBufferCache::Search(
    ULONG hash,
    ULONG commonPage )
{
    // ciDebugOut (( DEB_ITRACE, "CBufferCache::Search %d\n", ulPage ));

    for ( CPhysBuffer *pPhys = _aPhysBuf[ hash ];
          0 != pPhys;
          pPhys = pPhys->Next() )
    {
        if ( pPhys->PageNumber() == commonPage )
            break;
    }

    #if DBG==1 || CIDBG==1
        if ( 0 == pPhys )
            _cCacheMisses++;
        else
            _cCacheHits++;
    #endif // DBG==1 || CIDBG==1

    return pPhys;
} //Search

//+---------------------------------------------------------------------------
//
//  Function:   Flush
//
//  Synopsis:   Flushes all cached pages upto and including the specified
//              page to disk.
//
//  Arguments:  [nHighPage] -- The "CI" page number below which all pages
//              must be flushed to disk.
//
//  History:    4-20-94   srikants   Created
//
//  Notes:      nHighPage is in units of "CI" page (4K bytes) and not
//              CommonPage which is 64K.
//
//----------------------------------------------------------------------------

void CBufferCache::Flush( ULONG nHighPage, BOOL fLeaveFlushFlagAlone )
{
    nHighPage = PgCommonPgTrunc(nHighPage);
    ciDebugOut(( DEB_ITRACE, "CBufferCache : Flushing All Pages <= 0x%x\n",
                 nHighPage ));

    if ( 0 == _cBuffers )
        return;

    //
    // We Must Flush All pages <= the given page number.
    //

    for ( CBufferCacheIter iter(*this); !iter.AtEnd(); ++iter )
    {
        CPhysBuffer * pPhys = iter.Get();

        Win4Assert( 0 != pPhys );

        if ( pPhys->PageNumber() <= nHighPage )
        {
            // Call flush even if not required to get assertions checked

            BOOL fRequiresFlush = pPhys->RequiresFlush();

            pPhys->Flush( TRUE );

            if ( fLeaveFlushFlagAlone && fRequiresFlush )
                pPhys->SetIntentToWrite();
        }
    }
} //Flush

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::Destroy, public
//
//  Synopsis:   Deletes a buffer.
//
//  Arguments:  [ulPage] -- page number to be deleted
//
//  History:    18-Mar-93 BartoszM     Created
//
//--------------------------------------------------------------------------

void CBufferCache::Destroy( ULONG ulPage, BOOL fFailFlush )
{
    ulPage = PgCommonPgTrunc(ulPage);

    ULONG iHash = hash( ulPage );

    CPhysBuffer * pPhys = _aPhysBuf[ iHash ];
    Win4Assert( 0 != pPhys );

    // Find the page.  Don't remove from the list until it is flushed.
    // Is it first? (common case)

    if (pPhys->PageNumber() == ulPage)
    {
        // This should be flushed before being unmapped. Fix for bug 132655

        pPhys->Flush( fFailFlush );

        _aPhysBuf[iHash] = pPhys->Next();
    }
    else
    {
        // Search linked list

        CPhysBuffer * pPhysPrev;
        do
        {
            pPhysPrev = pPhys;
            pPhys = pPhys->Next();
            Win4Assert( 0 != pPhys );
        }
        while (pPhys->PageNumber() != ulPage );

        // This should be flushed before being unmapped. Fix for bug 132655

        pPhys->Flush( fFailFlush );

        // delete from the linked list

        pPhysPrev->Link( pPhys->Next() );
        Win4Assert ( !pPhys->IsReferenced() );
    }

    _cBuffers--;
    delete pPhys;
} //Destroy

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::Add, public
//
//  Synopsis:   Adds a buffer to the buffer hash.
//
//  Arguments:  [pPhysBuf] -- Buffer to add.
//
//  History:    09-Mar-92 KyleP     Created
//
//--------------------------------------------------------------------------

void CBufferCache::Add( CPhysBuffer * pPhysBuf, ULONG hash )
{
    _cBuffers++;

    pPhysBuf->Reference();

    pPhysBuf->Link( _aPhysBuf[ hash ] );

    _aPhysBuf[ hash ] = pPhysBuf;
} //Add

//+-------------------------------------------------------------------------
//
//  Member:     CBufferCache::MinPageInUse, public
//
//  Synopsis:   Finds the smallest page in use within the cache
//
//  Returns :   TRUE if any page is in the cache; FALSE o/w
//              If TRUE is returned, then minPageInUse will contain the
//              minimum page that is present in the cache.
//
//  Arguments:  minPageInUse    (out) Will be set to the minimum page in
//              the cache if the return value is TRUE.
//
//  History:    02-May-94   DwightKr    Created
//              08-Dec-94   SrikantS    Fixed to return the correct value for
//                                      empty cache.
//
//--------------------------------------------------------------------------
BOOL CBufferCache::MinPageInUse(ULONG &minPageInUse)
{
    ULONG minPage = 0xFFFFFFFF;

    for (CBufferCacheIter iter(*this); !iter.AtEnd(); ++iter)
    {
        minPage = min(minPage, iter.Get()->PageNumber() );
    }

    if ( minPage == 0xFFFFFFFF )
    {
        return FALSE;
    }
    else
    {
        minPageInUse = minPage;
        return TRUE;
    }
} //MinPageInUse

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::CPhysStorage, public
//
//  Effects:    Creates a new index.
//
//  Arguments:  [storage]   -- physical storage to create index in
//              [obj]          -- Storage object
//              [objectId]  -- Index ID
//              [cPageReqSize] --
//              [stream]       -- Memory mapped stream
//              [fThrowCorruptErrorOnFailures  -- If true, then errors during
//                                                opening will be marked as
//                                                catalog corruption
//              [cCachedItems] -- # of non-referenced pages to cache
//              [fAllowReadAhead] -- if TRUE, non-mapped IO will be used
//                                   when the physical storage says it's ok.
//
//  Signals:    Cannot create.
//
//  Modifies:   A new index (file) is created on the disk.
//
//  History:    07-Mar-92 KyleP    Created
//
//--------------------------------------------------------------------------

CPhysStorage::CPhysStorage( PStorage & storage,
                            PStorageObject& obj,
                            WORKID objectId,
                            UINT cpageReqSize,
                            PMmStream * stream,
                            BOOL fThrowCorruptErrorOnFailures,
                            unsigned cCachedItems,
                            BOOL fAllowReadAhead )
        : _sigPhysStorage(eSigPhysStorage),
          _fWritable( TRUE ),
          _objectId( objectId ),
          _obj ( obj),
          _cpageUsedFileSize( 0 ),
          _storage(storage),
          _stream(stream),
          _iFirstNonShrunkPage( 0 ),
          _fThrowCorruptErrorOnFailures(fThrowCorruptErrorOnFailures),
          _cpageGrowth(1),
          _cMaxCachedBuffers( cCachedItems ),
          _cache( __max( 17, ( cCachedItems & 1 ) ? cCachedItems :
                                                    cCachedItems + 1 ) ),
          _usnGen( 0 ),
          _fAllowReadAhead( fAllowReadAhead )
{
    //
    // Use different locking schemes depending on usage
    //

    if ( 0 == _cMaxCachedBuffers )
        _xMutex.Set( new CMutexSem() );
    else
        _xRWAccess.Set( new CReadWriteAccess() );

    if ( _stream.IsNull() || !_stream->Ok() )
    {
        ciDebugOut((DEB_ERROR, "Index Creation failed %08x\n", objectId ));

        if ( _fThrowCorruptErrorOnFailures || _stream->FStatusFileNotFound() )
        {
           //
           // We don't have code to handle such failures, hence mark
           // catalog as corrupt; otherwise throw e_fail
           //
           Win4Assert( !"Corrupt catalog" );
           _storage.ReportCorruptComponent( L"PhysStorage1" );
           THROW( CException( CI_CORRUPT_DATABASE ));
        }
        else
           THROW( CException( E_FAIL ));
    }

    if (cpageReqSize == 0)
        cpageReqSize = 1;

    TRY
    {
        _GrowFile( cpageReqSize );
    }
    CATCH( CException, e )
    {
        //
        // There may not be enough space on the disk to allocate the
        // requested size.  If not enough space, do the best that we
        // can and hope space gets freed.
        //

        _GrowFile( 1 );
    }
    END_CATCH

    ciDebugOut(( DEB_ITRACE, "Physical index %08x created.\n", _objectId ));
}

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::CPhysStorage, public
//
//  Effects:    Opens an existing index.
//
//  Arguments:  [storage]   -- physical storage to open index in
//              [obj]       -- Storage object
//              [objectId]  -- Index ID.
//              [stream]    -- Memory mapped stream
//              [mode]      -- Open mode
//              [fThrowCorruptErrorOnFailures  -- If true, then errors during
//                                                opening will be marked as
//                                                catalog corruption
//              [cCachedItems] -- # of non-referenced pages to cache
//              [fAllowReadAhead] -- if TRUE, non-mapped IO will be used
//                                   when the physical storage says it's ok.
//
//  History:    07-Mar-92 KyleP    Created
//
//--------------------------------------------------------------------------

CPhysStorage::CPhysStorage( PStorage & storage,
                            PStorageObject& obj,
                            WORKID objectId,
                            PMmStream * stream,
                            PStorage::EOpenMode mode,
                            BOOL fThrowCorruptErrorOnFailures,
                            unsigned cCachedItems,
                            BOOL fAllowReadAhead )
        : _sigPhysStorage(eSigPhysStorage),
          _fWritable( PStorage::eOpenForWrite == mode ),
          _objectId( objectId ),
          _obj ( obj ),
          _cpageUsedFileSize( 0 ),
          _storage(storage),
          _stream(stream),
          _fThrowCorruptErrorOnFailures(fThrowCorruptErrorOnFailures),
          _cpageGrowth(1),
          _iFirstNonShrunkPage( 0 ),
          _cMaxCachedBuffers( cCachedItems ),
          _cache( __max( 17, ( cCachedItems & 1 ) ? cCachedItems :
                                                    cCachedItems + 1 ) ),
          _usnGen( 0 ),
          _fAllowReadAhead( fAllowReadAhead )
{
    //
    // Use different locking schemes depending on usage
    //

    if ( 0 == _cMaxCachedBuffers )
        _xMutex.Set( new CMutexSem() );
    else
        _xRWAccess.Set( new CReadWriteAccess() );

    if ( _stream.IsNull() || !_stream->Ok() )
    {
        ciDebugOut(( DEB_ERROR, "Open of index %08x failed\n", objectId ));

        if ( _fThrowCorruptErrorOnFailures || _stream->FStatusFileNotFound() )
        {
           //
           // We don't have code to handle such failures, hence mark
           // catalog as corrupt; otherwise throw e_fail
           //
           Win4Assert( !"Corrupt catalog" );
           _storage.ReportCorruptComponent( L"PhysStorage2" );
           THROW( CException( CI_CORRUPT_DATABASE ));
        }
        else
           THROW( CException( E_FAIL ));
    }

    _cpageFileSize = ToPages(_stream->SizeLow(), _stream->SizeHigh());

    ciDebugOut(( DEB_ITRACE, "Physical index %08x (%d pages) opened.\n",
            _objectId, _cpageFileSize ));
}


//+---------------------------------------------------------------------------
//
//  Member:     CPhysStorage::MakeBackupCopy
//
//  Synopsis:   Makes a backup copy of the current storage using the
//              destination storage.
//
//  Arguments:  [dst]             - Destination storage.
//              [progressTracker] - Progress tracking
//
//  History:    3-18-97   srikants   Created
//
//----------------------------------------------------------------------------

void CPhysStorage::MakeBackupCopy( CPhysStorage & dst,
                                   PSaveProgressTracker & progressTracker )
{
    //
    // Copy one page at a time.
    //
    ULONG * pSrcNext = 0;
    ULONG * pDstNext = 0;

    unsigned iCurrBorrow = 0; // always has the current buffers borrowed.

    SCODE sc = S_OK;

    TRY
    {
        //
        // Grow the destination to be as big as the current one.
        //
        dst._GrowFile( _cpageFileSize );

        if ( _cpageFileSize > 0 )
        {
            iCurrBorrow = 0;

            pSrcNext = BorrowBuffer(iCurrBorrow, FALSE );
            pDstNext = dst.BorrowNewBuffer(iCurrBorrow);

            const unsigned iLastPage = _cpageFileSize-1;

            for ( unsigned j = 0; j < _cpageFileSize; j++ )
            {
                ULONG * pSrc = pSrcNext;
                ULONG * pDst = pDstNext;

                Win4Assert( 0 != pSrc && 0 != pDst );

                RtlCopyMemory( pDst, pSrc, CI_PAGE_SIZE );

                //
                // Before returning buffers, grab the next ones to prevent
                // the large page from being taken out of cache.
                //

                if ( j != iLastPage )
                {
                    iCurrBorrow = j+1;
                    pSrcNext = BorrowBuffer(iCurrBorrow, FALSE );
                    pDstNext = dst.BorrowNewBuffer(iCurrBorrow);
                }
                else
                {
                    pSrcNext = pDstNext = 0;
                }

                ReturnBuffer( j );
                dst.ReturnBuffer( j, TRUE );

                if ( progressTracker.IsAbort() )
                    THROW( CException( STATUS_TOO_LATE ) );

                //
                // Update the progress every 64 K.
                //
                if ( (j % PAGES_PER_COMMON_PAGE) == (PAGES_PER_COMMON_PAGE-1) )
                    progressTracker.UpdateCopyProgress( (ULONG) j, _cpageFileSize );
            }
        }

        dst.Flush();
    }
    CATCH( CException, e )
    {
        if ( pSrcNext )
            ReturnBuffer( iCurrBorrow );

        if ( pDstNext )
            dst.ReturnBuffer(iCurrBorrow);

        sc = e.GetErrorCode();
    }
    END_CATCH

    if ( S_OK != sc )
    {
        //
        // _GrowFile throws CI_CORRUPT_DATABASE if it cannot create
        // the file that big. We don't want to consider that a
        // corruption.
        //
        if ( CI_CORRUPT_DATABASE == sc )
            sc = E_FAIL;

        THROW( CException( sc ) );
    }

    //
    // Consider 100% of the copy work is done.
    //
    progressTracker.UpdateCopyProgress( 1,1 );
}


//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::~CPhysStorage, public
//
//  Synopsis:   closes index.
//
//  History:    09-Mar-92 KyleP     Created
//
//  Notes:      Don't write back pages. This is either a read only
//              index or we are aborting a merge. Pages are written
//              back after a successful merge using Reopen().
//
//--------------------------------------------------------------------------

CPhysStorage::~CPhysStorage()
{
    ciDebugOut(( DEB_ITRACE, "Physical index %lx closed.\n", _objectId ));

    _cache.Free( FALSE );
}

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::Close, public
//
//  Effects:    Closes the stream.
//
//  History:    07-Mar-92 KyleP    Created
//
//--------------------------------------------------------------------------

void CPhysStorage::Close()
{
    _cache.Free();
    _stream.Free();
}

//+---------------------------------------------------------------------------
//
//  Function:   Flush
//
//  Synopsis:   Flushes all the pages that were unmapped since the last
//              flush.
//
//  History:    4-29-94   srikants   Created
//
//  Notes:
//
//----------------------------------------------------------------------------

void CPhysStorage::Flush( BOOL fLeaveFlushFlagAlone )
{
    Win4Assert( _fWritable );

    // Grab the lock to make sure the pages aren't deleted while
    // they are being flushed.

    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        _cache.Flush( ULONG_MAX, fLeaveFlushFlagAlone );
    }
    else
    {
        CReadAccess readLock( _xRWAccess.GetReference() );

        _cache.Flush( ULONG_MAX, fLeaveFlushFlagAlone );
    }
} //Flush

//+---------------------------------------------------------------------------
//
//  Function:   ShrinkToFit
//
//  Synopsis:   Reduces the size of the stream by giving up pages in the end
//              which are not needed.
//
//  Arguments:  (none)
//
//  History:    9-08-94   srikants   Created
//
//  Notes:
//
//----------------------------------------------------------------------------

void CPhysStorage::ShrinkToFit()
{
    _cache.Free();
    _stream->SetSize( _storage,
                      PageToLow(_cpageUsedFileSize),
                      PageToHigh(_cpageUsedFileSize) );
    _cpageFileSize = _cpageUsedFileSize;
}

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::Reopen, public
//
//  Synopsis:   Flushes, closes, and reopens for reading.
//
//  History:    09-Mar-92 KyleP     Created
//              26-Aug-92 BartoszM  Separated from destructor
//              06-May-94 Srikants  Add fSetSize as a parameter to truncate
//                                  the stream optionally.
//
//--------------------------------------------------------------------------
void CPhysStorage::Reopen( BOOL fWritable )
{
    // No need to lock.

    _cache.Free(); // will flush all buffers

    if ( _fWritable && !_stream.IsNull() )
    {
        //
        // Give it an accurate used length.
        //

        _stream->SetSize( _storage,
                          PageToLow(_cpageUsedFileSize),
                          PageToHigh(_cpageUsedFileSize) );

        _cpageFileSize = _cpageUsedFileSize;

        _stream.Free();

        //
        // reopen shadow indexes as read and master indexes as write to
        // support shrink from front.
        //

        _fWritable = fWritable;
    }

    //
    // reopen it with the new mode
    //

    ReOpenStream();

    if ( _stream.IsNull() || !_stream->Ok() )
    {
        ciDebugOut(( DEB_ERROR, "Re-Open file of index %08x failed\n",
                     _objectId ));

        if ( _fThrowCorruptErrorOnFailures || _stream->FStatusFileNotFound() )
        {
           //
           // We don't have code to handle such failures, hence mark
           // catalog as corrupt; otherwise throw e_fail
           //
           Win4Assert( !"Corrupt catalog" );
           _storage.ReportCorruptComponent( L"PhysStorage3" );
           THROW( CException( CI_CORRUPT_DATABASE ));
        }
        else
           THROW( CException( E_FAIL ));
    }

    _cpageFileSize = ToPages(_stream->SizeLow(), _stream->SizeHigh());

    ciDebugOut(( DEB_ITRACE, "Physical index %08x opened.\n", _objectId ));
} //Reopen

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::LokBorrowNewBuffer, public
//
//  Arguments:  [hash]        -- hash value for doing the lookup
//              [commonPage]] -- the common page of nPage
//              [nPage]       -- page number
//
//  Returns:    A buffer for a new (unused) page.
//
//  History:    03-Mar-98 dlee    Created from existing BorrowNewBuffer
//
//  Notes:      On creation, the buffer is filled with zeros.
//
//--------------------------------------------------------------------------

ULONG * CPhysStorage::LokBorrowNewBuffer(
    ULONG hash,
    ULONG commonPage,
    ULONG nPage )
{
    CPhysBuffer* pBuf = _cache.Search( hash, commonPage );

    if (pBuf)
    {
        pBuf->Reference();
        Win4Assert( nPage < _cpageFileSize );

        // fIntentToWrite should be TRUE

        pBuf->SetIntentToWrite();
    }
    else
    {
        // If we've moved beyond the previous end of file, set the new
        // size of file.

        if ( nPage >= _cpageFileSize )
            _GrowFile( nPage + _cpageGrowth );

        // First try to release unreferenced buffers if the cache is full.

        _cache.TryToRemoveBuffers( _cMaxCachedBuffers );

        pBuf = new CPhysBuffer( _stream.GetReference(),
                                nPage,
                                TRUE,          // writable
                                TRUE,          // intent to write
                                ! ( _fAllowReadAhead &&
                                    _storage.FavorReadAhead() ) );
        _cache.Add( pBuf, hash );
    }

    Win4Assert( nPage + 1 > _cpageUsedFileSize );

    _cpageUsedFileSize = nPage + 1;
    Win4Assert( _cpageUsedFileSize <= _cpageFileSize );

    RtlZeroMemory( pBuf->GetPage( nPage ), CI_PAGE_SIZE );

    return pBuf->GetPage( nPage );
} //LokBorrowNewBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::BorrowNewBuffer, public
//
//  Arguments:  [nPage] -- page number
//
//  Returns:    A buffer for a new (unused) page.
//
//  Signals:    No more disk space.
//
//  History:    09-Mar-92 KyleP     Created
//
//  Notes:      On creation, the buffer is filled with zeros.
//
//--------------------------------------------------------------------------

ULONG * CPhysStorage::BorrowNewBuffer( ULONG nPage )
{
    Win4Assert( _fWritable );

    ULONG commonPage = PgCommonPgTrunc( nPage );
    ULONG hash = _cache.hash( commonPage );

    // Use appropriate locking based on whether any pages are in the cache.

    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        return LokBorrowNewBuffer( hash, commonPage, nPage );
    }
    else
    {
        CWriteAccess writeLock( _xRWAccess.GetReference() );

        return LokBorrowNewBuffer( hash, commonPage, nPage );
    }
} //BorrowNewBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::LokBorrowOrAddBuffer, public
//
//  Arguments:  [hash]           -- hash value for doing the lookup
//              [commonPage]]    -- the common page of nPage
//              [nPage]          -- page number
//              [fAdd]           -- if TRUE, create the page if not found
//                                  if FALSE and not found, return 0
//              [fWritable]      -- if TRUE, the page is writable
//              [fIntentToWrite] -- TRUE if the caller intends to write
//
//  Returns:    A buffer loaded with page [nPage]
//
//  History:    09-Mar-92 KyleP     Created
//
//--------------------------------------------------------------------------

ULONG * CPhysStorage::LokBorrowOrAddBuffer(
    ULONG hash,
    ULONG commonPage,
    ULONG nPage,
    BOOL  fAdd,
    BOOL  fWritable,
    BOOL  fIntentToWrite )
{
    // First, look in the cache.  Either we haven't looked yet (if no
    // caching) or another thread may have added it between the time the
    // read lock was released and the write lock taken.

    Win4Assert( commonPage >= _iFirstNonShrunkPage );
    CPhysBuffer * pbuf = _cache.Search( hash, commonPage );

    if ( 0 != pbuf )
    {
        Win4Assert( !fWritable || pbuf->IsWritable() );

        pbuf->Reference();
        if ( fIntentToWrite )
            pbuf->SetIntentToWrite();

        return pbuf->GetPage( nPage );
    }

    ULONG * pulRet = 0;

    if ( fAdd )
    {
        pbuf = new CPhysBuffer( _stream.GetReference(),
                                nPage,
                                fWritable,
                                fIntentToWrite,
                                ! ( _fAllowReadAhead &&
                                    _storage.FavorReadAhead() ) );
    
        _cache.Add( pbuf, hash );
        pulRet = pbuf->GetPage( nPage );
    
        // Try to release unreferenced buffers if the cache is full.
    
        _cache.TryToRemoveBuffers( _cMaxCachedBuffers );
    }

    return pulRet;
} //LokBorrowOrAddBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::BorrowBuffer, public
//
//  Arguments:  [nPage] -- Page number of buffer to find.
//
//  Returns:    A buffer loaded with page [nPage]
//
//  Signals:    Out of memory?  Buffer not found?
//
//  History:    09-Mar-92 KyleP     Created
//
//--------------------------------------------------------------------------

ULONG * CPhysStorage::BorrowBuffer(
    ULONG nPage,
    BOOL  fWritable,
    BOOL  fIntentToWrite )
{
    ULONG commonPage = PgCommonPgTrunc( nPage );
    ULONG hash = _cache.hash( commonPage );

    Win4Assert( fWritable || !fIntentToWrite );

    // Make sure we didn't walk off the end.

    if ( nPage >= _cpageFileSize )
    {
        ciDebugOut(( DEB_WARN, "Asking for page %d, file size %d pages\n",
                     nPage, _cpageFileSize ));
        Win4Assert( !"BorrowBuffer walked off end of file" );

        _storage.ReportCorruptComponent( L"PhysStorage4" );

        THROW( CException( CI_CORRUPT_DATABASE ) );
    }

    // If caching, try finding it in the cache first under a read lock
    // This is the fast path for when the entire property cache is in memory.

    if ( 0 != _cMaxCachedBuffers )
    {
        CReadAccess readLock( _xRWAccess.GetReference() );

        ULONG * p = LokBorrowOrAddBuffer( hash, commonPage, nPage, FALSE,
                                          fWritable, fIntentToWrite );
        if ( 0 != p )
            return p;
    }

    // Ok, maybe we have to add it.

    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        return LokBorrowOrAddBuffer( hash, commonPage, nPage, TRUE,
                                     fWritable, fIntentToWrite );
    }
    else
    {
        CWriteAccess writeLock( _xRWAccess.GetReference() );

        return LokBorrowOrAddBuffer( hash, commonPage, nPage, TRUE,
                                     fWritable, fIntentToWrite );
    }
} //BorrowBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::ReturnBuffer, public
//
//  Synopsis:   Dereference and return the buffer to the pool.
//
//  Arguments:  [nPage]  -- Page number of buffer being returned.
//              [fFlush] -- TRUE if the buffer should be flushed if this
//                          is the last reference to the buffer.
//              [fFailFlush] -- TRUE if an exception should be raised if
//                              the flush fails.  Note that even if fFlush
//                              is FALSE, the buffer may still be flushed
//                              because previously it was borrowed with
//                              intent to write.
//
//  Notes:      If flush throws, this call doesn't return the buffer.
//
//  History:    16-Mar-93       BartoszM     Created
//
//--------------------------------------------------------------------------

void CPhysStorage::ReturnBuffer(
    ULONG nPage,
    BOOL  fFlush,
    BOOL  fFailFlush )
{
    ULONG commonPage = PgCommonPgTrunc( nPage );
    ULONG hash = _cache.hash( commonPage );

    // if caching is off, grab the write lock, else the read lock

    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        Win4Assert( commonPage >= _iFirstNonShrunkPage );
        CPhysBuffer *pbuf = _cache.Search( hash, commonPage );

        Win4Assert( 0 != pbuf );
        Win4Assert( pbuf->IsReferenced() );

        // If the refcount is 1, it's about to be destroyed.  Do the Flush
        // now, in case it fails, before mucking with any data structures.

        BOOL fDestroy = pbuf->IsRefCountOne();

        if ( fDestroy && fFlush )
        {
            Win4Assert( pbuf->IsWritable() );
            pbuf->Flush( fFailFlush );
        }

        pbuf->DeReference( _usnGen++ );

        if ( fDestroy )
        {
            Win4Assert( !pbuf->IsReferenced() );

            _cache.Destroy( nPage, fFailFlush );
            Win4Assert( !_cache.Search( hash, commonPage ) );
        }
    }
    else
    {
        CReadAccess readLock( _xRWAccess.GetReference() );

        Win4Assert( commonPage >= _iFirstNonShrunkPage );
        CPhysBuffer *pbuf = _cache.Search( hash, commonPage );

        Win4Assert( 0 != pbuf );
        Win4Assert( pbuf->IsReferenced() );

        if ( fFlush )
        {
            Win4Assert( pbuf->IsWritable() );

            if ( pbuf->IsRefCountOne() )
                pbuf->Flush( fFailFlush );
        }

        pbuf->DeReference( _usnGen++ );

        // leave it in the cache -- it'll be cleared out by BorrowBuffer
    }
} //ReturnBuffer

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::RequiresFlush, public
//
//  Synopsis:   Determines if the page is scheduled for flushing
//
//  Arguments:  [nPage]  -- Page number to check
//
//  Returns:    TRUE if the buffer will be flushed if Flush or Destroy is
//              called.
//
//  History:    3-Nov-98       dlee     Created
//
//--------------------------------------------------------------------------

BOOL CPhysStorage::RequiresFlush( ULONG nPage )
{
    ULONG commonPage = PgCommonPgTrunc( nPage );
    ULONG hash = _cache.hash( commonPage );

    // if caching is off, grab the write lock, else the read lock

    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        Win4Assert( commonPage >= _iFirstNonShrunkPage );
        CPhysBuffer *pbuf = _cache.Search( hash, commonPage );

        Win4Assert( 0 != pbuf );
        Win4Assert( pbuf->IsReferenced() );

        return pbuf->RequiresFlush();
    }
    else
    {
        CReadAccess readLock( _xRWAccess.GetReference() );

        Win4Assert( commonPage >= _iFirstNonShrunkPage );
        CPhysBuffer *pbuf = _cache.Search( hash, commonPage );

        Win4Assert( 0 != pbuf );
        Win4Assert( pbuf->IsReferenced() );

        return pbuf->RequiresFlush();
    }
} //RequiresFlush

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::_GrowFile, private
//
//  Synopsis:   Increases the physical (disk) size of the file.
//
//  Arguments:  [cpageSize] -- New file size, in pages.
//
//  Signals:    Out of space.
//
//  History:    09-Mar-92 KyleP     Created
//
//--------------------------------------------------------------------------

void CPhysStorage::_GrowFile( ULONG cpageSize )
{
    Win4Assert( cpageSize > 0 );

    cpageSize = PgCommonPgRound(cpageSize);

    ciDebugOut(( DEB_ITRACE, "  Growing Index to %d pages\n", cpageSize ));

    _stream->SetSize( _storage,
                      PageToLow(cpageSize),
                      PageToHigh(cpageSize) );

    if (!_stream->Ok())
    {
        ciDebugOut(( DEB_ERROR, "GrowFile of index %08x failed: %d\n",
                     _objectId ));

        if ( _fThrowCorruptErrorOnFailures )
        {
           //
           // We don't have code to handle such failures, hence mark
           // catalog as corrupt; otherwise throw e_fail
           //
           Win4Assert( !"Corrupt catalog" );
           _storage.ReportCorruptComponent( L"PhysStorage5" );
           THROW( CException( CI_CORRUPT_DATABASE ) );
        }
        else
           THROW( CException( E_FAIL ) );
    }

    _cpageFileSize = cpageSize;
} //_GrowFile

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::ShrinkFromFront, public
//
//  Synopsis:   Makes the front part of a file sparse
//
//  Arguments:  [iFirstPage] -- first 4k page -- 64k granular
//              [cPages]     -- number of 4k pages
//
//  Returns:    The # of 4k pages actually shrunk, maybe 0
//
//  History:    09-Jan-97 dlee     Moved from .hxx
//
//--------------------------------------------------------------------------

ULONG CPhysStorage::ShrinkFromFront(
    ULONG iFirstPage,
    ULONG cPages )
{
    ULONG cShrunk = 0;

    if ( _storage.SupportsShrinkFromFront() )
    {
        //
        // Make sure the caller isn't leaving any gaps and was paying
        // attention to the return value from previous calls.
        //

        Win4Assert( iFirstPage == _iFirstNonShrunkPage );

        //
        // We must shrink on common page boundaries since we borrow on
        // common pages (even though the api is page granular)
        //

        ULONG cPagesToShrink = PgCommonPgTrunc( cPages );

        if ( 0 == cPagesToShrink )
            return 0;

        Win4Assert( _iFirstNonShrunkPage ==
                    PgCommonPgTrunc( _iFirstNonShrunkPage ) );
        Win4Assert( iFirstPage == PgCommonPgTrunc( iFirstPage ) );

        // Take a lock so no one else tries to borrow or free any pages

        Win4Assert ( 0 == _cMaxCachedBuffers );

        CLock lock( _xMutex.GetReference() );

        cShrunk = _stream->ShrinkFromFront( iFirstPage, cPagesToShrink );
        Win4Assert( cShrunk == PgCommonPgTrunc( cShrunk ) );

        _iFirstNonShrunkPage += cShrunk;
    }

    return cShrunk;
} //ShrinkFromFront

//+-------------------------------------------------------------------------
//
//  Member:     CPhysStorage::MinPageInUse, public
//
//  Synopsis:   Finds the smallest page in use within the cache
//
//  Arguments:  [minPage] -- returns the result
//
//  Returns :   TRUE if any page is in the cache; FALSE o/w
//              If TRUE is returned, then minPage will contain the
//              minimum page that is present in the cache.
//
//  History:    26-Mar-98 dlee     Moved from .hxx
//
//--------------------------------------------------------------------------

BOOL CPhysStorage::MinPageInUse( ULONG &minPage )
{
    if ( 0 == _cMaxCachedBuffers )
    {
        CLock lock( _xMutex.GetReference() );

        return _cache.MinPageInUse(minPage);
    }
    else
    {
        CReadAccess readLock( _xRWAccess.GetReference() );

        return _cache.MinPageInUse(minPage);
    }
} //MinPageInUse