You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
532 lines
13 KiB
532 lines
13 KiB
//====== Copyright © 1996-2005, Valve Corporation, All rights reserved. =======
|
|
//
|
|
// Purpose:
|
|
//
|
|
//=============================================================================
|
|
|
|
#include "basefilesystem.h"
|
|
|
|
// NOTE: This has to be the last file included!
|
|
#include "tier0/memdbgon.h"
|
|
|
|
|
|
|
|
bool CompareFilenames( const char *pa, const char *pb )
|
|
{
|
|
// Case-insensitive and path separator-insensitive compare.
|
|
const char *a = pa;
|
|
const char *b = pb;
|
|
while ( *a && *b )
|
|
{
|
|
char ca = *a;
|
|
char cb = *b;
|
|
|
|
if ( ca >= 'a' && ca <= 'z' )
|
|
ca = 'A' + (ca - 'a');
|
|
else if ( ca == '/' )
|
|
ca = '\\';
|
|
|
|
if ( cb >= 'a' && cb <= 'z' )
|
|
cb = 'A' + (cb - 'a');
|
|
else if ( cb == '/' )
|
|
cb = '\\';
|
|
|
|
if ( ca != cb )
|
|
return false;
|
|
|
|
++a;
|
|
++b;
|
|
}
|
|
|
|
// Filenames also must be the same length.
|
|
if ( *a != *b )
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// CFileTracker.
|
|
//-----------------------------------------------------------------------------
|
|
|
|
CFileTracker::CFileTracker( CBaseFileSystem *pFileSystem )
|
|
{
|
|
m_pFileSystem = pFileSystem;
|
|
}
|
|
|
|
CFileTracker::~CFileTracker()
|
|
{
|
|
Clear();
|
|
}
|
|
|
|
void CFileTracker::NoteFileLoadedFromDisk( const char *pFilename, const char *pPathID, FileHandle_t fp )
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
if ( !pPathID )
|
|
pPathID = "";
|
|
|
|
CPathIDFileList *pPath = GetPathIDFileList( pPathID );
|
|
CFileInfo *pInfo = pPath->FindFileInfo( pFilename );
|
|
|
|
if ( m_pFileSystem->m_WhitelistSpewFlags & WHITELIST_SPEW_WHILE_LOADING )
|
|
{
|
|
if ( pInfo )
|
|
Warning( "(Duplicate): [%s]\\%s", pPathID, pFilename );
|
|
else
|
|
Warning( "(Unique ): [%s]\\%s", pPathID, pFilename );
|
|
}
|
|
|
|
if ( pInfo )
|
|
{
|
|
// Clear all the flags, but remember if we ever had a CRC.
|
|
pInfo->m_Flags &= k_eFileFlagsGotCRCOnce;
|
|
pInfo->m_Flags &= ~k_eFileFlagsFailedToLoadLastTime;
|
|
}
|
|
else
|
|
{
|
|
pInfo = pPath->AddFileInfo( pFilename );
|
|
pInfo->m_Flags = (EFileFlags)0;
|
|
}
|
|
|
|
if ( !fp )
|
|
{
|
|
if ( m_pFileSystem->m_WhitelistSpewFlags & WHITELIST_SPEW_WHILE_LOADING )
|
|
{
|
|
Warning( "\n" );
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
|
|
// Remember that we calculated the CRC and that it is unverified.
|
|
pInfo->m_CRC = CalculateCRCForFile( fp );
|
|
pInfo->m_Flags |= k_eFileFlagsHasCRC | k_eFileFlagsGotCRCOnce;
|
|
if ( pInfo->m_iNeedsVerificationListIndex == -1 )
|
|
pInfo->m_iNeedsVerificationListIndex = m_NeedsVerificationList.AddToTail( pInfo );
|
|
|
|
if ( m_pFileSystem->m_WhitelistSpewFlags & WHITELIST_SPEW_WHILE_LOADING )
|
|
{
|
|
Warning( " - %u\n", pInfo->m_CRC );
|
|
}
|
|
}
|
|
|
|
void CFileTracker::NoteFileFailedToLoad( const char *pFilename, const char *pPathID )
|
|
{
|
|
CPathIDFileList *pPath = GetPathIDFileList( pPathID );
|
|
CFileInfo *pInfo = pPath->FindFileInfo( pFilename );
|
|
if ( pInfo )
|
|
{
|
|
pInfo->m_Flags |= k_eFileFlagsFailedToLoadLastTime;
|
|
}
|
|
}
|
|
|
|
CRC32_t CFileTracker::CalculateCRCForFile( FileHandle_t fp )
|
|
{
|
|
CRC32_t crc;
|
|
|
|
// Calculate the CRC.
|
|
unsigned int initialFilePos = m_pFileSystem->Tell( fp );
|
|
m_pFileSystem->Seek( fp, 0, FILESYSTEM_SEEK_HEAD );
|
|
|
|
#define CRC_CHUNK_SIZE (32*1024)
|
|
char tempBuf[CRC_CHUNK_SIZE];
|
|
|
|
CRC32_Init( &crc );
|
|
|
|
unsigned int fileLength = m_pFileSystem->Size( fp );
|
|
|
|
int nChunks = fileLength / CRC_CHUNK_SIZE + 1;
|
|
unsigned int curStartByte = 0;
|
|
for ( int iChunk=0; iChunk < nChunks; iChunk++ )
|
|
{
|
|
int curEndByte = min( curStartByte + CRC_CHUNK_SIZE, fileLength );
|
|
int chunkLen = curEndByte - curStartByte;
|
|
if ( chunkLen == 0 )
|
|
break;
|
|
|
|
m_pFileSystem->Read( tempBuf, chunkLen, fp ); // TODO: handle errors here..
|
|
CRC32_ProcessBuffer( &crc, tempBuf, chunkLen );
|
|
|
|
curStartByte += CRC_CHUNK_SIZE;
|
|
}
|
|
CRC32_Final( &crc );
|
|
|
|
// Go back to where we started.
|
|
m_pFileSystem->Seek( fp, initialFilePos, FILESYSTEM_SEEK_HEAD );
|
|
return crc;
|
|
}
|
|
|
|
CFileInfo* CFileTracker::GetFileInfo( const char *pFilename, const char *pPathID )
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
CPathIDFileList *pPath = GetPathIDFileList( pPathID, false );
|
|
if ( !pPath )
|
|
return NULL;
|
|
|
|
return pPath->FindFileInfo( pFilename );
|
|
}
|
|
|
|
int CFileTracker::GetFileInfos( CFileInfo **ppFileInfos, int nMaxFileInfos, const char *pFilename )
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
int nOut = 0;
|
|
for ( int i=m_PathIDs.First(); i != m_PathIDs.InvalidIndex(); i=m_PathIDs.Next( i ) )
|
|
{
|
|
CFileInfo *pCur = m_PathIDs[i]->FindFileInfo( pFilename );
|
|
if ( pCur )
|
|
{
|
|
if ( nOut < nMaxFileInfos )
|
|
{
|
|
ppFileInfos[nOut++] = pCur;
|
|
}
|
|
else
|
|
{
|
|
Assert( !"CFileTracker::GetFileInfos - overflowed list!" );
|
|
}
|
|
}
|
|
}
|
|
|
|
return nOut;
|
|
}
|
|
|
|
void CFileTracker::NoteFileLoadedFromSteam( const char *pFilename, const char *pPathID, bool bForcedLoadFromSteam )
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
if ( !pPathID )
|
|
pPathID = "";
|
|
|
|
CPathIDFileList *pPath = GetPathIDFileList( pPathID );
|
|
CFileInfo *pInfo = pPath->FindFileInfo( pFilename );
|
|
if ( !pInfo )
|
|
pInfo = pPath->AddFileInfo( pFilename );
|
|
|
|
if ( m_pFileSystem->m_WhitelistSpewFlags & WHITELIST_SPEW_WHILE_LOADING )
|
|
{
|
|
Warning( "From Steam: [%s]\\%s\n", pPathID, pFilename );
|
|
}
|
|
|
|
pInfo->m_Flags = k_eFileFlagsLoadedFromSteam;
|
|
if ( bForcedLoadFromSteam )
|
|
pInfo->m_Flags |= k_eFileFlagsForcedLoadFromSteam;
|
|
}
|
|
|
|
|
|
void CFileTracker::CalculateMissingCRCs( IFileList *pWantCRCList )
|
|
{
|
|
// First build a list of files that need a CRC and don't have one.
|
|
m_Mutex.Lock();
|
|
CUtlLinkedList<CFileInfo*,int> needCRCList;
|
|
|
|
for ( int i=m_PathIDs.First(); i != m_PathIDs.InvalidIndex(); i=m_PathIDs.Next( i ) )
|
|
{
|
|
CPathIDFileList *pPath = m_PathIDs[i];
|
|
FOR_EACH_LL( pPath->m_Files, j )
|
|
{
|
|
CFileInfo *pInfo = pPath->m_Files[j];
|
|
|
|
if ( !( pInfo->m_Flags & k_eFileFlagsLoadedFromSteam ) && !( pInfo->m_Flags & k_eFileFlagsHasCRC ) )
|
|
{
|
|
// If the new "force match" list doesn't care whether the file has a CRC or not, then don't bother to calculate it.
|
|
if ( !pWantCRCList->IsFileInList( pInfo->m_pFilename ) )
|
|
continue;
|
|
|
|
needCRCList.AddToTail( pInfo );
|
|
}
|
|
}
|
|
}
|
|
|
|
m_Mutex.Unlock();
|
|
|
|
// Then, when the mutex is not locked, go generate the CRCs for them.
|
|
FOR_EACH_LL( needCRCList, i )
|
|
{
|
|
CFileInfo *pInfo = needCRCList[i];
|
|
CalculateMissingCRC( pInfo->m_pFilename, pInfo->GetPathIDString() );
|
|
}
|
|
}
|
|
|
|
|
|
void CFileTracker::CacheFileCRC( const char *pPathID, const char *pRelativeFilename )
|
|
{
|
|
Assert( ThreadInMainThread() );
|
|
|
|
// Get the file's info. Load the file if necessary.
|
|
CFileInfo *pInfo = GetFileInfo( pRelativeFilename, pPathID );
|
|
if ( !pInfo )
|
|
{
|
|
CalculateMissingCRC( pRelativeFilename, pPathID );
|
|
pInfo = GetFileInfo( pRelativeFilename, pPathID );
|
|
}
|
|
|
|
if ( !pInfo )
|
|
return;
|
|
|
|
// Already cached a CRC for this file?
|
|
if ( !( pInfo->m_Flags & k_eFileFlagsGotCRCOnce ) )
|
|
{
|
|
// Ok, it's from disk but we don't have the CRC.
|
|
CalculateMissingCRC( pInfo->GetFilename(), pInfo->GetPathIDString() );
|
|
}
|
|
}
|
|
|
|
|
|
void CFileTracker::CacheFileCRC_Copy( const char *pPathID, const char *pRelativeFilename, const char *pPathIDToCopyFrom )
|
|
{
|
|
Assert( ThreadInMainThread() );
|
|
|
|
// Get the file's info. Load the file if necessary.
|
|
CFileInfo *pSourceInfo = GetFileInfo( pRelativeFilename, pPathIDToCopyFrom );
|
|
if ( !pSourceInfo || !( pSourceInfo->m_Flags & k_eFileFlagsGotCRCOnce ) )
|
|
{
|
|
// Strange, we don't have a CRC for the one they wanted to copy from, so calculate that CRC.
|
|
CacheFileCRC( pPathIDToCopyFrom, pRelativeFilename );
|
|
if ( !( pSourceInfo->m_Flags & k_eFileFlagsGotCRCOnce ) )
|
|
{
|
|
// Still didn't get it. Ok.. well get a CRC for the target one anyway.
|
|
CacheFileCRC( pPathID, pRelativeFilename );
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Setup a CFileInfo for the target..
|
|
CPathIDFileList *pPath = GetPathIDFileList( pPathID );
|
|
CFileInfo *pDestInfo = pPath->FindFileInfo( pRelativeFilename );
|
|
if ( !pDestInfo )
|
|
pDestInfo = pPath->AddFileInfo( pRelativeFilename );
|
|
|
|
pDestInfo->m_CRC = pSourceInfo->m_CRC;
|
|
pDestInfo->m_Flags = pSourceInfo->m_Flags;
|
|
}
|
|
|
|
|
|
EFileCRCStatus CFileTracker::CheckCachedFileCRC( const char *pPathID, const char *pRelativeFilename, CRC32_t *pCRC )
|
|
{
|
|
Assert( ThreadInMainThread() );
|
|
|
|
// Get the file's info. Load the file if necessary.
|
|
CFileInfo *pInfo = GetFileInfo( pRelativeFilename, pPathID );
|
|
if ( pInfo && (pInfo->m_Flags & k_eFileFlagsGotCRCOnce) )
|
|
{
|
|
*pCRC = pInfo->m_CRC;
|
|
return k_eFileCRCStatus_GotCRC;
|
|
}
|
|
else
|
|
{
|
|
return k_eFileCRCStatus_CantOpenFile;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
void CFileTracker::CalculateMissingCRC( const char *pFilename, const char *pPathID )
|
|
{
|
|
// Force it to make a CRC of disk files.
|
|
FileHandle_t fh = m_pFileSystem->FindFileInSearchPaths( pFilename, "rb", pPathID, FSOPEN_FORCE_TRACK_CRC, NULL, true );
|
|
if ( !fh )
|
|
return;
|
|
|
|
CFileInfo *pInfo = GetFileInfo( pFilename, pPathID );
|
|
if ( pInfo )
|
|
{
|
|
// Now we're about to modify the file itself.. lock the mutex.
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
// The FindFileInSearchPaths call might have done the CRC for us.
|
|
if ( !( pInfo->m_Flags & k_eFileFlagsHasCRC ) )
|
|
{
|
|
pInfo->m_CRC = CalculateCRCForFile( fh );
|
|
pInfo->m_Flags |= k_eFileFlagsHasCRC | k_eFileFlagsGotCRCOnce;
|
|
if ( pInfo->m_iNeedsVerificationListIndex == -1 )
|
|
{
|
|
pInfo->m_iNeedsVerificationListIndex = m_NeedsVerificationList.AddToTail( pInfo );
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Assert( false );
|
|
}
|
|
|
|
m_pFileSystem->Close( fh );
|
|
}
|
|
|
|
|
|
void CFileTracker::MarkAllCRCsUnverified()
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
// First clear the 'needs verification' list.
|
|
MarkAllCRCsVerified();
|
|
|
|
Assert( m_NeedsVerificationList.Count() == 0 );
|
|
for ( int i=m_PathIDs.First(); i != m_PathIDs.InvalidIndex(); i=m_PathIDs.Next( i ) )
|
|
{
|
|
CPathIDFileList *pPath = m_PathIDs[i];
|
|
FOR_EACH_LL( pPath->m_Files, j )
|
|
{
|
|
CFileInfo *pInfo = pPath->m_Files[j];
|
|
|
|
if ( !(pInfo->m_Flags & k_eFileFlagsLoadedFromSteam) && ( pInfo->m_Flags & k_eFileFlagsHasCRC ) )
|
|
{
|
|
pInfo->m_iNeedsVerificationListIndex = m_NeedsVerificationList.AddToTail( pInfo );
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
void CFileTracker::MarkAllCRCsVerified( bool bLockMutex )
|
|
{
|
|
if ( bLockMutex )
|
|
m_Mutex.Lock();
|
|
|
|
FOR_EACH_LL( m_NeedsVerificationList, i )
|
|
{
|
|
m_NeedsVerificationList[i]->m_iNeedsVerificationListIndex = -1;
|
|
}
|
|
|
|
m_NeedsVerificationList.Purge();
|
|
|
|
if ( bLockMutex )
|
|
m_Mutex.Unlock();
|
|
}
|
|
|
|
|
|
int CFileTracker::GetUnverifiedCRCFiles( CUnverifiedCRCFile *pFiles, int nMaxFiles )
|
|
{
|
|
Assert( nMaxFiles > 0 );
|
|
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
int iOutFile = 0;
|
|
int iNext = 0;
|
|
for ( int i=m_NeedsVerificationList.Head(); i != m_NeedsVerificationList.InvalidIndex(); i=iNext )
|
|
{
|
|
iNext = m_NeedsVerificationList.Next( i );
|
|
|
|
CFileInfo *pInfo = m_NeedsVerificationList[i];
|
|
|
|
// Remove this entry from the list.
|
|
m_NeedsVerificationList.Remove( i );
|
|
pInfo->m_iNeedsVerificationListIndex = -1;
|
|
|
|
// This can happen if a file that was in this list was loaded from Steam since it got added to the list.
|
|
// In that case, just act like it's not in the list.
|
|
if ( pInfo->m_Flags & k_eFileFlagsLoadedFromSteam )
|
|
continue;
|
|
|
|
Assert( pInfo->m_Flags & k_eFileFlagsHasCRC );
|
|
|
|
// Add this file to their list.
|
|
CUnverifiedCRCFile *pOutFile = &pFiles[iOutFile];
|
|
|
|
V_strncpy( pOutFile->m_Filename, pInfo->m_pFilename, sizeof( pOutFile->m_Filename ) );
|
|
V_strncpy( pOutFile->m_PathID, pInfo->m_pPathIDFileList->m_PathID.String(), sizeof( pOutFile->m_PathID ) );
|
|
pOutFile->m_CRC = pInfo->m_CRC;
|
|
|
|
|
|
++iOutFile;
|
|
if ( iOutFile >= nMaxFiles )
|
|
return iOutFile;
|
|
}
|
|
|
|
return iOutFile;
|
|
}
|
|
|
|
|
|
void CFileTracker::Clear()
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
m_PathIDs.PurgeAndDeleteElements();
|
|
}
|
|
|
|
CPathIDFileList* CFileTracker::GetPathIDFileList( const char *pPathID, bool bAutoAdd )
|
|
{
|
|
AUTO_LOCK( m_Mutex );
|
|
|
|
if ( !pPathID )
|
|
pPathID = "";
|
|
|
|
int i = m_PathIDs.Find( pPathID );
|
|
if ( i == m_PathIDs.InvalidIndex() )
|
|
{
|
|
if ( bAutoAdd )
|
|
{
|
|
CPathIDFileList *pPath = new CPathIDFileList;
|
|
pPath->m_PathID = pPathID;
|
|
m_PathIDs.Insert( pPathID, pPath );
|
|
return pPath;
|
|
}
|
|
else
|
|
{
|
|
return NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
return m_PathIDs[i];
|
|
}
|
|
}
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// CFileInfo implementation.
|
|
//-----------------------------------------------------------------------------
|
|
|
|
CFileInfo::CFileInfo( const char *pFilename )
|
|
{
|
|
int len = V_strlen( pFilename ) + 1;
|
|
m_pFilename = new char[ len ];
|
|
Q_strncpy( m_pFilename, pFilename, len );
|
|
m_iNeedsVerificationListIndex = -1;
|
|
}
|
|
|
|
CFileInfo::~CFileInfo()
|
|
{
|
|
delete [] m_pFilename;
|
|
}
|
|
|
|
|
|
//-----------------------------------------------------------------------------
|
|
// CPathIDFileList implementation..
|
|
//-----------------------------------------------------------------------------
|
|
|
|
CPathIDFileList::~CPathIDFileList()
|
|
{
|
|
m_Files.PurgeAndDeleteElements();
|
|
}
|
|
|
|
CFileInfo* CPathIDFileList::FindFileInfo( const char *pFilename )
|
|
{
|
|
Assert( !V_IsAbsolutePath( pFilename ) );
|
|
|
|
FOR_EACH_LL( m_Files, i )
|
|
{
|
|
CFileInfo *pFileInfo = m_Files[i];
|
|
|
|
if ( CompareFilenames( pFilename, pFileInfo->GetFilename() ) )
|
|
return m_Files[i];
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
CFileInfo* CPathIDFileList::AddFileInfo( const char *pFilename )
|
|
{
|
|
Assert( !V_IsAbsolutePath( pFilename ) );
|
|
|
|
CFileInfo *pFileInfo = new CFileInfo( pFilename );
|
|
pFileInfo->m_pPathIDFileList = this;
|
|
m_Files.AddToTail( pFileInfo );
|
|
return pFileInfo;
|
|
}
|
|
|
|
|
|
|
|
|