tf2/tf2_src/filesystem/packfile.cpp

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose:
//
// $NoKeywords: $
//===========================================================================//

#include "packfile.h"
#include "zip_utils.h"
#include "tier0/basetypes.h"
#include "tier1/convar.h"
#include "tier1/lzmaDecoder.h"
#include "tier1/utlbuffer.h"
#include "tier1/generichash.h"

ConVar fs_monitor_read_from_pack( "fs_monitor_read_from_pack", "0", 0, "0:Off, 1:Any, 2:Sync only" );

// How many bytes we should decode at a time when doing pseudo-reads to seek forward in a compressed file handle,
// (affects maximum stack allocation by a forward seek)
#define COMPRESSED_SEEK_READ_CHUNK 1024

CPackFile::CPackFile()
{
	m_FileLength = 0;
	m_hPackFileHandleFS = NULL;
	m_fs = NULL;
	m_nBaseOffset = 0;
	m_bIsMapPath = false;
	m_lPackFileTime = 0L;
	m_refCount = 0;
	m_nOpenFiles = 0;
	m_PackFileID = 0;
}

CPackFile::~CPackFile()
{
	if ( m_nOpenFiles )
	{
		Error( "Closing pack file with %d open files!\n", m_nOpenFiles );
	}

	if ( m_hPackFileHandleFS )
	{
		m_fs->FS_fclose( m_hPackFileHandleFS );
		m_hPackFileHandleFS = NULL;
	}

	m_fs->m_ZipFiles.FindAndRemove( this );
}

int CPackFile::GetSectorSize()
{
	if ( m_hPackFileHandleFS )
	{
		return m_fs->FS_GetSectorSize( m_hPackFileHandleFS );
	}
#if defined( SUPPORT_PACKED_STORE )
	else if ( m_hPackFileHandleVPK )
	{
		return 2048;
	}
#endif
	else
	{
		return -1;
	}
}

// Read a bit of the file from the pack file:
int CZipPackFileHandle::Read( void* pBuffer, int nDestSize, int nBytes )
{
	// Clamp nBytes to not go past the end of the file (async is still possible due to nDestSize)
	if ( nBytes + m_nFilePointer > m_nLength )
	{
		nBytes = m_nLength - m_nFilePointer;
	}

	// Seek to the given file pointer and read
	int nBytesRead = m_pOwner->ReadFromPack( m_nIndex, pBuffer, nDestSize, nBytes, m_nBase + m_nFilePointer );

	m_nFilePointer += nBytesRead;

	return nBytesRead;
}

// Seek around inside the pack:
int CZipPackFileHandle::Seek( int nOffset, int nWhence )
{
	if ( nWhence == SEEK_SET )
	{
		m_nFilePointer = nOffset;
	}
	else if ( nWhence == SEEK_CUR )
	{
		m_nFilePointer += nOffset;
	}
	else if ( nWhence == SEEK_END )
	{
		m_nFilePointer = m_nLength + nOffset;
	}

	// Clamp the file pointer to the actual bounds of the file:
	if ( m_nFilePointer > m_nLength )
	{
		m_nFilePointer = m_nLength;
	}

	return m_nFilePointer;
}

//-----------------------------------------------------------------------------
// Open a file inside of a pack file.
//-----------------------------------------------------------------------------
CFileHandle *CZipPackFile::OpenFile( const char *pFileName, const char *pOptions )
{
	int nIndex, nOriginalSize, nCompressedSize;
	int64 nPosition;
	unsigned short nCompressionMethod;

	// find the file's location in the pack
	if ( GetFileInfo( pFileName, nIndex, nPosition, nOriginalSize, nCompressedSize, nCompressionMethod ) )
	{
		m_mutex.Lock();
#if defined( SUPPORT_PACKED_STORE )
		if ( m_nOpenFiles == 0 && m_hPackFileHandleFS == NULL && !m_hPackFileHandleVPK )
#else
		if ( m_nOpenFiles == 0 && m_hPackFileHandleFS == NULL )
#endif
		{
			// Try to open it as a regular file first
			m_hPackFileHandleFS = m_fs->Trace_FOpen( m_ZipName, "rb", 0, NULL );

			// !NOTE! Pack files inside of VPK not supported
		}
		m_nOpenFiles++;
		m_mutex.Unlock();
		CPackFileHandle* ph = NULL;
		if ( nCompressionMethod == ZIP_COMPRESSION_LZMA )
		{
			ph = new CLZMAZipPackFileHandle( this, nPosition, nOriginalSize, nCompressedSize, nIndex );
		}
		else
		{
			AssertMsg( nCompressionMethod == ZIP_COMPRESSION_NONE, "Unsupported compression type in zip pack file" );
			ph = new CZipPackFileHandle( this, nPosition, nOriginalSize, nIndex );
		}
		CFileHandle *fh = new CFileHandle( m_fs );
		fh->m_pPackFileHandle = ph;
		fh->m_nLength = nOriginalSize;

		// The default mode for fopen is text, so require 'b' for binary
		if ( strstr( pOptions, "b" ) == NULL )
		{
			fh->m_type = FT_PACK_TEXT;
		}
		else
		{
			fh->m_type = FT_PACK_BINARY;
		}

#if !defined( _RETAIL )
		fh->SetName( pFileName );
#endif
		return fh;
	}

	return NULL;
}

//-----------------------------------------------------------------------------
//	Get a directory entry from a pack's preload section
//-----------------------------------------------------------------------------
ZIP_PreloadDirectoryEntry* CZipPackFile::GetPreloadEntry( int nEntryIndex )
{
	if ( !m_pPreloadHeader )
	{
		return NULL;
	}

	// If this entry doesn't have a corresponding preload entry, fail.
	if ( m_PackFiles[nEntryIndex].m_nPreloadIdx == INVALID_PRELOAD_ENTRY )
	{
		return NULL;
	}

	return &m_pPreloadDirectory[m_PackFiles[nEntryIndex].m_nPreloadIdx];
}

//-----------------------------------------------------------------------------
//	Read a file from the pack
//-----------------------------------------------------------------------------
int CZipPackFile::ReadFromPack( int nEntryIndex, void* pBuffer, int nDestBytes, int nBytes, int64 nOffset )
{
	if ( nEntryIndex >= 0 )
	{
		if ( nBytes <= 0 )
		{
			return 0;
		}

		// X360TBD: This is screwy, it works because m_nBaseOffset is 0 for preload capable zips
		// It comes into play for files out of the embedded bsp zip,
		// this hackery is a pre-bias expecting ReadFromPack() do a symmetric post bias, yuck.

		// Attempt to satisfy request from possible preload section, otherwise fall through
		// A preload entry may be compressed
		ZIP_PreloadDirectoryEntry *pPreloadEntry = GetPreloadEntry( nEntryIndex );
		if ( pPreloadEntry )
		{
			// convert the absolute pack file position to a local file position
			int nLocalOffset = nOffset - m_PackFiles[nEntryIndex].m_nPosition;
			byte *pPreloadData = (byte*)m_pPreloadData + pPreloadEntry->DataOffset;

			if ( CLZMA::IsCompressed( pPreloadData ) )
			{
				unsigned int actualSize = CLZMA::GetActualSize( pPreloadData );
				if ( nLocalOffset + nBytes <= (int)actualSize )
				{
					// satisfy from compressed preload
					if ( fs_monitor_read_from_pack.GetInt() == 1 )
					{
						char szName[MAX_PATH];
						IndexToFilename( nEntryIndex, szName, sizeof( szName ) );
						Msg( "Read From Pack: [Preload] Requested:%d, Compressed:%d, %s\n", nBytes, pPreloadEntry->Length, szName );
					}

					if ( nLocalOffset == 0 && nDestBytes >= (int)actualSize && nBytes == (int)actualSize )
					{
						// uncompress directly into caller's buffer
						CLZMA::Uncompress( (unsigned char *)pPreloadData, (unsigned char *)pBuffer );
						return nBytes;
					}

					// uncompress into temporary memory
					CUtlMemory< byte > tempMemory;
					tempMemory.EnsureCapacity( actualSize );
					CLZMA::Uncompress( pPreloadData, tempMemory.Base() );
					// copy only what caller expects
					V_memcpy( pBuffer, (byte*)tempMemory.Base() + nLocalOffset, nBytes );
					return nBytes;
				}
			}
			else if ( nLocalOffset + nBytes <= (int)pPreloadEntry->Length )
			{
				// satisfy from uncompressed preload
				if ( fs_monitor_read_from_pack.GetInt() == 1 )
				{
					char szName[MAX_PATH];
					IndexToFilename( nEntryIndex, szName, sizeof( szName ) );
					Msg( "Read From Pack: [Preload] Requested:%d, Total:%d, %s\n", nBytes, pPreloadEntry->Length, szName );
				}

				V_memcpy( pBuffer, pPreloadData + nLocalOffset, nBytes );
				return nBytes;
			}
		}
	}

#if defined ( _X360 )
	// fell through as a direct request from within the pack
	// intercept to possible embedded section
	if ( m_pSection )
	{
		// a section is a special update zip that has no files, only preload
		// it has to be in the section
		V_memcpy( pBuffer, (byte*)m_pSection + nOffset, nBytes );
		return nBytes;
	}
#endif

	// Otherwise, do the read from the pack
	m_mutex.Lock();

	if ( fs_monitor_read_from_pack.GetInt() == 1 || ( fs_monitor_read_from_pack.GetInt() == 2 && ThreadInMainThread() ) )
	{
		// spew info about real i/o request
		char szName[MAX_PATH];
		IndexToFilename( nEntryIndex, szName, sizeof( szName ) );
		Msg( "Read From Pack: Sync I/O: Requested:%7d, Offset:0x%16.16llx, %s\n", nBytes, m_nBaseOffset + nOffset, szName );
	}

	int nBytesRead = 0;
	// Seek to the start of the read area and perform the read: TODO: CHANGE THIS INTO A CFileHandle
	if ( m_hPackFileHandleFS )
	{
		m_fs->FS_fseek( m_hPackFileHandleFS, m_nBaseOffset + nOffset, SEEK_SET );
		nBytesRead = m_fs->FS_fread( pBuffer, nDestBytes, nBytes, m_hPackFileHandleFS );
	}
#if defined( SUPPORT_PACKED_STORE )
	else
	{
		// We're a packfile embedded in a VPK
		m_hPackFileHandleVPK.Seek( m_nBaseOffset + nOffset, FILESYSTEM_SEEK_HEAD );
		nBytesRead = m_hPackFileHandleVPK.Read( pBuffer, nBytes );
	}
#endif
	m_mutex.Unlock();

	return nBytesRead;
}

//-----------------------------------------------------------------------------
//	Gets size, position, and index for a file in the pack.
//-----------------------------------------------------------------------------
bool CZipPackFile::GetFileInfo( const char *pFileName, int &nBaseIndex, int64 &nFileOffset, int &nOriginalSize, int &nCompressedSize, unsigned short &nCompressionMethod )
{
	char szCleanName[MAX_FILEPATH];
	Q_strncpy( szCleanName, pFileName, sizeof( szCleanName ) );
#ifdef _WIN32
	Q_strlower( szCleanName );
#endif
	Q_FixSlashes( szCleanName );

	if ( !Q_RemoveDotSlashes( szCleanName, CORRECT_PATH_SEPARATOR, false ) )
	{
		return false;
	}

	CZipPackFile::CPackFileEntry lookup;

	// We may get passed non-canonicalized filenames, so we need to remove the ../ from the path
	char szFixedName[MAX_PATH] = {0};
	V_strcpy_safe( szFixedName, pFileName );
	V_RemoveDotSlashes( szFixedName );

	lookup.m_HashName = HashStringCaselessConventional( szFixedName );

	int idx = m_PackFiles.Find( lookup );
	if ( -1 != idx  )
	{
		nFileOffset = m_PackFiles[idx].m_nPosition;
		nOriginalSize = m_PackFiles[idx].m_nOriginalSize;
		nCompressedSize = m_PackFiles[idx].m_nCompressedSize;
		nBaseIndex = idx;
		nCompressionMethod = m_PackFiles[idx].m_nCompressionMethod;
		return true;
	}

	return false;
}

bool CZipPackFile::IndexToFilename( int nIndex, char *pBuffer, int nBufferSize )
{
	AssertMsg( nIndex >= 0 && nIndex < m_PackFiles.Count(), "Out of bounds vector access in IndexToFilename" );
	if ( nIndex >= 0 )
	{
		m_fs->String( m_PackFiles[nIndex].m_hFileName, pBuffer, nBufferSize );
		return true;
	}

	Q_strncpy( pBuffer, "unknown", nBufferSize );

	return false;
}

//-----------------------------------------------------------------------------
//	Find a file in the pack.
//-----------------------------------------------------------------------------
bool CZipPackFile::ContainsFile( const char *pFileName )
{
	int nIndex, nOriginalSize, nCompressedSize;
	int64 nOffset;
	unsigned short nCompressionMethod;
	bool bFound = GetFileInfo( pFileName, nIndex, nOffset, nOriginalSize, nCompressedSize, nCompressionMethod );
	return bFound;
}

//-----------------------------------------------------------------------------
//	Build a list of matching files and directories given a FindFirst() style wildcard
//-----------------------------------------------------------------------------
void CZipPackFile::GetFileAndDirLists( const char *pRawWildCard, CUtlStringList &outDirnames, CUtlStringList &outFilenames, bool bSortedOutput )
{
	// See also: VPKlib function with same name.

	CUtlDict<int,int> AddedDirectories; // Used to remove duplicate paths

	char szWildCard[MAX_PATH] = { 0 };
	char szWildCardPath[MAX_PATH] = { 0 };
	char szWildCardBase[MAX_PATH] = { 0 };
	char szWildCardExt[MAX_PATH] = { 0 };

	size_t nLenWildcardPath = 0;
	size_t nLenWildcardBase = 0;

	bool bBaseWildcard = true;
	bool bExtWildcard = true;

	//
	// Parse the wildcard string into a base and extension used for string comparisons
	//
	V_strncpy( szWildCard, pRawWildCard, sizeof( szWildCard ) );
	V_FixSlashes( szWildCard, '/' );
	V_RemoveDotSlashes( szWildCard, '/', /* bRemoveDoubleSlashes */ true );

	// Workaround edge case in crappy path code. ExtractFilePath extracts a/b/ from a/b/c/ but FileBase would return the empty string.
	size_t nLenWildCard = V_strlen( szWildCard );
	if ( nLenWildCard && szWildCard[ nLenWildCard - 1 ] == '/' )
	{
		V_strncpy( szWildCardPath, szWildCard, sizeof( szWildCardPath ) );
	}
	else
	{
		V_ExtractFilePath( szWildCard, szWildCardPath, sizeof( szWildCardPath ) );
	}

	V_FileBase( szWildCard, szWildCardBase, sizeof( szWildCardBase ) );
	bool bWildcardHasExt = !!V_strrchr( szWildCard, '.' );
	V_ExtractFileExtension( szWildCard, szWildCardExt, sizeof( szWildCardExt ) );

	// From the pattern, we now have the directory path up to the file pattern, the filename base, and the filename
	// extension.

	// We don't support partial wildcards here (foo*bar.*). This support is massively inconsistent in our codebase and
	// there's no one point where we implement it, so rather than trying to match one of our broken implementations
	// (windows stdio is the only one I could find that was actually right), I'm going with "you shouldn't use this API
	// for that".
	bBaseWildcard = ( V_strcmp( szWildCardBase, "*" ) == 0 );
	bExtWildcard = ( V_strcmp( szWildCardExt, "*" ) == 0 );

	if ( !bWildcardHasExt && bBaseWildcard )
	{
		// For the special case of just '*' (and not, e.g., '*.') match '*.*'
		bExtWildcard = true;
	}

	nLenWildcardPath = V_strlen( szWildCardPath );
	nLenWildcardBase = V_strlen( szWildCardBase );

	// Generate the list of directories and files that match the wildcard
	//

	// For each candidate we attempt to walk up its path and consider the directories it represents as well (the
	// directories in a zip only exist in that files contain them, there are no empty directories)
	FOR_EACH_VEC( m_PackFiles, filesIdx )
	{
		char szCandidateName[MAX_PATH] = { 0 };
		IndexToFilename( filesIdx, szCandidateName, sizeof( szCandidateName ));

		if ( !szCandidateName[0] )
		{
			continue;
		}

		// Check if this file starts with the wildcard selector's path.
		// Note that we only ensure the prefix is the same. There are no specific entries for directories in a zip, they
		// only exist in that files in the zip reference them, so handle subdirectory matches from filenames as well.
		CUtlDict<int,int> ConsideredDirectories; // Will have duplicate directory matches when multiple files reside in them
		if  ( ( nLenWildcardPath && ( 0 == V_strnicmp( szCandidateName, szWildCardPath, nLenWildcardPath ) ) )
		      || ( !nLenWildcardPath && strchr( szCandidateName, '/' ) ) )
		{
			// Check if we matched because of a sub-directory, e.g. a/b/*.* would match /a/b/c/d/foo (in which case we
			// want to add /a/b/c to the matched directories list, ignoring the actual specific file)
			char szCandidateBaseName[MAX_PATH] = { 0 };
			bool bIsDir = false;
			size_t nSubDirLen = 0;
			char *pSubDirSlash = strchr( szCandidateName + nLenWildcardPath, '/' );
			if ( pSubDirSlash )
			{
				// This is a subdirectory match, drop everything after it and continue with it as the filename
				nSubDirLen = (size_t)( (ptrdiff_t)pSubDirSlash - (ptrdiff_t)( szCandidateName + nLenWildcardPath ) );
				V_strncpy( szCandidateBaseName, szCandidateName + nLenWildcardPath, nSubDirLen + 1 );
				bIsDir = true;

				// Early out if we already considered this exact directory from another file
				if ( ConsideredDirectories.Find( szCandidateBaseName ) != ConsideredDirectories.InvalidIndex() )
				{
					continue;
				}

				ConsideredDirectories.Insert( szCandidateBaseName, 0 );
			}
			else
			{
				V_strncpy( szCandidateBaseName, szCandidateName + nLenWildcardPath, sizeof( szCandidateBaseName ) );
			}

			char *pExt = strchr( szCandidateBaseName, '.' );
			if ( pExt )
			{
				// Null out the . and move to point to the extension
				*pExt = '\0';
				pExt++;
			}

			// Determine if this file matches the wildcart (*.*, *.ext, ext.*)
			bool bBaseMatch = false;
			bool bExtMatch = false;

			// If we have a base dir name, and we have a szWildCardBase to match against
			if ( bBaseWildcard )
				bBaseMatch = true;  // The base is the wildCard ("*"), so whatever we have as the base matches
			else
				bBaseMatch = ( 0 == V_stricmp( szCandidateBaseName, szWildCardBase ) );

			// If we have an extension and we have a szWildCardExtension to mach against
			if ( ( bExtWildcard && pExt ) || ( !pExt && !bWildcardHasExt ) )
				bExtMatch = true;
			else
				bExtMatch = bWildcardHasExt && pExt && ( 0 == V_stricmp( pExt, szWildCardExt ) );

			// If both parts match, then add it to the list
			if ( bBaseMatch && bExtMatch )
			{
				if ( bIsDir )
				{
					// Pull up to the subdir we considered out of szCandidateName
					size_t nMatchSize = nLenWildcardPath + nSubDirLen + 1;
					char *pszFullMatch = new char[ nMatchSize ];
					V_strncpy( pszFullMatch, szCandidateName, nMatchSize );
					outDirnames.AddToTail( pszFullMatch );
				}
				else
				{
					size_t nMatchSize = V_strlen( szCandidateName ) + 1;
					char *pszFullMatch = new char[ nMatchSize ];
					V_strncpy( pszFullMatch, szCandidateName, nMatchSize );
					outFilenames.AddToTail( pszFullMatch );
				}
			}
		}
	}

	// Sort the output if requested
	if ( bSortedOutput )
	{
		outDirnames.Sort( &CUtlStringList::SortFunc );
		outFilenames.Sort( &CUtlStringList::SortFunc );
	}
}


//-----------------------------------------------------------------------------
//	Set up the preload section
//-----------------------------------------------------------------------------
void CZipPackFile::SetupPreloadData()
{
	if ( m_pPreloadHeader || !m_nPreloadSectionSize )
	{
		// already loaded or not available
		return;
	}

	MEM_ALLOC_CREDIT_( "xZip" );

	void *pPreload;
#if defined ( _X360 )
	if ( m_pSection )
	{
		pPreload = (byte*)m_pSection + m_nPreloadSectionOffset;
	}
	else
#endif
	{
		pPreload = malloc( m_nPreloadSectionSize );
		if ( !pPreload )
		{
			return;
		}

		if ( IsX360() )
		{
			// 360 XZips are always dvd aligned
			Assert( ( m_nPreloadSectionSize % XBOX_DVD_SECTORSIZE ) == 0 );
			Assert( ( m_nPreloadSectionOffset % XBOX_DVD_SECTORSIZE ) == 0 );
		}

		// preload data is loaded as a single unbuffered i/o operation
		ReadFromPack( -1, pPreload, -1, m_nPreloadSectionSize, m_nPreloadSectionOffset );
	}

	// setup the header
	m_pPreloadHeader = (ZIP_PreloadHeader *)pPreload;

	// setup the preload directory
	m_pPreloadDirectory = (ZIP_PreloadDirectoryEntry *)((byte *)m_pPreloadHeader + sizeof( ZIP_PreloadHeader ) );

	// setup the remap table
	m_pPreloadRemapTable = (unsigned short *)((byte *)m_pPreloadDirectory + m_pPreloadHeader->PreloadDirectoryEntries * sizeof( ZIP_PreloadDirectoryEntry ) );

	// set the preload data base
	m_pPreloadData = (byte *)m_pPreloadRemapTable + m_pPreloadHeader->DirectoryEntries * sizeof( unsigned short );
}

void CZipPackFile::DiscardPreloadData()
{
	if ( !m_pPreloadHeader )
	{
		// already discarded
		return;
	}

#if defined ( _X360 )
	// a section is an alias, the header becomes an alias, not owned memory
	if ( !m_pSection )
	{
		free( m_pPreloadHeader );
	}
#else
	free( m_pPreloadHeader );
#endif
	m_pPreloadHeader = NULL;
}

//-----------------------------------------------------------------------------
//	Parse the zip file to build the file directory and preload section
//-----------------------------------------------------------------------------
bool CZipPackFile::Prepare( int64 fileLen, int64 nFileOfs )
{
	if ( !fileLen || fileLen < sizeof( ZIP_EndOfCentralDirRecord ) )
	{
		// nonsense zip
		return false;
	}

	// Pack files are always little-endian
	m_swap.ActivateByteSwapping( IsX360() );

	m_FileLength = fileLen;
	m_nBaseOffset = nFileOfs;

	ZIP_EndOfCentralDirRecord rec = { 0 };

	// Find and read the central header directory from its expected position at end of the file
	bool bCentralDirRecord = false;
	int64 offset = fileLen - sizeof( ZIP_EndOfCentralDirRecord );

	// 360 can have an incompatible format
	bool bCompatibleFormat = true;
	if ( IsX360() )
	{
		// 360 has dependable exact zips, backup to handle possible xzip format
		if ( offset - XZIP_COMMENT_LENGTH >= 0 )
		{
			offset -= XZIP_COMMENT_LENGTH;
		}

		// single i/o operation, scanning forward
		char *pTemp = (char *)_alloca( fileLen - offset );
		ReadFromPack( -1, pTemp, -1, fileLen - offset, offset );
		while ( offset <= (int64)(fileLen - sizeof( ZIP_EndOfCentralDirRecord )) )
		{
			memcpy( &rec, pTemp, sizeof( ZIP_EndOfCentralDirRecord ) );
			m_swap.SwapFieldsToTargetEndian( &rec );
			if ( rec.signature == PKID( 5, 6 ) )
			{
				bCentralDirRecord = true;
				if ( rec.commentLength >= 4 )
				{
					char *pComment = pTemp + sizeof( ZIP_EndOfCentralDirRecord );
					if ( !V_strnicmp( pComment, "XZP2", 4 ) )
					{
						bCompatibleFormat = false;
					}
				}
				break;
			}
			offset++;
			pTemp++;
		}
	}
	else
	{
		// scan entire file from expected location for central dir
		for ( ; offset >= 0; offset-- )
		{
			ReadFromPack( -1, (void*)&rec, -1, sizeof( rec ), offset );
			m_swap.SwapFieldsToTargetEndian( &rec );
			if ( rec.signature == PKID( 5, 6 ) )
			{
				bCentralDirRecord = true;
				break;
			}
		}
	}
	Assert( bCentralDirRecord );
	if ( !bCentralDirRecord )
	{
		// no zip directory, bad zip
		return false;
	}

	int numFilesInZip = rec.nCentralDirectoryEntries_Total;
	if ( numFilesInZip <= 0 )
	{
		// empty valid zip
		return true;
	}

	int firstFileIdx = 0;

	MEM_ALLOC_CREDIT();

	// read central directory into memory and parse
	CUtlBuffer zipDirBuff( 0, rec.centralDirectorySize, 0 );
	zipDirBuff.EnsureCapacity( rec.centralDirectorySize );
	zipDirBuff.ActivateByteSwapping( IsX360() );
	ReadFromPack( -1, zipDirBuff.Base(), -1, rec.centralDirectorySize, rec.startOfCentralDirOffset );
	zipDirBuff.SeekPut( CUtlBuffer::SEEK_HEAD, rec.centralDirectorySize );

	ZIP_FileHeader zipFileHeader;
	char filename[MAX_PATH] = { 0 };

	// Check for a preload section, expected to be the first file in the zip
	zipDirBuff.GetObjects( &zipFileHeader );
	zipDirBuff.Get( filename, Min( (size_t)zipFileHeader.fileNameLength, sizeof(filename) - 1 ) );
	if ( !V_stricmp( filename, PRELOAD_SECTION_NAME ) )
	{
		m_nPreloadSectionSize = zipFileHeader.uncompressedSize;
		m_nPreloadSectionOffset = zipFileHeader.relativeOffsetOfLocalHeader +
						  sizeof( ZIP_LocalFileHeader ) +
						  zipFileHeader.fileNameLength +
						  zipFileHeader.extraFieldLength;
		SetupPreloadData();

		// Set up to extract the remaining files
		int nextOffset = bCompatibleFormat ? zipFileHeader.extraFieldLength + zipFileHeader.fileCommentLength : 0;
		zipDirBuff.SeekGet( CUtlBuffer::SEEK_CURRENT, nextOffset );
		firstFileIdx = 1;
	}
	else
	{
		if ( IsX360() )
		{
			// all 360 zip files are expected to have preload sections
			// only during development, maps are allowed to lack them, due to auto-conversion
			if ( !m_bIsMapPath || g_pFullFileSystem->GetDVDMode() == DVDMODE_STRICT )
			{
				Warning( "ZipFile '%s' missing preload section\n", m_ZipName.String() );
			}
		}

		// No preload section, reset buffer pointer
		zipDirBuff.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );
	}

	// Parse out central directory and determine absolute file positions of data.
	// Supports uncompressed zip files, with or without preload sections
	bool bSuccess = true;
	char tmpString[MAX_PATH] = { 0 };

	m_PackFiles.EnsureCapacity( numFilesInZip );

	for ( int i = firstFileIdx; i < numFilesInZip; ++i )
	{
		CZipPackFile::CPackFileEntry lookup;
		zipDirBuff.GetObjects( &zipFileHeader );

		if ( zipFileHeader.signature != PKID( 1, 2 ) )
		{
			Warning( "Invalid pack file signature\n" );
			bSuccess = false;
			break;
		}

		if ( zipFileHeader.compressionMethod != ZIP_COMPRESSION_NONE && zipFileHeader.compressionMethod != ZIP_COMPRESSION_LZMA )
		{
			Warning( "Pack file uses unsupported compression method: %hi\n", zipFileHeader.compressionMethod );
			bSuccess = false;
			break;
		}

		Assert( zipFileHeader.fileNameLength < sizeof( tmpString ) );
		unsigned int fileNameLen = Min( (size_t)zipFileHeader.fileNameLength, sizeof( tmpString ) - 1 );
		zipDirBuff.Get( (void *)tmpString, fileNameLen );
		tmpString[fileNameLen] = '\0';
		Q_FixSlashes( tmpString );

		lookup.m_hFileName = m_fs->FindOrAddFileName( tmpString );
		lookup.m_HashName = HashStringCaselessConventional( tmpString );
		lookup.m_nOriginalSize = zipFileHeader.uncompressedSize;
		lookup.m_nCompressedSize = zipFileHeader.compressedSize;
		lookup.m_nPosition = zipFileHeader.relativeOffsetOfLocalHeader +
								sizeof( ZIP_LocalFileHeader ) +
								zipFileHeader.fileNameLength +
								zipFileHeader.extraFieldLength;
		lookup.m_nCompressionMethod = zipFileHeader.compressionMethod;

		// track the index to this file's possible preload directory entry
		if ( m_pPreloadRemapTable )
		{
			lookup.m_nPreloadIdx = m_pPreloadRemapTable[i];
		}
		else
		{
			lookup.m_nPreloadIdx = INVALID_PRELOAD_ENTRY;
		}
		m_PackFiles.InsertNoSort( lookup );

		int nextOffset = bCompatibleFormat ? zipFileHeader.extraFieldLength + zipFileHeader.fileCommentLength : 0;
		zipDirBuff.SeekGet( CUtlBuffer::SEEK_CURRENT, nextOffset );
	}

	m_PackFiles.RedoSort();

	return bSuccess;
}


//-----------------------------------------------------------------------------
//
//-----------------------------------------------------------------------------
CZipPackFile::CZipPackFile( CBaseFileSystem* fs, void *pSection )
 : m_PackFiles()
{
	m_fs = fs;
	m_pPreloadDirectory = NULL;
	m_pPreloadData = NULL;
	m_pPreloadHeader = NULL;
	m_pPreloadRemapTable = NULL;
	m_nPreloadSectionOffset = 0;
	m_nPreloadSectionSize = 0;

#if defined( _X360 )
	m_pSection = pSection;
#endif
}

CZipPackFile::~CZipPackFile()
{
	DiscardPreloadData();
}

//-----------------------------------------------------------------------------
// Purpose:
// Input  : src1 -
//			src2 -
// Output : Returns true on success, false on failure.
//-----------------------------------------------------------------------------
bool CZipPackFile::CPackFileLessFunc::Less( CZipPackFile::CPackFileEntry const& src1, CZipPackFile::CPackFileEntry const& src2, void *pCtx )
{
	return ( src1.m_HashName < src2.m_HashName );
}

//-----------------------------------------------------------------------------
// Purpose: Zip Pack file handle implementation
//-----------------------------------------------------------------------------
CZipPackFileHandle::CZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nLength, unsigned int nIndex, unsigned int nFilePointer )
{
	m_pOwner = pOwner;
	m_nBase = nBase;
	m_nLength = nLength;
	m_nIndex = nIndex;
	m_nFilePointer = nFilePointer;
	pOwner->AddRef();
}

CZipPackFileHandle::~CZipPackFileHandle()
{
	m_pOwner->m_mutex.Lock();
	--m_pOwner->m_nOpenFiles;
	// XXX(johns) this doesn't go here, the hell
	if ( m_pOwner->m_nOpenFiles == 0 && m_pOwner->m_bIsMapPath )
	{
		if ( m_pOwner->m_hPackFileHandleFS )
		{
			m_pOwner->FileSystem()->Trace_FClose( m_pOwner->m_hPackFileHandleFS );
			m_pOwner->m_hPackFileHandleFS = NULL;
		}
	}
	m_pOwner->Release();
	m_pOwner->m_mutex.Unlock();
}

void CZipPackFileHandle::SetBufferSize( int nBytes )
{
	if ( m_pOwner->m_hPackFileHandleFS )
	{
		m_pOwner->FileSystem()->FS_setbufsize( m_pOwner->m_hPackFileHandleFS, nBytes );
	}
}

int CZipPackFileHandle::GetSectorSize()
{
	return m_pOwner->GetSectorSize();
}

int64 CZipPackFileHandle::AbsoluteBaseOffset()
{
	return m_pOwner->GetPackFileBaseOffset() + m_nBase;
}

#if defined( _DEBUG ) && !defined( OSX )
#include <atomic>
static std::atomic<int> sLZMAPackFileHandles( 0 );
#endif // defined( _DEBUG ) && !defined( OSX )

CLZMAZipPackFileHandle::CLZMAZipPackFileHandle( CZipPackFile* pOwner, int64 nBase, unsigned int nOriginalSize, unsigned int nCompressedSize,
                                                unsigned int nIndex, unsigned int nFilePointer )
	: CZipPackFileHandle( pOwner, nBase, nCompressedSize, nIndex, nFilePointer ),
	  m_BackSeekBuffer( 0, PACKFILE_COMPRESSED_FILEHANDLE_SEEK_BUFFER ),
	  m_ReadBuffer( 0, PACKFILE_COMPRESSED_FILEHANDLE_READ_BUFFER ),
	  m_pLZMAStream( NULL ), m_nSeekPosition( 0 ), m_nOriginalSize( nOriginalSize )
{
	Reset();
#if defined( _DEBUG ) && !defined( OSX )
	if ( ++sLZMAPackFileHandles == PACKFILE_COMPRESSED_FILE_HANDLES_WARNING )
	{
		// By my count a live filehandle is currently around 270k, mostly due to the LZMA dictionary (256k) with the
		// rest being the read/seek buffers.
		Warning( "More than %u compressed file handles in use. "
		         "These carry large buffers around, and can cause high memory usage\n",
		         PACKFILE_COMPRESSED_FILE_HANDLES_WARNING );
	}
#endif // defined( _DEBUG ) && !defined( OSX )
}

CLZMAZipPackFileHandle::~CLZMAZipPackFileHandle()
{
	delete m_pLZMAStream;
	m_pLZMAStream = NULL;
#if defined( _DEBUG ) && !defined( OSX )
	sLZMAPackFileHandles--;
	Assert( sLZMAPackFileHandles >= 0 );
#endif // defined( _DEBUG ) && !defined( OSX )
}

int CLZMAZipPackFileHandle::Read( void* pBuffer, int nDestSize, int nBytes )
{
	int nMaxRead = Min( Min( nDestSize, nBytes ), Size() - Tell() );
	int nBytesRead = 0;

	// If we have seeked backwards into our buffer, read from there first
	int nBackSeek = m_BackSeekBuffer.TellPut() - m_BackSeekBuffer.TellGet();
	Assert( nBackSeek >= 0 );
	if ( nBackSeek > 0 )
	{
		int nBackSeekRead = Min( nBackSeek, nMaxRead );
		m_BackSeekBuffer.Get( pBuffer, nBackSeekRead );
		nBytesRead += nBackSeekRead;
	}

	// Done if nothing to read
	if ( nMaxRead - nBytesRead <= 0 )
	{
		m_nSeekPosition += nBytesRead;
		return nBytesRead;
	}

	// Read bytes not fulfilled by backbuffer
	Assert( m_BackSeekBuffer.TellPut() == m_BackSeekBuffer.TellGet() );
	while ( nBytesRead < nMaxRead )
	{
		// refill read buffer if empty
		int nRemainingReadBuffer = FillReadBuffer();

		// Consume from read buffer
		unsigned int nCompressedBytesRead = 0;
		unsigned int nOutputBytesWritten = 0;
		bool bSuccess = m_pLZMAStream->Read( (unsigned char *)m_ReadBuffer.PeekGet(), nRemainingReadBuffer,
		                                     (unsigned char *)pBuffer + nBytesRead, nMaxRead - nBytesRead,
		                                     nCompressedBytesRead, nOutputBytesWritten );

		if ( bSuccess )
		{
			// fixup get position
			m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, nCompressedBytesRead );

			nBytesRead += nOutputBytesWritten;

			AssertMsg( nCompressedBytesRead == (unsigned int)nRemainingReadBuffer || nBytesRead == nMaxRead,
			           "Should have consumed the readbuffer or reached nMaxRead" );

			if ( nCompressedBytesRead == 0 && nOutputBytesWritten == 0 )
			{
				AssertMsg( nCompressedBytesRead > 0 || nOutputBytesWritten > 0,
				           "Stuck progress in read loop, aborting. Stream may be defunct." );
				break;
			}
		}
		else
		{
			Warning( "Pack file: reading from LZMA stream failed\n" );
			break;
		}
	}

	// Finally, store last bytes output to the backseek buffer

	// If we read less than BackSeekBuffer.Size() bytes, shift the end of the old backseek buffer up
	int nOldBackSeek = m_BackSeekBuffer.TellPut();
	int nReuseBackSeek = Max( Min( m_BackSeekBuffer.Size() - nBytesRead, nOldBackSeek ), 0 );
	if ( nReuseBackSeek )
	{
		// Shift the reused chunk to the front
		V_memmove( m_BackSeekBuffer.Base(),
		           (unsigned char *)m_BackSeekBuffer.Base() + m_BackSeekBuffer.TellPut() - nReuseBackSeek,
		           nReuseBackSeek );
	}

	// Update get/put position
	m_BackSeekBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, nReuseBackSeek );
	m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, nReuseBackSeek );

	// Fill in remainder from what we just read
	int nReadIntoBackSeek = Min( m_BackSeekBuffer.Size() - nReuseBackSeek, nBytesRead );
	m_BackSeekBuffer.Put( (unsigned char *)pBuffer + nBytesRead - nReadIntoBackSeek, nReadIntoBackSeek );
	m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, nReadIntoBackSeek );

	m_nSeekPosition += nBytesRead;
	return nBytesRead;
}

int CLZMAZipPackFileHandle::Seek( int nOffset, int nWhence )
{
	int nNewPosition = m_nSeekPosition;

	if ( nWhence == SEEK_CUR )
	{
		nNewPosition = m_nSeekPosition + nOffset;
	}
	else if ( nWhence == SEEK_END )
	{
		nNewPosition = Size() + nOffset;
	}
	else if ( nWhence == SEEK_SET )
	{
		nNewPosition = nOffset;
	}
	else
	{
		AssertMsg( false, "Unknown seek type" );
	}

	nNewPosition = Min( Size(), nNewPosition );
	nNewPosition = Max( 0, nNewPosition );

	if ( nNewPosition == m_nSeekPosition )
	{
		return nNewPosition;
	}

	// Backwards seek
	if ( nNewPosition < m_nSeekPosition )
	{
		int nBackSeekAvailable = m_BackSeekBuffer.TellGet();
		int nDesiredBackSeek = m_nSeekPosition - nNewPosition;

		if ( nBackSeekAvailable >= nDesiredBackSeek )
		{
			// Move get backwards into backseek buffer to account for seek
			m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_CURRENT, -nDesiredBackSeek );
			m_nSeekPosition = nNewPosition;
		}
		else
		{
			// Seeking backwards beyond our backseek buffer. Have to restart stream. This kills the performance.
			Warning( "LZMA file handle: seeking backwards beyond backseek buffer size ( %u ), "
			         "replaying read & decompression of %u bytes. Should avoid large back seeks in compressed files or "
			         "increase backseek buffer sizing.",
			         m_BackSeekBuffer.Size(), nNewPosition );

			// Reset to beginning of underlying stream
			Reset();

			// Fall through to performing a forward seek
		}
	}

	// Forward seek
	if ( nNewPosition > m_nSeekPosition )
	{
		// Can't actually seek forward without making decode progress. Issue fake reads until we've reached our target.
		unsigned char dummyBuffer[COMPRESSED_SEEK_READ_CHUNK];
		while ( nNewPosition > m_nSeekPosition )
		{
			int nReadSize = Min( nNewPosition - m_nSeekPosition, COMPRESSED_SEEK_READ_CHUNK );
			unsigned int nBytesRead = Read( &dummyBuffer, sizeof(dummyBuffer), nReadSize );
			m_nSeekPosition += nBytesRead;
			if ( !nBytesRead )
			{
				Warning( "LZMA file handle: failed reading forward to desired seek position\n" );
				break;
			}
		}
	}

	return m_nSeekPosition;
}

int CLZMAZipPackFileHandle::Tell()
{
	return m_nSeekPosition;
}

int CLZMAZipPackFileHandle::Size()
{
	return m_nOriginalSize;
}

int CLZMAZipPackFileHandle::FillReadBuffer()
{
	int nRemainingReadBuffer = m_ReadBuffer.TellPut() - m_ReadBuffer.TellGet();
	int nRemainingCompressedBytes = CZipPackFileHandle::Size() - CZipPackFileHandle::Tell();

	if ( nRemainingReadBuffer > 0 || nRemainingCompressedBytes <= 0 )
	{
		// No action if read buffer isn't empty
		return nRemainingReadBuffer;
	}

	// Reset empty read buffer
	m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 );
	m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );
	int nRefillSize = Min( nRemainingCompressedBytes, m_ReadBuffer.Size() );
	int nRefillResult = CZipPackFileHandle::Read( m_ReadBuffer.PeekPut(), m_ReadBuffer.Size(), nRefillSize );
	AssertMsg( nRefillSize == nRefillResult, "Don't expect to fail to read here" );

	// Fixup put pointer after writing into buffer's memory
	m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_CURRENT, nRefillResult );

	return nRefillResult;
}

void CLZMAZipPackFileHandle::Reset()
{
	// Seek underlying stream back to start
	CZipPackFileHandle::Seek( SEEK_SET, 0 );

	delete m_pLZMAStream;
	m_pLZMAStream = new CLZMAStream();
	m_pLZMAStream->InitZIPHeader( CZipPackFileHandle::Size(), m_nOriginalSize );
	m_nSeekPosition = 0;
	m_BackSeekBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );
	m_BackSeekBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 );
	m_ReadBuffer.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );
	m_ReadBuffer.SeekPut( CUtlBuffer::SEEK_HEAD, 0 );
}