tf2/tf2_src/public/xzp.cpp

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: 
//
//=============================================================================

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <malloc.h>
#ifdef _WIN32
#include <process.h>
#include <io.h>
#endif
#include <stddef.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "tier1/utlbuffer.h"
#include "tier1/strtools.h"
#include "tier2/riff.h"

#if defined( _WIN32 ) && !defined( _X360 )
#include <windows.h>
#endif

#ifdef MAKE_GAMEDATA_TOOL
	#include "../public/materialsystem/shader_vcs_version.h"
	#include "../public/materialsystem/imaterial.h"
	#include "../public/materialsystem/hardwareverts.h"
	#include "../public/vtf/vtf.h"
#else
	#include "materialsystem/shader_vcs_version.h"
	#include "materialsystem/imaterial.h"
	#include "materialsystem/hardwareverts.h"
#endif

#include "xwvfile.h"
#include "xzp.h"

CByteswap g_xzpSwap;
extern IFileReadBinary *g_pSndIO;

// memdbgon must be the last include file in a .cpp file!!!
#include "tier0/memdbgon.h"

//-----------------------------------------------------------------------------
// Datadesc blocks for byteswapping:
//-----------------------------------------------------------------------------
BEGIN_BYTESWAP_DATADESC( xZipHeader_t )
	DEFINE_FIELD( Magic, FIELD_INTEGER ),
	DEFINE_FIELD( Version, FIELD_INTEGER ),
	DEFINE_FIELD( PreloadDirectoryEntries, FIELD_INTEGER ),
	DEFINE_FIELD( DirectoryEntries, FIELD_INTEGER ),
	DEFINE_FIELD( PreloadBytes, FIELD_INTEGER ),
	DEFINE_FIELD( HeaderLength, FIELD_INTEGER ),
	DEFINE_FIELD( FilenameEntries, FIELD_INTEGER ),
	DEFINE_FIELD( FilenameStringsOffset, FIELD_INTEGER ),
	DEFINE_FIELD( FilenameStringsLength, FIELD_INTEGER ),
END_BYTESWAP_DATADESC()

BEGIN_BYTESWAP_DATADESC( xZipDirectoryEntry_t )
	DEFINE_FIELD( FilenameCRC, FIELD_INTEGER ),
	DEFINE_FIELD( Length, FIELD_INTEGER ),
	DEFINE_FIELD( StoredOffset, FIELD_INTEGER ),
END_BYTESWAP_DATADESC()

BEGIN_BYTESWAP_DATADESC( xZipFilenameEntry_t )
	DEFINE_FIELD( FilenameCRC, FIELD_INTEGER ),
	DEFINE_FIELD( FilenameOffset, FIELD_INTEGER ),
	DEFINE_FIELD( TimeStamp, FIELD_INTEGER ),
END_BYTESWAP_DATADESC()

BEGIN_BYTESWAP_DATADESC( xZipFooter_t )
	DEFINE_FIELD( Size, FIELD_INTEGER ),
	DEFINE_FIELD( Magic, FIELD_INTEGER ),
END_BYTESWAP_DATADESC()

CXZip::CXZip()
{
	// Ensure that the header doesn't contain a valid magic yet.
	m_Header.Magic = 0;	
	m_pPreloadedData = NULL;
	m_nPreloadStart = 0;
	m_pDirectory = NULL;
	m_pPreloadDirectory = NULL;
	m_nRegular2PreloadEntryMapping = NULL;

	m_bByteSwapped = false;

	m_pFilenames = NULL;
	m_hZip = NULL;
	
	m_pRead = NULL;
	m_hUser = 0;
	m_nMonitorLevel = 0;
}

CXZip::CXZip( const char* filename )
{
	// Ensure that the header doesn't contain a valid magic yet.
	m_Header.Magic = 0;	
	m_nPreloadStart = 0;
	m_pPreloadedData = NULL;
	m_pDirectory = NULL;
	m_pPreloadDirectory = NULL;
	m_nRegular2PreloadEntryMapping = NULL;

	m_bByteSwapped = false;

	m_pFilenames = NULL;
	m_hZip = NULL;

	m_pRead = NULL;
	m_hUser = 0;
	m_nMonitorLevel = 0;

	Load( filename );
}

CXZip::CXZip( FILE* handle, int offset, int size )	// file handle and offset of the zip file
{
	m_pRead = NULL;
	m_hUser = 0;
	m_nPreloadStart = 0;
	m_pDirectory = NULL;
	m_pPreloadDirectory = NULL;
	m_nRegular2PreloadEntryMapping = NULL;

	m_bByteSwapped = false;

	m_pFilenames = NULL;
	m_pPreloadedData = NULL;
	m_nMonitorLevel = 0;

	Load( handle, offset, size );
}

CXZip::~CXZip()
{
	Unload();
}

bool CXZip::InstallAlternateIO( int (*read)( void* buffer, int offset, int length, int nDestLength, int hUser), int hUser )
{
	m_pRead = read;
	m_hUser = hUser;
	return true;
}


// Loads an xZip file into memory:
bool CXZip::Load( const char* filename, bool bPreload )
{
	FILE* hZip = fopen( filename, "rb" );
	fseek(hZip,0,SEEK_END);
	int nSize = ftell( hZip );
	return Load( hZip, 0, nSize, bPreload );
}

bool CXZip::Load( FILE* handle, int nOffset, int nSize, bool bPreload )	// Load a pack file into this instance.  Returns true on success.
{
	Unload();

	m_bByteSwapped = false;

	m_hZip = handle;
	m_nOffset = nOffset;
	m_nSize = nSize;

	// Hacky, clean up:
	if( m_hZip && !m_pRead )
	{
		InstallAlternateIO( defaultRead, (int)m_hZip );
	}

	if( m_hZip == NULL && m_pRead == NULL )
	{
		return false;
	}

	// Read the header:
	m_pRead( &m_Header, 0, -1, sizeof(m_Header), m_hUser );

	// Validate the Magic number and at the same time determine if I am reading a regular or swappped xZip file:
	switch( m_Swap.SourceIsNativeEndian<int>( m_Header.Magic, xZipHeader_t::MAGIC ) )
	{
		// Does the magic match exactly?
		case 1:
			m_Swap.ActivateByteSwapping( false );
			m_bByteSwapped = false;
			break;

		// Does the magic match, but is swapped?
		case 0:
			m_bByteSwapped = true;
			m_Swap.ActivateByteSwapping( true );	// We must be reading the opposite endianness.
			m_Swap.SwapFieldsToTargetEndian<xZipHeader_t>( &m_Header );
			break;

		default: 
			assert( 0 );
			// Fail gently in release:
		
		// The magic doesn't match in any respect:
		case -1:
			{
				printf("Invalid xZip file\n");

				if( m_hZip )
				{
					fclose( m_hZip );
					m_hZip = NULL;
				}
				return false;
			}
	}

	// Validate the archive version:
	if( m_Header.Version != xZipHeader_t::VERSION )
	{
		// Backward compatable support for version 1
		Msg("Incorrect xZip version found %u - expected %u\n", m_Header.Version, xZipHeader_t::VERSION );
		if( m_hZip )
		{
			fclose( m_hZip );
			m_hZip = NULL;
		}

		m_Header.Magic = xZipHeader_t::FREE;
		return false;
	}

	// Read the directory:
	{
		MEM_ALLOC_CREDIT();

		m_pDirectory = (xZipDirectoryEntry_t*)malloc( sizeof(xZipDirectoryEntry_t) * m_Header.DirectoryEntries );
		m_pRead( m_pDirectory, m_Header.HeaderLength, -1, sizeof( xZipDirectoryEntry_t ) * m_Header.DirectoryEntries, m_hUser );

		// Swap the directory entries if nessecary
		if( m_bByteSwapped )
		{
			for( unsigned nDirectoryEntry = 0; nDirectoryEntry < m_Header.DirectoryEntries; nDirectoryEntry++ )
			{
				m_Swap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>(  &( m_pDirectory[nDirectoryEntry] ) );
			}
		}


		m_nPreloadStart = m_Header.HeaderLength + ( sizeof( xZipDirectoryEntry_t ) * m_Header.DirectoryEntries );
	}

	// Preload the preload chunk if desired:
	if( bPreload )
	{
		PreloadData();
	}

	return true;
}

void CXZip::Unload()
{
	DiscardPreloadedData();

	// Dump the directory:
	if( m_pDirectory )
	{
		free( m_pDirectory );
		m_pDirectory = NULL;
	}

	if( m_pFilenames )
	{
		free( m_pFilenames );
		m_pFilenames = NULL;
	}

	// Invalidate the header:
	m_Header.Magic = 0;

	if( m_hZip )
	{
		fclose( m_hZip );
		m_hZip = NULL;
	}

}

//-----------------------------------------------------------------------------
// CXZip::PreloadData
//
// Loads the preloaded data if it isn't already.
//-----------------------------------------------------------------------------

void CXZip::PreloadData()
{
	Assert( IsValid() );

	// Ensure it isn't already preloaded
	if( m_pPreloadedData )
		return;

	// If I don't have a preloaded section, ignore the request.
	if( !m_Header.PreloadBytes || !m_Header.PreloadDirectoryEntries )
		return;

	// Allocate and read the data block in:
#ifndef _X360
	MEM_ALLOC_CREDIT_( "xZip" );
	m_pPreloadedData = malloc( m_Header.PreloadBytes );

	// Just drop out if allocation fails;
	if ( !m_pPreloadedData )
		return;

	m_pRead( m_pPreloadedData, m_nPreloadStart, -1, m_Header.PreloadBytes, m_hUser );
#else
	int nAlignedStart = AlignValue( ( m_nPreloadStart - XBOX_HDD_SECTORSIZE ) + 1, XBOX_HDD_SECTORSIZE );
	int nBytesToRead = AlignValue( ( m_nPreloadStart - nAlignedStart ) + m_Header.PreloadBytes, XBOX_HDD_SECTORSIZE );
	int nBytesBuffer = AlignValue( nBytesToRead, XBOX_HDD_SECTORSIZE );
	byte *pReadData = (byte *)malloc( nBytesBuffer );
	
	// Just drop out if allocation fails;
	if ( !pReadData )
		return; 
	
	MEM_ALLOC_CREDIT_( "xZip" );
	m_pRead( pReadData, nAlignedStart, nBytesBuffer,nBytesToRead, m_hUser );
	m_pPreloadedData = pReadData + ( m_nPreloadStart - nAlignedStart );
#endif

	// Set up the preload directory:
	m_pPreloadDirectory = (xZipDirectoryEntry_t*)m_pPreloadedData;

	// Swap the preload directory:
	if ( m_bByteSwapped )
	{
		for ( unsigned nDirectoryEntry = 0; nDirectoryEntry < m_Header.PreloadDirectoryEntries; nDirectoryEntry++ )
		{
			m_Swap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>( &( m_pPreloadDirectory[nDirectoryEntry] ) );
		}
	}

	// Set up the regular 2 preload mapping section:
	m_nRegular2PreloadEntryMapping = (unsigned short*)(((unsigned char*)m_pPreloadDirectory) + ( sizeof(xZipDirectoryEntry_t) * m_Header.PreloadDirectoryEntries ));

	// Swap the regular to preload mapping
	if ( m_bByteSwapped )
	{
		m_Swap.SwapBufferToTargetEndian<short>( (short *)m_nRegular2PreloadEntryMapping,  (short *)m_nRegular2PreloadEntryMapping, m_Header.DirectoryEntries );
	}

}

//-----------------------------------------------------------------------------
// CXZip::DiscardPreloadedData
//
// frees the preloaded data cache if it's present.
//-----------------------------------------------------------------------------

void CXZip::DiscardPreloadedData()
{
	if ( m_pPreloadedData )
	{
#ifndef _X360
		free( m_pPreloadedData );
#else
		int nAlignedStart = AlignValue( ( m_nPreloadStart - XBOX_HDD_SECTORSIZE ) + 1, XBOX_HDD_SECTORSIZE );
		byte *pReadData = (byte *)m_pPreloadedData - ( m_nPreloadStart - nAlignedStart );
		free( pReadData );
#endif
		m_pPreloadedData = NULL;
		m_pPreloadDirectory = NULL;
		m_nRegular2PreloadEntryMapping = NULL;
	}
}

int CXZip::defaultRead( void* buffer, int offset, int destLength, int length, int hUser)
{
	fseek( (FILE*)hUser, offset, SEEK_SET );
	return fread( buffer, 1, length, (FILE*)hUser );
}

char* CXZip::GetEntryFileName( unsigned CRC, char* pDefault )
{
	Assert( IsValid() );

	if( IsRetail() )
	{
		return pDefault;
	}
	else
	{

		// Make sure I have a filename section:
		if( m_Header.FilenameStringsOffset == 0 || m_Header.FilenameEntries == 0 || CRC == 0 )
		{
			return pDefault;
		}

		// If the filename chunk isn't here, load it up:
		if( !m_pFilenames )
		{
			MEM_ALLOC_CREDIT_("xZip");
			m_pFilenames = (xZipFilenameEntry_t*)malloc( m_Header.FilenameStringsLength );
			m_pRead( m_pFilenames, m_Header.FilenameStringsOffset, -1, m_Header.FilenameStringsLength, m_hUser );

			// TODO: Swap!
			for( unsigned int i=0; i< m_Header.FilenameEntries;i++ )
			{
				m_Swap.SwapFieldsToTargetEndian<xZipFilenameEntry_t>(&m_pFilenames[i]);
			}
		}

		// Find this entry in the preload directory
		xZipFilenameEntry_t entry;
		entry.FilenameCRC = CRC;

		xZipFilenameEntry_t* found = (xZipFilenameEntry_t*)bsearch( &entry, m_pFilenames, m_Header.FilenameEntries, sizeof(xZipFilenameEntry_t), xZipFilenameEntry_t::xZipFilenameEntryCompare );

		if( !found ) 
			return pDefault;

		return (((char*)m_pFilenames) + found->FilenameOffset) - m_Header.FilenameStringsOffset; 	
	}
}

// Sanity checks that the zip file is ready and readable:
bool CXZip::IsValid()
{
	if( m_Header.Magic != xZipHeader_t::MAGIC )
		return false;

	if( m_Header.Version > xZipHeader_t::VERSION )
		return false;

	if( !m_pDirectory )
		return false;

	return true;
}

void CXZip::WarningDir()
{
	Assert( IsValid());
	
	for( unsigned i = 0; i< m_Header.DirectoryEntries; i++ )
	{
		Msg( GetEntryFileName( m_pDirectory[i].FilenameCRC ) );
	}
}


int CXZip::ReadIndex( int nEntryIndex, int nFileOffset, int nDestBytes, int nLength, void* pBuffer )
{
	Assert( IsValid() );

	if( nLength <=0 || nEntryIndex < 0 ) 
		return 0;

	// HACK HACK HACK - convert the pack file index to a local file index (ie, assuming the full file index is being passed in)
	nFileOffset -= m_pDirectory[nEntryIndex].StoredOffset;
	// HACK HACK HACK

	// If I've got my preload section loaded, first check there:
	xZipDirectoryEntry_t* pPreloadEntry = GetPreloadEntry(nEntryIndex);

	if( pPreloadEntry )
	{
		Assert( pPreloadEntry->FilenameCRC == m_pDirectory[nEntryIndex].FilenameCRC );

		if( nFileOffset + nLength <= (int)pPreloadEntry->Length )
		{
			if( m_nMonitorLevel >= 2 )
			{
				char* filename = GetEntryFileName( m_pDirectory[nEntryIndex].FilenameCRC, "(!!! unknown !!!)" );

				Msg("PACK(preload) %s: length:%i offset:%i",filename,nLength, nFileOffset);

			}

			memcpy( pBuffer, (char*)m_pPreloadedData + pPreloadEntry->StoredOffset + nFileOffset - m_nPreloadStart, nLength );
			return nLength;
		}
	}

	// Offset int the zip to start the read:
	int ZipOffset = m_pDirectory[nEntryIndex].StoredOffset + nFileOffset;
	int nBytesRead = m_pRead( pBuffer, ZipOffset, nDestBytes, nLength, m_hUser);

	if( m_nMonitorLevel )
	{
		char* filename = GetEntryFileName( m_pDirectory[nEntryIndex].FilenameCRC, "(!!! unknown !!!)" );

		unsigned preload = 0;
		if( m_pPreloadedData && m_nRegular2PreloadEntryMapping[nEntryIndex] != 0xFFFF )
		{
			// Find this entry in the preload directory
			xZipDirectoryEntry_t* entry = &(m_pPreloadDirectory[m_nRegular2PreloadEntryMapping[nEntryIndex]]);
			Assert(entry->FilenameCRC == m_pDirectory[nEntryIndex].FilenameCRC);

			preload = entry->Length;
		}

		Msg("PACK %s: length:%i offset:%i (preload bytes:%i)",filename,nLength, nFileOffset, preload);
	}
	
	return nBytesRead;
}

bool CXZip::GetSimpleFileOffsetLength( const char* FileName, int& nBaseIndex, int &nFileOffset, int &nLength )
{
	Assert( IsValid() );

	xZipDirectoryEntry_t entry;
	entry.FilenameCRC = xZipCRCFilename( FileName );

	xZipDirectoryEntry_t* found = (xZipDirectoryEntry_t*)bsearch( &entry, m_pDirectory, m_Header.DirectoryEntries, sizeof(xZipDirectoryEntry_t), xZipDirectoryEntry_t::xZipDirectoryEntryFindCompare );

	if( found == NULL )
		return false;

	nFileOffset = found[0].StoredOffset;
	nLength = found[0].Length;
	nBaseIndex = (((int)((char*)found - (char*)m_pDirectory))/sizeof(xZipDirectoryEntry_t));

	return true;
}

bool CXZip::ExtractFile( const char* FileName )
{
	return false;
}

// Compares to xZipDirectoryEntries.
//
// Sorts in the following order:  
//		FilenameCRC
//		FileOffset
//		Length
//		StoredOffset
//
// The sort function may look overly complex, but it is actually useful for locating different pieces of 
// the same file in a meaningful order.
//
int __cdecl xZipDirectoryEntry_t::xZipDirectoryEntrySortCompare( const void* left, const void* right )
{
	xZipDirectoryEntry_t *l = (xZipDirectoryEntry_t*)left,
					     *r = (xZipDirectoryEntry_t*)right;

	if( l->FilenameCRC < r->FilenameCRC )
	{
		return -1;
	}

	else if( l->FilenameCRC > r->FilenameCRC )
	{
		return 1;
	}

	// else l->FileOffset == r->FileOffset
	if( l->Length < r->Length )
	{
		return -1;
	}
	else if( l->Length > r->Length )
	{
		return 1;
	}

	// else l->Length == r->Length
	if( l->StoredOffset < r->StoredOffset )
	{
		return -1;
	}
	else if( l->StoredOffset > r->StoredOffset )
	{
		return 1;
	}

	// else everything is identical:
	return 0;

}

// Find an entry with matching CRC only
int __cdecl xZipDirectoryEntry_t::xZipDirectoryEntryFindCompare( const void* left, const void* right )
{
	xZipDirectoryEntry_t *l = (xZipDirectoryEntry_t*)left,
					     *r = (xZipDirectoryEntry_t*)right;

	if( l->FilenameCRC < r->FilenameCRC )
	{
		return -1;
	}

	else if( l->FilenameCRC > r->FilenameCRC )
	{
		return 1;
	}

	return 0;

}

int __cdecl xZipFilenameEntry_t::xZipFilenameEntryCompare( const void* left, const void* right )
{
	xZipFilenameEntry_t *l = (xZipFilenameEntry_t*)left,
					    *r = (xZipFilenameEntry_t*)right;

	if( l->FilenameCRC < r->FilenameCRC )
	{
		return -1;
	}

	else if( l->FilenameCRC > r->FilenameCRC )
	{
		return 1;
	}

	return 0;

}


// CRC's an individual xZip filename:
unsigned xZipCRCFilename( const char* filename )
{
	unsigned hash = 0xAAAAAAAA;	// Alternating 1's and 0's

	for( ; *filename ; filename++ )
	{
		char c = *filename;
	
		// Fix slashes
		if( c == '/' )
			c = '\\';
		else
			c = (char)tolower(c);

		hash = hash * 33 + c;
	}

	return hash;
}

#if defined( MAKE_GAMEDATA_TOOL )

// ------------
xZipHeader_t Header;
xZipDirectoryEntry_t	 *pDirectoryEntries = NULL;
xZipDirectoryEntry_t	 *pPreloadDirectoryEntries = NULL;
xZipFilenameEntry_t		 *pFilenameEntries = NULL;
char					 *pFilenameData = NULL;
unsigned				  nFilenameDataLength = 0;

unsigned			     InputFileBytes = 0;

char* CleanFilename( char* filename )
{
	// Trim leading white space:
	while( isspace(*filename) )
		filename++;

	// Trim trailing white space:
	while( isspace( filename[strlen(filename)-1] ) )
	{
		filename[strlen(filename)-1] = '\0';
	}

	return filename;
}


bool CopyFileBytes( FILE* hDestination, FILE* hSource, unsigned nBytes )
{
	char buffer[16384];

	while( nBytes > 0 )
	{
		int nBytesRead = fread( buffer, 1, nBytes > sizeof(buffer) ? sizeof(buffer) : nBytes,  hSource );
		fwrite(buffer, 1, nBytesRead,  hDestination );
		nBytes -= nBytesRead;
	}

	return true;
}

bool WriteFileBytes( FILE* hDestination, CUtlBuffer &source, unsigned nBytes )
{
	unsigned int nBytesWritten = fwrite(source.Base(), 1, nBytes,  hDestination );
	return (nBytesWritten == nBytes);
}

void PadFileBytes(FILE* hFile, int nPreloadPadding )
{
	if( nPreloadPadding < 0 || nPreloadPadding >= 512)
	{
		puts("Invalid padding");
		return;
	}
	
	char padding[512];
	memset(padding,0,nPreloadPadding);
	fwrite(padding,1,nPreloadPadding,hFile);
}

void AddFilename( const char* filename )
{
	unsigned CRCfilename = xZipCRCFilename( filename );

	// If we already have this filename don't add it again:
	for( int i = 0; i < (int)Header.FilenameEntries; i++ )
	{
		if( pFilenameEntries[i].FilenameCRC == CRCfilename )
		{
			return;
		}
	}

	Header.FilenameEntries++;
	
	// Add the file to the file string table:
	pFilenameEntries = (xZipFilenameEntry_t*)realloc( pFilenameEntries, sizeof(xZipFilenameEntry_t) * Header.FilenameEntries );
	
	int filenameLength = (int)strlen(filename) + 1;
	pFilenameEntries[Header.FilenameEntries-1].FilenameCRC = CRCfilename;
	pFilenameEntries[Header.FilenameEntries-1].FilenameOffset = nFilenameDataLength;

	// Grab the timestamp for the file:
	struct stat buf;
	if( stat( filename, &buf ) != -1 )
	{
		pFilenameEntries[Header.FilenameEntries - 1].TimeStamp = buf.st_mtime;
	}
	else
	{
		pFilenameEntries[Header.FilenameEntries - 1].TimeStamp = 0;
	}

	nFilenameDataLength += filenameLength;
	pFilenameData = (char*)realloc(pFilenameData, nFilenameDataLength);
	memcpy(pFilenameData + nFilenameDataLength - filenameLength, filename, filenameLength);
}

FILE* hTempFilePreload;
FILE* hTempFileData;
FILE* hOutputFile;

bool xZipAddFile( const char* filename, CUtlBuffer &fileBuff, bool bPrecacheEntireFile, bool bProcessPrecacheHeader, bool bProcessPrecacheHeaderOnly )
{
	unsigned int fileSize = fileBuff.TellMaxPut();

	// Track total input bytes for stats reasons
	InputFileBytes += fileSize;

	unsigned customPreloadSize = 0;
	
	if( bPrecacheEntireFile  )
	{
		customPreloadSize = fileSize;
	} 
	else if( bProcessPrecacheHeader )
	{
		customPreloadSize = xZipComputeCustomPreloads( filename );
	}
	else if( bProcessPrecacheHeaderOnly )
	{
		customPreloadSize = xZipComputeCustomPreloads( filename );
		fileSize = min( fileSize, customPreloadSize );
	}

	unsigned CRC = xZipCRCFilename( filename );

	// Does this file have a split header?
	if( customPreloadSize > 0  ) 
	{
		// Initialize the entry header:
		xZipDirectoryEntry_t entry;
		memset( &entry, 0, sizeof( entry ) );
		
		entry.FilenameCRC = CRC;
		entry.Length = customPreloadSize;
		entry.StoredOffset = ftell(hTempFilePreload);

		// Add the directory entry to the preload table:
		Header.PreloadDirectoryEntries++;
		pPreloadDirectoryEntries = (xZipDirectoryEntry_t*)realloc( pPreloadDirectoryEntries, sizeof( xZipDirectoryEntry_t ) * Header.PreloadDirectoryEntries );
		memcpy( pPreloadDirectoryEntries + Header.PreloadDirectoryEntries - 1, &entry, sizeof( entry ) );

		// Concatenate the data in the preload file:
		fileBuff.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );
		WriteFileBytes( hTempFilePreload, fileBuff, entry.Length );
		fileBuff.SeekGet( CUtlBuffer::SEEK_HEAD, 0 );

		
		// Add the filename entry:
		AddFilename( filename );

		// Spew it:
		printf("+Preload: \"%s\": Length:%u\n", filename, entry.Length );
	}

	// Copy the file to the regular data region:
	xZipDirectoryEntry_t entry;
	memset(&entry,0,sizeof(entry));
	entry.FilenameCRC = CRC;
	entry.Length = fileSize;
	entry.StoredOffset = ftell(hTempFileData);		

	// Add the directory entry to the table:
	Header.DirectoryEntries++;
	pDirectoryEntries = (xZipDirectoryEntry_t*)realloc( pDirectoryEntries, sizeof( xZipDirectoryEntry_t ) * Header.DirectoryEntries );
	memcpy( pDirectoryEntries + Header.DirectoryEntries - 1, &entry, sizeof( entry ) );

	WriteFileBytes( hTempFileData, fileBuff, entry.Length );

	// Align the data region to a 512 byte boundry:  (has to be on last entry as well to ensure enough space to perform the final read,
	// and initial alignment is taken careof by assembexzip)
	int nPadding = ( XBOX_HDD_SECTORSIZE - ( ftell( hTempFileData ) % XBOX_HDD_SECTORSIZE) ) % XBOX_HDD_SECTORSIZE;

	PadFileBytes( hTempFileData, nPadding );

	// Add the file to the file string table:
	AddFilename( filename );

	// Print a summary
	printf("+File: \"%s\": Length:%u Padding:%i\n", filename,  entry.Length, nPadding );

	return true;
}

bool xZipAddFile( const char* zipname, bool bPrecacheEntireFile, bool bProcessPrecacheHeader, bool bProcessPrecacheHeaderOnly )
{
	// Clean up the filename:
	char buffer[MAX_PATH];
	strcpy(buffer, zipname);

	// Fix slashes and convert it to lower case:
	char *filename;
	for( filename = buffer; *filename; filename++ )
	{
		if( *filename == '/' )
			*filename = '\\';
		else
		{
			*filename = (char)tolower(*filename);
		}
	}

	// Skip leading white space:
	for( filename = buffer; isspace(*filename); filename++ )
		;

	// Obliterate trailing white space:
	for(;;)
	{
		int len = (int)strlen( filename );
		if( len <= 0 )
		{
			printf("!!!! BAD FILENAME: \"%s\"\n", filename );
			return false;
		}
		
		if( isspace( filename[len-1] ) )
			filename[len-1]='\0';
		else
			break;
	}

	// Ensure we don't already have this file:
	unsigned CRC = xZipCRCFilename( filename );

	for( unsigned i=0; i < Header.DirectoryEntries; i++ )
	{
		if( pDirectoryEntries[i].FilenameCRC == CRC )
		{
			printf("!!!! NOT ADDING DUPLICATE FILENAME: \"%s\"\n", filename );
			return false;
		}
	}

	// Attempt to open the file:
	FILE* hFile = fopen( filename, "rb" );
	if( !hFile )
	{
		printf("!!!! FAILED TO OPEN FILE: \"%s\"\n", filename );
		return false;
	}

	// Get the length of the file:
	fseek(hFile,0,SEEK_END);
	unsigned fileSize = ftell(hFile);
	fseek(hFile,0,SEEK_SET);

	CUtlBuffer fileBuff;
	fileBuff.EnsureCapacity( fileSize );
	fread( fileBuff.Base(), fileSize, 1, hFile );
	fclose( hFile );

	fileBuff.SeekPut( CUtlBuffer::SEEK_HEAD, fileSize );

	return xZipAddFile( zipname, fileBuff, bPrecacheEntireFile, bProcessPrecacheHeader, bProcessPrecacheHeaderOnly );
}

int xZipBegin( const char* fileNameXzip )
{
	// Create and initialize the header:
	memset( &Header, 0, sizeof(Header) );	// Zero out the header:
	Header.Magic = xZipHeader_t::MAGIC;
	Header.Version = xZipHeader_t::VERSION;
	Header.HeaderLength = sizeof(Header);

	// Open the output file:
	hOutputFile = fopen(fileNameXzip,"wb+");
	if( !hOutputFile )
	{
		printf("Failed to open \"%s\" for writing.\n", fileNameXzip);
		exit( EXIT_FAILURE);
	}
	
	// Create a temporary file for storing the preloaded data:
	hTempFilePreload = tmpfile();
	if( !hTempFilePreload )
	{
		printf( "Error: failed to create temporary file\n" );
		return EXIT_FAILURE;
	}

	// Create a temporary file for storing the non preloaded data
	hTempFileData = tmpfile();
	if( !hTempFileData )
	{
		printf( "Error: failed to create temporary file\n");
		return EXIT_FAILURE;
	}
	
	return EXIT_SUCCESS;
}

bool xZipEnd()
{
	int nPreloadDirectorySize = sizeof(xZipDirectoryEntry_t)*Header.PreloadDirectoryEntries;
	int nRegular2PreloadSize = sizeof(unsigned short) * Header.DirectoryEntries;

	// Compute the size of the preloaded section:
	if( Header.PreloadDirectoryEntries )
	{
		fseek( hTempFilePreload, 0, SEEK_END );
		Header.PreloadBytes = ftell(hTempFilePreload) + nPreloadDirectorySize + nRegular2PreloadSize;	// Raw# of bytes to preload
		fseek( hTempFilePreload, 0, SEEK_SET );
	}
	else
	{
		Header.PreloadBytes = 0;
	}

	// Number of bytes preceeding the preloaded section:
	int nPreloadOffset = sizeof( Header ) + ( sizeof( xZipDirectoryEntry_t ) * Header.DirectoryEntries );

	// Number of bytes to pad between the end of the preload section and the start of the data section:
	int nPadding = ( 512 - ( ( nPreloadOffset + Header.PreloadBytes ) % 512) ) %512;				// Number of alignment bytes after the preload section

	// Offset past the preload section:
	int nDataOffset = nPreloadOffset + Header.PreloadBytes + nPadding;

	// Write out the header: (will need to be rewritten at the end as well) - note: not even bothering to byteswap at this point
	fwrite(&Header,sizeof(Header),1,hOutputFile);


	// Fixup each of the directory entries to make them relative to the beginning of the file.
	for( unsigned i=0; i< Header.DirectoryEntries;i++ )
	{
		xZipDirectoryEntry_t* pDir = &(pDirectoryEntries[i]);

		// Adjust files in the regular data area:
		pDir->StoredOffset = nDataOffset + pDir->StoredOffset;
	}

	// Sort and write the directory:
	printf("Sorting and writing %i directory entries...\n",Header.DirectoryEntries);
	qsort(pDirectoryEntries,Header.DirectoryEntries,sizeof(xZipDirectoryEntry_t),&xZipDirectoryEntry_t::xZipDirectoryEntrySortCompare); 

	// Swap the directory entries:
	for( unsigned i=0; i < Header.DirectoryEntries; i++ )
	{
		g_xzpSwap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>(&pDirectoryEntries[i]);
	}

	fwrite(pDirectoryEntries,Header.DirectoryEntries*sizeof(xZipDirectoryEntry_t),1, hOutputFile);

	// Swap the directory back for later use:
	for( unsigned i=0; i < Header.DirectoryEntries; i++ )
	{
		g_xzpSwap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>(&pDirectoryEntries[i]);
	}

	// Copy the preload section:
	if( Header.PreloadBytes > 0 )
	{
		printf("Generating the preload section...(%u)\n", Header.PreloadBytes);


		// Fixup each of the directory entries to make them relative to the beginning of the file.
		for( unsigned i=0; i< Header.PreloadDirectoryEntries;i++ )
		{
			xZipDirectoryEntry_t* pDir = &(pPreloadDirectoryEntries[i]);

			// Shift preload data down by preload bytes (and skipping over the directory):
			pDir->StoredOffset += nPreloadOffset + nPreloadDirectorySize + nRegular2PreloadSize;
		}

		printf("Sorting %u preload directory entries...\n",Header.PreloadDirectoryEntries);
		qsort(pPreloadDirectoryEntries,Header.PreloadDirectoryEntries,sizeof(xZipDirectoryEntry_t),&xZipDirectoryEntry_t::xZipDirectoryEntrySortCompare); 

		printf("Building regular to preload mapping table for %u entries...\n", Header.DirectoryEntries );
		unsigned short* Regular2Preload = (unsigned short*)malloc( nRegular2PreloadSize );
		for( unsigned i = 0; i < Header.DirectoryEntries; i++ )
		{
			unsigned short j;
			for( j = 0; j < Header.PreloadDirectoryEntries; j++ )
			{
				if( pDirectoryEntries[i].FilenameCRC == pPreloadDirectoryEntries[j].FilenameCRC )
					break;
			}

			// If I couldn't find it mark it as non-existant:
			if( j == Header.PreloadDirectoryEntries )
				j = 0xFFFF;

			Regular2Preload[i] = j;
		}

		printf("Writing preloaded directory entreis...\n" );

		// Swap the preload directory entries:
		for( unsigned i=0; i < Header.PreloadDirectoryEntries; i++ )
		{
			g_xzpSwap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>(&pPreloadDirectoryEntries[i]);
		}

		fwrite( pPreloadDirectoryEntries, Header.PreloadDirectoryEntries*sizeof(xZipDirectoryEntry_t),1, hOutputFile );

		// Swap them back:
		for( unsigned i=0; i < Header.PreloadDirectoryEntries; i++ )
		{
			g_xzpSwap.SwapFieldsToTargetEndian<xZipDirectoryEntry_t>(&pPreloadDirectoryEntries[i]);
		}

		printf("Writing regular to preload mapping (%u bytes)...\n", sizeof(unsigned short)*Header.DirectoryEntries );
		
		// Swap regular to preload mapping:
		g_xzpSwap.SwapBufferToTargetEndian<short>((short*)Regular2Preload, (short*)Regular2Preload, nRegular2PreloadSize / sizeof(short) );

		fwrite( Regular2Preload, nRegular2PreloadSize,1,hOutputFile );

		// Swap it back
		g_xzpSwap.SwapBufferToTargetEndian<short>((short*)Regular2Preload, (short*)Regular2Preload, nRegular2PreloadSize / sizeof(short) );

		printf("Copying %u Preloadable Bytes...\n", Header.PreloadBytes - nPreloadDirectorySize - nRegular2PreloadSize );
		fseek(hTempFilePreload,0,SEEK_SET);
		CopyFileBytes(hOutputFile, hTempFilePreload, Header.PreloadBytes - nPreloadDirectorySize - nRegular2PreloadSize  );
	}

	// Align the data section following the preload section:
	if( nPadding )
	{
		printf("Aligning Data Section Start by %u bytes...\n", nPadding );
		PadFileBytes(hOutputFile, nPadding );
	}

	// Copy the data section:
	fseek(hTempFileData, 0, SEEK_END );
	unsigned length = ftell( hTempFileData );
	fseek(hTempFileData, 0, SEEK_SET );
	printf("Copying %u Bytes...\n",length);

	CopyFileBytes(hOutputFile, hTempFileData, length);

	// Write out the filename data if present:
	if( nFilenameDataLength && Header.FilenameEntries )
	{
		Header.FilenameStringsOffset = ftell(hOutputFile);
		Header.FilenameStringsLength = (Header.FilenameEntries*sizeof(xZipFilenameEntry_t)) + nFilenameDataLength;
		
		// Adjust the offset in each of the filename offsets to absolute position in the file.
		for( unsigned i=0;i<Header.FilenameEntries;i++ )
		{
			pFilenameEntries[i].FilenameOffset += ( Header.FilenameStringsOffset + (Header.DirectoryEntries*sizeof(xZipFilenameEntry_t)));
		}

		printf("Sorting and writing %u filename directory entries...\n",Header.FilenameEntries);

		// Sort the data:
		qsort(pFilenameEntries,Header.FilenameEntries,sizeof(xZipFilenameEntry_t),&xZipFilenameEntry_t::xZipFilenameEntryCompare); 

		// Write the data out:
		for( unsigned int i = 0; i < Header.FilenameEntries; i++ )
		{
			g_xzpSwap.SwapFieldsToTargetEndian<xZipFilenameEntry_t>(&pFilenameEntries[i]);
		}

		fwrite(pFilenameEntries,1,Header.FilenameEntries*sizeof(xZipFilenameEntry_t),hOutputFile);

		// Swap them back:
		for( unsigned int i = 0; i < Header.FilenameEntries; i++ )
		{
			g_xzpSwap.SwapFieldsToTargetEndian<xZipFilenameEntry_t>(&pFilenameEntries[i]);
		}

		printf("Writing %u bytes of filename data...\n",nFilenameDataLength);
		fwrite(pFilenameData,1,nFilenameDataLength,hOutputFile);
	}

	// Compute the total file size, including the size of the footer:
	unsigned OutputFileBytes = ftell(hOutputFile) + sizeof(xZipFooter_t);

	// Write the footer:  (block used to keep possibly swapped footer from being used later)
	{
		xZipFooter_t footer;
		footer.Magic = xZipFooter_t::MAGIC;
		footer.Size = OutputFileBytes;

		g_xzpSwap.SwapFieldsToTargetEndian<xZipFooter_t>( &footer );	// Swap the footer
		fwrite( &footer, 1, sizeof(footer), hOutputFile );
	}

	// Seek back and rewrite the header (filename data changes it for example)
	fseek(hOutputFile,0,SEEK_SET);
	g_xzpSwap.SwapFieldsToTargetEndian<xZipHeader_t>( &Header );	// Swap it to write out:
	fwrite(&Header,1,sizeof(Header),hOutputFile);
	g_xzpSwap.SwapFieldsToTargetEndian<xZipHeader_t>( &Header );	// But then swap it back so we can use it in  memory

	// Shut down
	fclose(hOutputFile);

	// Print the summary
	printf("\n\nSummary: Input:%u, XZip:%u, Directory Entries:%u (%u preloaded), Preloaded Bytes:%u\n\n",InputFileBytes,OutputFileBytes,Header.DirectoryEntries, Header.PreloadDirectoryEntries, Header.PreloadBytes);

	// Shut down:
	fclose(hTempFileData);
	fclose(hTempFilePreload);

	return true;
}

#define PADD_ID	MAKEID('P','A','D','D')

//-----------------------------------------------------------------------------
// xZipComputeWAVPreload
//
// Returns the number of bytes from a xbox compliant WAV file that should go into 
// the preload section:
//-----------------------------------------------------------------------------
unsigned xZipComputeWAVPreload( char *pFileName )
{
	InFileRIFF riff( pFileName, *g_pSndIO );
	if ( riff.RIFFName() != RIFF_WAVE )
	{
		return 0;
	}

	IterateRIFF walk( riff, riff.RIFFSize() );

	while ( walk.ChunkAvailable() )
	{
		// xbox compliant wavs have a single PADD chunk
		if ( walk.ChunkName() == PADD_ID )
		{
			// want to preload data up through PADD chunk header
			// and not the actual pad bytes
			return walk.ChunkFilePosition() + 2*sizeof( int );
		}
		walk.ChunkNext();
	}

	return 0;
}


//-----------------------------------------------------------------------------
// xZipComputeXWVPreload
//
// Returns the number of bytes from a XWV file that should go into the preload
// section:
//-----------------------------------------------------------------------------
unsigned xZipComputeXWVPreload( const char* filename )
{
	FILE* hFile = fopen( filename, "rb" );
	if ( !hFile )
	{
		printf( "Failed to open xwv file: %s\n", filename );
		return 0;
	}

	// Read and validate the XWV header:
	xwvHeader_t header;
	memset( &header, 0, sizeof(header) );
	fread( &header, 1, sizeof(header), hFile );
	fclose( hFile );
	
	if ( header.id != XWV_ID || header.headerSize != sizeof(header) )
		return 0;

	return header.GetPreloadSize();
}

unsigned xZipComputeXTFPreload( const char* filename )
{
#if 0 // X360TBD: Not using XTF anymore
	FILE* hFile = fopen( filename, "rb" );
	if ( !hFile )
	{
		printf("Failed to open file: %s\n", filename);
		return 0;
	}

	XTFFileHeader_t header;
	memset( &header,0, sizeof( header ) );
	fread( &header,1,sizeof(header),hFile);

	fclose(hFile);

	if ( !strncmp( header.fileTypeString, "XTF", 4 ) )
		return header.preloadDataSize;
#endif
	return 0;
}

// TODO: ONLY store them in the preload section:
unsigned xZipComputeVMTPreload( const char* filename )
{
	// Store VMT's entirely
	if ( !strstr(filename,".vmt") )
		return 0;

	FILE* hFile = fopen( filename, "rb" );
	if ( !hFile )
	{
		printf("Failed to open file: %s\n", filename);
		return 0;
	}

	fseek( hFile, 0, SEEK_END );
	unsigned offset = ftell( hFile );
	fclose( hFile );
	return offset;
}

// TODO: ONLY store them in the preload section:
unsigned xZipComputeVHVPreload( const char* filename )
{
	// Store VMT's entirely
	if ( !strstr(filename,".vhv") )
		return 0;

	FILE* hFile = fopen( filename, "rb" );
	if ( !hFile )
	{
		printf("Failed to open file: %s\n", filename);
		return 0;
	}

	fclose( hFile );

	// Just load the header:
	return sizeof(HardwareVerts::FileHeader_t);
}

unsigned xZipComputeXCSPreload( const char* filename )
{
	if( !strstr(filename,".vcs") )
		return 0;

	FILE* hFile = fopen( filename, "rb" );
	if ( !hFile )
	{
		printf("Failed to open file: %s\n", filename);
		return 0;
	}

	XShaderHeader_t header;
	fread(&header,1,sizeof(XShaderHeader_t), hFile);
	fseek(hFile,0,SEEK_END);
	fclose(hFile);
	
	if (!header.IsValid())
		return 0;

	return header.BytesToPreload();
}

unsigned xZipComputeCustomPreloads( const char* filename )
{
	// X360TBD: These all need to act on a utlbuffer
	Assert( 0 );
	return 0;

//	strlwr(filename);

	unsigned offset = xZipComputeXWVPreload( filename );
	if ( offset )
		return offset;

	offset = xZipComputeVMTPreload( filename );
	if ( offset )
		return offset;

	offset = xZipComputeXCSPreload( filename );
	if ( offset )
		return offset;

	offset = xZipComputeVHVPreload( filename );
	if ( offset )
		return offset;

	return xZipComputeXTFPreload( filename );
}

#endif // MAKE_GAMEDATA_TOOL