tf2/tf2_src/utils/shadercompile/shadercompile.cpp

//========= Copyright Valve Corporation, All rights reserved. ============//
//
// Purpose: 
//
// $NoKeywords: $
//
//=============================================================================//
// vmpi_bareshell.cpp : Defines the entry point for the console application.
//

#include <windows.h>
#include <conio.h>
#include <process.h>
#include "vmpi.h"
#include "filesystem.h"
#include "vmpi_filesystem.h"
#include "vmpi_distribute_work.h"
#include "vmpi_tools_shared.h"
#include "cmdlib.h"
#include "utlvector.h"
#include "Utlhash.h"
#include "UtlBuffer.h"
#include "utlstring.h"
#include "utlbinaryblock.h"
#include "tier2/utlstreambuffer.h"
#include "UtlLinkedList.h"
#include "UtlStringMap.h"
#include "tier0/icommandline.h"
#include "tier1/strtools.h"
#include "vstdlib/jobthread.h"
#include "threads.h"
#include "tier0/dbg.h"
#include "tier1/smartptr.h"
#include "interface.h"
#include "ishadercompiledll.h"
#include <direct.h>
#include "io.h"
#include <sys/types.h>
#include <sys/stat.h>
#include "materialsystem/shader_vcs_version.h"
#include "ilaunchabledll.h"
#include <tier1/diff.h>
#include "utlnodehash.h"
#include "lzma/lzma.h"
#include "mathlib/mathlib.h"
#include "tier1/checksum_crc.h"
#include "tier0/tslist.h"
#include "tools_minidump.h"

#include "cmdsink.h"
#include "d3dxfxc.h"
#include "subprocess.h"
#include "cfgprocessor.h"


// Type conversions should be controlled by programmer explicitly - shadercompile makes use of 64-bit integer arithmetics
#pragma warning( error : 4244 )

static inline uint32 uint64_as_uint32( uint64 x )
{
	Assert( x < uint64( uint32( ~0 ) ) );
	return uint32( x );
}

static inline UtlSymId_t int_as_symid( int x )
{
	Assert( ( sizeof( UtlSymId_t ) >= sizeof( int ) ) || ( x >= 0 && x < ( int )( unsigned int )( UtlSymId_t(~0) ) ) );
	return UtlSymId_t( x );
}


// VMPI packets
#define STARTWORK_PACKETID	5
#define WORKUNIT_PACKETID	6
#define ERRMSG_PACKETID		7
#define SHADERHADERROR_PACKETID		8
#define MACHINE_NAME 9

#ifdef _DEBUG
//#define DEBUGFP
#endif


// Dealing with job list
namespace
{

CArrayAutoPtr< CfgProcessor::CfgEntryInfo > g_arrCompileEntries;
uint64 g_numShaders = 0, g_numCompileCommands = 0, g_numStaticCombos = 0;
uint64 g_nStaticCombosPerWorkUnit = 0, g_numCompletedStaticCombos = 0, g_numCommandsCompleted = 0;
uint64 g_numSkippedStaticCombos = 0;

CfgProcessor::CfgEntryInfo const * GetEntryByStaticComboNum( uint64 nStaticCombo, uint64 *pnStaticCombo )
{
	CfgProcessor::CfgEntryInfo const *pInfo;
	uint64 nRemainStaticCombos = nStaticCombo;
	
	for ( pInfo = g_arrCompileEntries.Get(); pInfo && pInfo->m_szName; ++ pInfo )
	{
		if ( nRemainStaticCombos >= pInfo->m_numStaticCombos )
			nRemainStaticCombos -= pInfo->m_numStaticCombos;
		else
			break;
	}

	if ( pnStaticCombo )
		*pnStaticCombo = nRemainStaticCombos;

	return pInfo;
}

}; // `anonymous` namespace

char * PrettyPrintNumber( uint64 k )
{
	static char chCompileString[50] = {0};
	char *pchPrint = chCompileString + sizeof( chCompileString ) - 3;
	for ( uint64 j = 0; k > 0; k /= 10, ++ j )
	{
		( j && !( j % 3 ) ) ? ( * pchPrint -- = ',' ) : 0;
		* pchPrint -- = '0' + char( k % 10 );
	}
	( * ++ pchPrint ) ? 0 : ( * pchPrint = 0 );
	return pchPrint;
}


const char *g_pShaderPath = NULL;
char g_WorkerTempPath[MAX_PATH];
char g_ExeDir[MAX_PATH];
#ifdef DEBUGFP
FILE *g_WorkerDebugFp = NULL;
#endif
bool g_bGotStartWorkPacket = false;
double g_flStartTime;
bool g_bVerbose = false;
bool g_bIsX360 = false;
bool g_bSuppressWarnings = false;

FORCEINLINE long AsTargetLong( long x ) { return ( ( g_bIsX360 ) ? ( BigLong( x ) ) : ( x ) ); }


struct ShaderInfo_t
{
	ShaderInfo_t() { memset( this, 0, sizeof( *this ) ); }

	uint64 m_nShaderCombo;
	uint64 m_nTotalShaderCombos;
	const char *m_pShaderName;
	const char *m_pShaderSrc;
	unsigned m_CentroidMask;
	uint64 m_nDynamicCombos;
	uint64 m_nStaticCombo;
	unsigned m_Flags; // from IShader.h
	char m_szShaderModel[ 12 ];
};

void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo );

struct CByteCodeBlock
{

	CByteCodeBlock *m_pNext, *m_pPrev;
	int m_nCRC32;
	uint64 m_nComboID;
	size_t m_nCodeSize;
	uint8 *m_ByteCode;

	CByteCodeBlock( void )
	{
		m_ByteCode = NULL;
	}

	CByteCodeBlock( void const *pByteCode, size_t nCodeSize, uint64 nComboID )
	{
		m_ByteCode = new uint8[nCodeSize];
		m_nComboID = nComboID;
		m_nCodeSize = nCodeSize;
		memcpy( m_ByteCode, pByteCode, nCodeSize );
		m_nCRC32 = CRC32_ProcessSingleBuffer( pByteCode, nCodeSize );
	}
	
	~CByteCodeBlock( void )
	{
		if ( m_ByteCode )
			delete[] m_ByteCode;
	}
	
};

static int __cdecl CompareDynamicComboIDs( CByteCodeBlock * const *pA, CByteCodeBlock * const *pB )
{
	if ( (*pA)->m_nComboID < (*pB)->m_nComboID )
		return -1;
	if ( (*pA)->m_nComboID > (*pB)->m_nComboID )
		return 1;
	return 0;
}


struct CStaticCombo									// all the data for one static combo
{
	CStaticCombo *m_pNext, *m_pPrev;
	
	uint64 m_nStaticComboID;

	CUtlVector< CByteCodeBlock* > m_DynamicCombos;

	struct PackedCode : protected CArrayAutoPtr<uint8> {
		size_t GetLength() const		{ if( uint8 *pb = Get() ) return *reinterpret_cast<size_t *>( pb ); else return 0; }
		uint8 *GetData() const			{ if( uint8 *pb = Get() ) return pb + sizeof( size_t ); else return NULL; }
		uint8 *AllocData( size_t len )	{ Delete(); if ( len ) { Attach( new uint8[ len + sizeof( size_t ) ] ); *reinterpret_cast<size_t *>( Get() ) = len; } return GetData(); }
	} m_abPackedCode;			// Packed code for entire static combo

	uint64 Key( void ) const
	{
		return m_nStaticComboID;
	}

	CStaticCombo( uint64 nComboID )
	{
		m_nStaticComboID = nComboID;
	}

	~CStaticCombo( void )
	{
		m_DynamicCombos.PurgeAndDeleteElements();
	}
	
	void AddDynamicCombo( uint64 nComboID, void const *pComboData, size_t nCodeSize )
	{
		CByteCodeBlock *pNewBlock = new CByteCodeBlock( pComboData, nCodeSize, nComboID );
		m_DynamicCombos.AddToTail( pNewBlock );
	}

	void SortDynamicCombos( void )
	{
		m_DynamicCombos.Sort( CompareDynamicComboIDs );
	}

	uint8 *AllocPackedCodeBlock( size_t nPackedCodeSize )
	{
		return m_abPackedCode.AllocData( nPackedCodeSize );
	}

};

typedef CUtlNodeHash<CStaticCombo, 7097, uint64> StaticComboNodeHash_t;

template <> 
inline StaticComboNodeHash_t **Construct( StaticComboNodeHash_t ** pMemory )
{
	return ::new( pMemory ) StaticComboNodeHash_t *( NULL ); // Explicitly new with NULL
}

struct CShaderMap : public CUtlStringMap<StaticComboNodeHash_t *> {
	;
} g_ShaderByteCode;



CStaticCombo * StaticComboFromDictAdd( char const *pszShaderName, uint64 nStaticComboId )
{
	StaticComboNodeHash_t *& rpNodeHash = g_ShaderByteCode[ pszShaderName ];
	if ( !rpNodeHash )
	{
		rpNodeHash = new StaticComboNodeHash_t;
	}

	// search for this static combo. make it if not found
	CStaticCombo *pStaticCombo = rpNodeHash->FindByKey( nStaticComboId );
	if ( !pStaticCombo )
	{
		pStaticCombo = new CStaticCombo( nStaticComboId );
		rpNodeHash->Add( pStaticCombo );
	}

	return pStaticCombo;
}

CStaticCombo * StaticComboFromDict( char const *pszShaderName, uint64 nStaticComboId )
{
	if ( StaticComboNodeHash_t *pNodeHash = g_ShaderByteCode[ pszShaderName ] )
		return pNodeHash->FindByKey( nStaticComboId );
	else
		return NULL;
}



CUtlStringMap<ShaderInfo_t> g_ShaderToShaderInfo;

class CompilerMsgInfo
{
public:
	CompilerMsgInfo() : m_numTimesReported( 0 ) {}

public:
	void SetMsgReportedCommand( char const *szCommand, int numTimesReported = 1 ) { if ( !m_numTimesReported ) m_sFirstCommand = szCommand; m_numTimesReported += numTimesReported; }

public:
	char const * GetFirstCommand() const { return m_sFirstCommand.String(); }
	int GetNumTimesReported() const { return m_numTimesReported; }

protected:
	CUtlString m_sFirstCommand;
	int m_numTimesReported;
};

CUtlStringMap<bool> g_Master_ShaderHadError;
CUtlStringMap<bool> g_Master_ShaderWrittenToDisk;
CUtlStringMap<CompilerMsgInfo> g_Master_CompilerMsgInfo;

namespace Threading
{

enum Mode { eSingleThreaded = 0, eMultiThreaded = 1 };

// A special object that makes single-threaded code incur no penalties
// and multithreaded code to be synchronized properly.
template < class MT_MUTEX_TYPE = CThreadFastMutex >
class CSwitchableMutex
{
public:

public:
	FORCEINLINE explicit CSwitchableMutex( Mode eMode, MT_MUTEX_TYPE *pMtMutex = NULL ) : m_pMtx( pMtMutex ), m_pUseMtx( eMode ? pMtMutex : NULL ) {}

public:
	FORCEINLINE void SetMtMutex( MT_MUTEX_TYPE *pMtMutex ) { m_pMtx = pMtMutex; m_pUseMtx = ( m_pUseMtx ? pMtMutex : NULL ); }
	FORCEINLINE void SetThreadedMode( Mode eMode ) { m_pUseMtx = ( eMode ? m_pMtx : NULL ); }

public:
	FORCEINLINE void Lock()				{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Lock(); }
	FORCEINLINE void Unlock()			{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->Unlock(); }

	FORCEINLINE bool TryLock()			{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->TryLock(); else return true; }
	FORCEINLINE bool AssertOwnedByCurrentThread() { if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->AssertOwnedByCurrentThread(); else return true; }
	FORCEINLINE void SetTrace( bool b )	{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) pUseMtx->SetTrace( b ); }

	FORCEINLINE uint32 GetOwnerId() 	{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetOwnerId(); else return 0; }
	FORCEINLINE int	GetDepth() 			{ if ( MT_MUTEX_TYPE *pUseMtx = m_pUseMtx ) return pUseMtx->GetDepth(); else return 0; }

private:
	MT_MUTEX_TYPE *m_pMtx;
	CInterlockedPtr< MT_MUTEX_TYPE > m_pUseMtx;
};


namespace Private
{

	typedef CThreadMutex MtMutexType_t;
	MtMutexType_t g_mtxSyncObjMT;

}; // namespace Private


CSwitchableMutex< Private::MtMutexType_t > g_mtxGlobal( eSingleThreaded, &Private::g_mtxSyncObjMT );


class CGlobalMutexAutoLock
{
public:
	CGlobalMutexAutoLock()		{ g_mtxGlobal.Lock(); }
	~CGlobalMutexAutoLock()		{ g_mtxGlobal.Unlock(); }
};

}; // namespace Threading

// Access to global data should be synchronized by these global locks
#define GLOBAL_DATA_MTX_LOCK()			Threading::g_mtxGlobal.Lock()
#define GLOBAL_DATA_MTX_UNLOCK()		Threading::g_mtxGlobal.Unlock()
#define GLOBAL_DATA_MTX_LOCK_AUTO		Threading::CGlobalMutexAutoLock UNIQUE_ID;



CDispatchReg g_DistributeWorkReg( WORKUNIT_PACKETID, DistributeWorkDispatch );

unsigned long VMPI_Stats_GetJobWorkerID( void )
{
	return 0;
}


bool StartWorkDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
	g_bGotStartWorkPacket = true;
	return true;
}

CDispatchReg g_StartWorkReg( STARTWORK_PACKETID, StartWorkDispatch );

// Consume all characters for which (isspace) is true
template < typename T >
char * ConsumeCharacters( char *szString, T pred )
{
	if ( szString )
	{
		while ( *szString && pred( *szString ) )
		{
			++ szString;
		}
	}

	return szString;
}

char * FindNext( char *szString, char *szSearchSet )
{
	bool bFound = (szString == NULL);
	char *szNext = NULL;

	if ( szString && szSearchSet )
	{
		for ( ; *szSearchSet; ++ szSearchSet )
		{
			if ( char *szTmp = strchr( szString, *szSearchSet ) )
			{
				szNext = bFound ? ( min( szNext, szTmp ) ) : szTmp;
				bFound = true;
			}
		}
	}

	return bFound ? szNext : ( szString + strlen( szString ) );
}

char * FindLast( char *szString, char *szSearchSet )
{
	bool bFound = (szString != NULL);
	char *szNext = NULL;

	if ( szString && szSearchSet )
	{
		for ( ; *szSearchSet; ++ szSearchSet )
		{
			if ( char *szTmp = strrchr( szString, *szSearchSet ) )
			{
				szNext = bFound ? ( max( szNext, szTmp ) ) : szTmp;
				bFound = true;
			}
		}
	}

	return bFound ? szNext : ( szString + strlen( szString ) );
}

void ErrMsgDispatchMsgLine( char const *szCommand, char *szMsgLine, char const *szShaderName = NULL )
{
	// When the filename is specified in front of the message, make sure it is truncated to the bare name only
	if ( V_isalpha( *szMsgLine ) && szMsgLine[1] == ':' )
	{
		// Preceded by drive letter
		szMsgLine += 2;
	}

	// Trim the path from the msg
	// e.g. make string
	//    c:\temp\shadercompiletemp\1234\myfile.fxc(435): warning X3083: Truncating ...
	// look like
	//    myfile.fxc(435): warning X3083: Truncating ...
	// which will be both readable and same coming from different worker machines
	char *szEndFileLinePlant = FindNext( szMsgLine, ":" );
	if ( ':' == *szEndFileLinePlant )
	{
		*szEndFileLinePlant = 0;
		if ( char *szLastSlash = FindLast( szMsgLine, "\\/" ) )
		{
			if ( *szLastSlash )
			{
				*szLastSlash = 0;
				szMsgLine = szLastSlash + 1;
			}
		}
		*szEndFileLinePlant = ':';
	}

	// If the shader file name is not given in the message add it
	if ( szShaderName )
	{
		static char chFitLongMsgLine[4096];
		
		if ( *szMsgLine == '(' )
		{
			sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine );
			szMsgLine = chFitLongMsgLine;
		}
		else if ( !strncmp( szMsgLine, "memory(", 7 ) )
		{
			sprintf( chFitLongMsgLine, "%s%s", szShaderName, szMsgLine+6 );
			szMsgLine = chFitLongMsgLine;
		}
	}

	// Now store the message with the command it was generated from
	g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand );
}

void ErrMsgDispatchInt( char *szMessage, char const *szShaderName = NULL )
{
	// First line is the command number "szCommand"
	char *szCommand = ConsumeCharacters( szMessage, isspace );
	char *szMessageListing = FindNext(szCommand, "\r\n");
	char chTerminator = *szMessageListing;
	*( szMessageListing ++ ) = 0;

	// Now come the command lines actually
	while ( chTerminator )
	{
		char *szMsgText = ConsumeCharacters( szMessageListing, isspace );
		szMessageListing = FindNext( szMsgText, "\r\n" );
		chTerminator = *szMessageListing;
		*( szMessageListing ++ ) = 0;

		if( *szMsgText )
		{
			// Trim command at redirection character if present
			* FindNext( szCommand, ">" ) = 0;
			ErrMsgDispatchMsgLine( szCommand, szMsgText, szShaderName );
		}
	}
}

//
//	BUFFER:
//			1 byte = *			= buffer type
//
//			string				= message
//			1 byte = \n			= newline delimiting the message
//
//			string				= command that first encountered the message
//			1 byte = \n			= newline delimiting the command
//
//			string				= printed number of times the message was encountered
//			1 byte = \n			= newline delimiting the number
//
//			1 byte = 0			= null-terminator for the buffer
//
bool ErrMsgDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
	GLOBAL_DATA_MTX_LOCK_AUTO;

	bool bInvalidPkgRetCode = true;

	// Parse the err msg packet
	char *szMsgLine = pBuf->data + 1;
	
	char *szCommand = FindNext( szMsgLine, "\n" );
	if ( !*szCommand )
		return bInvalidPkgRetCode;
	*( szCommand ++ ) = 0;

	char *szNumTimesReported = FindNext( szCommand, "\n" );
	if ( !*szNumTimesReported )
		return bInvalidPkgRetCode;
	*( szNumTimesReported ++ ) = 0;

	char *szTerminator = FindNext( szNumTimesReported, "\n" );
	if ( !*szTerminator )
		return bInvalidPkgRetCode;
	*( szTerminator ++ ) = 0;

	// Set the msg info
	g_Master_CompilerMsgInfo[ szMsgLine ].SetMsgReportedCommand( szCommand, atoi( szNumTimesReported ) );
	
	return true;
}

CDispatchReg g_ErrMsgReg( ERRMSG_PACKETID, ErrMsgDispatch );

void ShaderHadErrorDispatchInt( char const *szShader )
{
	g_Master_ShaderHadError[ szShader ] = true;
}

//
//	BUFFER:
//			1 byte = *			= buffer type
//
//			string				= shader name
//			1 byte = 0			= null-terminator for the name
//
bool ShaderHadErrorDispatch( MessageBuffer *pBuf, int iSource, int iPacketID )
{
	GLOBAL_DATA_MTX_LOCK_AUTO;

	ShaderHadErrorDispatchInt( pBuf->data + 1 );
	return true;
}

CDispatchReg g_ShaderHadErrorReg( SHADERHADERROR_PACKETID, ShaderHadErrorDispatch );

void DebugOut( const char *pMsg, ... )
{
	if (g_bVerbose)
	{
		char msg[2048];
		va_list marker;
		va_start( marker, pMsg );
		_vsnprintf( msg, sizeof( msg ), pMsg, marker );
		va_end( marker );

		Msg( "%s", msg );

#ifdef DEBUGFP
		fprintf( g_WorkerDebugFp, "%s", msg );
		fflush( g_WorkerDebugFp );
#endif
	}
}

void Vmpi_Worker_DefaultDisconnectHandler( int procID, const char *pReason )
{
	Msg( "Master disconnected.\n ");
	DebugOut( "Master disconnected.\n" );
	TerminateProcess( GetCurrentProcess(), 1 );
}

typedef void ( * DisconnectHandlerFn_t )( int procID, const char *pReason );
DisconnectHandlerFn_t g_fnDisconnectHandler = Vmpi_Worker_DefaultDisconnectHandler;

// Worker should implement this so it will quit nicely when the master disconnects.
void MyDisconnectHandler( int procID, const char *pReason )
{
	// If we're a worker, then it's a fatal error if we lose the connection to the master.
	if ( !g_bMPIMaster && g_fnDisconnectHandler )
	{
		(* g_fnDisconnectHandler)( procID, pReason );
	}
}



// new format:
// ver#
// total shader combos
// total dynamic combos
// flags
// centroid mask
// total non-skipped static combos
// [ (sorted by static combo id)
//   static combo id
//   file offset of packed dynamic combo
// ]
// 0xffffffff  (sentinel key)
// end of file offset (so can tell compressed size of last combo)
//
// # of duplicate static combos  (if version >= 6 )
// [ (sorted by static combo id)
//   static combo id
//   id of static bombo which is identical 
// ]
//
// each packed dynamic combo for a given static combo is stored as a series of compressed blocks.
//  block 1:
//     ulong blocksize  (high bit set means uncompressed)
//     block data
//  block2..
//  0xffffffff  indicates no more blocks for this combo
//
// each block, when uncompressed, holds one or more dynamic combos:
//   dynamic combo id   (full id if v<6, dynamic combo id only id >=6)
//   size of shader
//   ..
// there is no terminator - the size of the uncompressed shader tells you when to stop




// this record is then bzip2'd.

// qsort driver function
// returns negative number if idA is less than idB, positive when idA is greater than idB
// and zero if the ids are equal

static int __cdecl CompareDupComboIndices( const StaticComboAliasRecord_t *pA, const StaticComboAliasRecord_t *pB )
{
	if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
		return -1;
	if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
		return 1;
	return 0;
}

static void FlushCombos( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer, MessageBuffer *pBuf )
{
	if ( !pDynamicComboBuffer->TellPut() )
		// Nothing to do here
		return;

	size_t nCompressedSize;
	uint8 *pCompressedShader = LZMA_OpportunisticCompress( reinterpret_cast<uint8 *> ( pDynamicComboBuffer->Base() ),
	                                                       pDynamicComboBuffer->TellPut(),
	                                                       &nCompressedSize );
	// high 2 bits of length =
	// 00 = bzip2 compressed
	// 10 = uncompressed
	// 01 = lzma compressed
	// 11 = unused

	if ( ! pCompressedShader )
	{
		// it grew
		long lFlagSize = AsTargetLong( 0x80000000 | pDynamicComboBuffer->TellPut() );
		pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
		pBuf->write( pDynamicComboBuffer->Base(), pDynamicComboBuffer->TellPut() );
		*pnTotalFlushedSize += sizeof( lFlagSize ) + pDynamicComboBuffer->TellPut();
	}
	else
	{
		long lFlagSize = AsTargetLong( 0x40000000 | nCompressedSize );
		pBuf->write( &lFlagSize, sizeof( lFlagSize ) );
		pBuf->write( pCompressedShader, nCompressedSize );
		delete[] pCompressedShader;
		*pnTotalFlushedSize += sizeof( lFlagSize ) + nCompressedSize;
	}
	pDynamicComboBuffer->Clear();							// start over
}

static void OutputDynamicCombo( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
							    MessageBuffer *pBuf, uint64 nComboID, int nComboSize,
								uint8 *pComboCode )
{
	if ( pDynamicComboBuffer->TellPut() + nComboSize+16 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
	{
		FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
	}

	pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) );
	pDynamicComboBuffer->PutInt( nComboSize );
//	pDynamicComboBuffer->PutInt( CRC32_ProcessSingleBuffer( pComboCode, nComboSize ) );
	pDynamicComboBuffer->Put( pComboCode, nComboSize );
}

static void OutputDynamicComboDup( size_t *pnTotalFlushedSize, CUtlBuffer *pDynamicComboBuffer,
								   MessageBuffer *pBuf, uint64 nComboID, uint64 nBaseCombo )
{
	if ( pDynamicComboBuffer->TellPut() + 8 >= MAX_SHADER_UNPACKED_BLOCK_SIZE )
	{
		FlushCombos( pnTotalFlushedSize, pDynamicComboBuffer, pBuf );
	}
	pDynamicComboBuffer->PutInt( uint64_as_uint32( nComboID ) | 0x80000000 );
	pDynamicComboBuffer->PutInt( uint64_as_uint32( nBaseCombo ) );
}

void GetVCSFilenames( char *pszMainOutFileName, ShaderInfo_t const &si )
{
	sprintf( pszMainOutFileName, "%s\\shaders\\fxc", g_pShaderPath );

	struct	_stat buf;
	if( _stat( pszMainOutFileName, &buf ) == -1 )
	{
		printf( "mkdir %s\n", pszMainOutFileName );
		// doh. . need to make the directory that the vcs file is going to go into.
		_mkdir( pszMainOutFileName );
	}

	strcat( pszMainOutFileName, "\\" );
	strcat( pszMainOutFileName, si.m_pShaderName );

	if ( g_bIsX360 )
	{
		strcat( pszMainOutFileName, ".360" );
	}

	strcat( pszMainOutFileName, ".vcs" );					// Different extensions for main output file

	// Check status of vcs file...
	if( _stat( pszMainOutFileName, &buf ) != -1 )
	{
		// The file exists, let's see if it's writable.
		if( !( buf.st_mode & _S_IWRITE ) )
		{
			// It isn't writable. . we'd better change its permissions (or check it out possibly)
			printf( "Warning: making %s writable!\n", pszMainOutFileName );
			_chmod( pszMainOutFileName, _S_IREAD | _S_IWRITE );
		}
	}
}


// WriteShaderFiles
//
// should be called either on the main thread or
// on the async writing thread.
//
// So the function WriteShaderFiles should not be reentrant, however the
// data that it uses might be updated by the main thread when built pieces
// are received from the workers.
//
#define STATIC_COMBO_HASH_SIZE 73

struct StaticComboAuxInfo_t : StaticComboRecord_t
{
	uint32 m_nCRC32;											// CRC32 of packed data
	struct CStaticCombo *m_pByteCode;
};

static int __cdecl CompareComboIds( const StaticComboAuxInfo_t *pA, const StaticComboAuxInfo_t *pB )
{
	if ( pA->m_nStaticComboID < pB->m_nStaticComboID )
		return -1;
	if ( pA->m_nStaticComboID > pB->m_nStaticComboID )
		return 1;
	return 0;
}

static void WriteShaderFiles( const char *pShaderName )
{
	if ( !g_Master_ShaderWrittenToDisk.Defined( pShaderName ) )
		g_Master_ShaderWrittenToDisk[ pShaderName ] = true;
	else
		return;

	bool bShaderFailed = g_Master_ShaderHadError.Defined( pShaderName );
	char const *szShaderFileOperation = bShaderFailed ? "Removing failed" : "Writing";

	//
	// Progress indication
	//
	if ( g_numCommandsCompleted < g_numCompileCommands )
	{
		static char chProgress[] = { '/', '-', '\\', '|' };
		static int iProgressSymbol = 0;
		Msg( "\b%c", chProgress[ ( ++ iProgressSymbol ) % 4 ] );
	}
	else
	{
		char chShaderName[33];
		Q_snprintf( chShaderName, 29, "%s...", pShaderName );
		sprintf( chShaderName + sizeof( chShaderName ) - 5, "..." );
		Msg( "\r%s %s   \r", szShaderFileOperation, chShaderName );
	}

	//
	// Retrieve the data we are going to operate on
	// from global variables under lock.
	//
	GLOBAL_DATA_MTX_LOCK();
	StaticComboNodeHash_t *pByteCodeArray;
	{
		StaticComboNodeHash_t *&rp = g_ShaderByteCode[pShaderName]; // Get a static combo pointer, reset it as well
		pByteCodeArray = rp;
		rp = NULL;

		/*
		Assert( pByteCodeArray );
		if ( !pByteCodeArray )
			ShaderHadErrorDispatchInt( pShaderName );
		*/
	}
	ShaderInfo_t shaderInfo = g_ShaderToShaderInfo[pShaderName];
	if ( !shaderInfo.m_pShaderName )
	{
		for ( CfgProcessor::CfgEntryInfo const *pAnalyze = g_arrCompileEntries.Get() ;
				pAnalyze->m_szName ;
				++ pAnalyze )
		{
			if ( !strcmp( pAnalyze->m_szName, pShaderName ) )
			{
				Shader_ParseShaderInfoFromCompileCommands( pAnalyze, shaderInfo );
				g_ShaderToShaderInfo[ pShaderName ] = shaderInfo;
				break;
			}
		}
	}
	GLOBAL_DATA_MTX_UNLOCK();

	if ( !shaderInfo.m_pShaderName )
		return;

	//
	// Shader vcs file name
	//
	char szVCSfilename[MAX_PATH];
	GetVCSFilenames( szVCSfilename, shaderInfo );

	if ( bShaderFailed )
	{
		DebugOut( "Removing failed shader file \"%s\".\n", szVCSfilename );
		unlink( szVCSfilename );
		return;
	}
	
	if ( !pByteCodeArray )
		return;

	DebugOut( "%s : %I64u combos centroid mask: 0x%x numDynamicCombos: %I64u flags: 0x%x\n", 
		pShaderName, shaderInfo.m_nTotalShaderCombos, 
		shaderInfo.m_CentroidMask, shaderInfo.m_nDynamicCombos, shaderInfo.m_Flags );

	//
	// Static combo headers
	//
	CUtlVector< StaticComboAuxInfo_t > StaticComboHeaders;

	StaticComboHeaders.EnsureCapacity( 1 + pByteCodeArray->Count() ); // we know how much ram we need

	CUtlVector< int > comboIndicesHashedByCRC32[STATIC_COMBO_HASH_SIZE];
	CUtlVector< StaticComboAliasRecord_t > duplicateCombos;

	// now, lets fill in our combo headers, sort, and write
	for( int nChain = 0 ; nChain < NELEMS( pByteCodeArray->m_HashChains) ; nChain++ )
	{
		for( CStaticCombo *pStatic = pByteCodeArray->m_HashChains[ nChain ].m_pHead;
			 pStatic;
			 pStatic = pStatic->m_pNext )
		{
			if ( pStatic->m_abPackedCode.GetLength() )
			{
				StaticComboAuxInfo_t Hdr;
				Hdr.m_nStaticComboID = uint64_as_uint32( pStatic->m_nStaticComboID );
				Hdr.m_nFileOffset = 0;							// fill in later
				Hdr.m_nCRC32 = CRC32_ProcessSingleBuffer( pStatic->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetLength() );
				int nHashIdx = Hdr.m_nCRC32 % STATIC_COMBO_HASH_SIZE;
				Hdr.m_pByteCode = pStatic;
				// now, see if we have an identical static combo
				bool bIsDuplicate = false;
				for( int i = 0; i < comboIndicesHashedByCRC32[nHashIdx].Count() ; i++ )
				{
					StaticComboAuxInfo_t const &check = StaticComboHeaders[comboIndicesHashedByCRC32[nHashIdx][i]];
					if (
						( check.m_nCRC32 == Hdr.m_nCRC32 ) &&
						( check.m_pByteCode->m_abPackedCode.GetLength() == pStatic->m_abPackedCode.GetLength() ) &&
						( memcmp( check.m_pByteCode->m_abPackedCode.GetData(), pStatic->m_abPackedCode.GetData(), check.m_pByteCode->m_abPackedCode.GetLength() ) == 0 )
						)
					{
						// this static combo is the same as another one!!
						StaticComboAliasRecord_t aliasHdr;
						aliasHdr.m_nStaticComboID = Hdr.m_nStaticComboID;
						aliasHdr.m_nSourceStaticCombo = check.m_nStaticComboID;
						duplicateCombos.AddToTail( aliasHdr );
						bIsDuplicate = true;
						break;
					}
				}

				if ( ! bIsDuplicate )
				{
					StaticComboHeaders.AddToTail( Hdr );
					comboIndicesHashedByCRC32[nHashIdx].AddToTail( StaticComboHeaders.Count() - 1 );
				}
			}
		}
	}
	// add sentinel key
	StaticComboAuxInfo_t Hdr;
	Hdr.m_nStaticComboID = 0xffffffff;
	Hdr.m_nFileOffset = 0;
	StaticComboHeaders.AddToTail( Hdr );
	
	// now, sort. sentinel key will end up at end
	StaticComboHeaders.Sort( CompareComboIds );

	// Set the CRC to zero for now. . will patch in copyshaders.pl with the correct CRC.
	unsigned int crc32 = 0;

	//
	// Shader file stream buffer
	//
	CUtlStreamBuffer ShaderFile( szVCSfilename, NULL );			// Streaming buffer for vcs file (since this can blow memory)
	ShaderFile.SetBigEndian( g_bIsX360 );						// Swap the header bytes to X360 format

	// ------ Header --------------
	ShaderFile.PutInt( SHADER_VCS_VERSION_NUMBER );				// Version
	ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nTotalShaderCombos ) );
	ShaderFile.PutInt( uint64_as_uint32( shaderInfo.m_nDynamicCombos ) );
	ShaderFile.PutUnsignedInt( shaderInfo.m_Flags );
	ShaderFile.PutUnsignedInt( shaderInfo.m_CentroidMask );
	ShaderFile.PutUnsignedInt( StaticComboHeaders.Count() );
	ShaderFile.PutUnsignedInt( crc32 );

	// static combo dictionary
	int nDictionaryOffset= ShaderFile.TellPut();

	// we will re write this one we know the offsets
	ShaderFile.Put( StaticComboHeaders.Base(), sizeof( StaticComboRecord_t ) * StaticComboHeaders.Count() ); // dummy write, 8 bytes per static combo

	ShaderFile.PutUnsignedInt( duplicateCombos.Count() );
	// now, write out all duplicate header records
	// sort duplicate combo records for binary search
	duplicateCombos.Sort( CompareDupComboIndices );

	for( int i = 0; i < duplicateCombos.Count(); i++ )
	{
		ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nStaticComboID );
		ShaderFile.PutUnsignedInt( duplicateCombos[i].m_nSourceStaticCombo );
	}

	// now, write out all static combos
	for( int i=0 ; i<StaticComboHeaders.Count(); i++ )
	{
		StaticComboRecord_t &SRec = StaticComboHeaders[i];
		SRec.m_nFileOffset = ShaderFile.TellPut();
		if ( SRec.m_nStaticComboID != 0xffffffff )			// sentinel key?
		{
			CStaticCombo *pStatic=pByteCodeArray->FindByKey( SRec.m_nStaticComboID );
			Assert( pStatic );

			// Put the packed chunk of code for this static combo
			if ( size_t nPackedLen = pStatic->m_abPackedCode.GetLength() )
				ShaderFile.Put( pStatic->m_abPackedCode.GetData(), nPackedLen );

			ShaderFile.PutInt( 0xffffffff );				// end of dynamic combos
		}

		if ( g_bIsX360 )
		{
			SRec.m_nFileOffset = BigLong( SRec.m_nFileOffset );
			SRec.m_nStaticComboID = BigLong( SRec.m_nStaticComboID );
		}
	}
	ShaderFile.Close();

	//
	// Re-writing the combo header
	//
	{
		FILE *Handle=fopen( szVCSfilename, "rb+" );
		if (! Handle )
			printf(" failed to re-open %s\n",szVCSfilename );

		fseek( Handle, nDictionaryOffset, SEEK_SET );

		// now, rewrite header. data is already byte-swapped appropriately
		for( int i = 0; i < StaticComboHeaders.Count(); i++ )
		{
			fwrite( &( StaticComboHeaders[i].m_nStaticComboID ), 4, 1, Handle );
			fwrite( &( StaticComboHeaders[i].m_nFileOffset ), 4, 1, Handle );
		}
		fclose( Handle );
	}

	// Finalize, free memory
	delete pByteCodeArray;

	if ( g_numCommandsCompleted >= g_numCompileCommands )
	{
		Msg( "\r                                                                \r" );
	}
}

// pBuf is ready to read the results written to the buffer in ProcessWorkUnitFn.
// work is done. .master gets it back this way.
// compiled code in pBuf
void Master_ReceiveWorkUnitFn( uint64 iWorkUnit, MessageBuffer *pBuf, int iWorker )
{
	GLOBAL_DATA_MTX_LOCK_AUTO;

	uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
	uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
	comboEnd = min( g_numStaticCombos, comboEnd );

	char const *chLastShaderName = "";
	ShaderInfo_t siLastShaderInfo;
	memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );
	siLastShaderInfo.m_pShaderName = chLastShaderName;

	uint64 nComboOfTheEntry = 0;
	CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
	nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;

	for( uint64 iCombo = comboStart; iCombo ++ < comboEnd;
		 ( ( ! nComboOfTheEntry -- ) ? ( ++ pEntry, nComboOfTheEntry = pEntry->m_numStaticCombos - 1 ) : 0 ) )
	{
		Assert( nComboOfTheEntry < pEntry->m_numStaticCombos );

		// Read length
		int len;
		pBuf->read( &len, sizeof( len ) );

		// Length can indicate the number of skips to make
		if ( len <= 0 )
		{
			// remember how many static combos get skipped
			g_numSkippedStaticCombos += -len;

			// then we skip as instructed
			for ( int64 numSkips = - len - 1;
					numSkips > 0; )
			{
				if ( numSkips <= nComboOfTheEntry )
				{
					nComboOfTheEntry -= numSkips;
					iCombo += numSkips;
					numSkips = 0;
				}
				else
				{
					numSkips -= nComboOfTheEntry + 1;
					iCombo += nComboOfTheEntry + 1;
					++ pEntry;
					nComboOfTheEntry = pEntry->m_numStaticCombos - 1;
				}
			}
			
			if ( iCombo < comboEnd )
				continue;
			else
				break;
		}

		// Shader code arrived
		char const *chShaderName = pEntry->m_szName;

		// If starting new shader remember shader info
		if ( chLastShaderName != chShaderName )
		{
			Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );

			chLastShaderName = chShaderName;
			g_ShaderToShaderInfo[ chLastShaderName ] = siLastShaderInfo;
		}

		// Read buffer
		uint8 *pCodeBuffer = StaticComboFromDictAdd( chShaderName, nComboOfTheEntry )->AllocPackedCodeBlock( len );

		if ( pCodeBuffer )
			pBuf->read( pCodeBuffer, len );
	}
}


//
// A function that will wait for right Ctrl+Alt+Shift to be held down simultaneously.
// This is useful for debugging short-lived processes and gives time for debugger to
// get attached.
//
void DebugSafeWaitPoint( bool bForceWait = false )
{
	static bool s_bDebuggerAttached = ( CommandLine()->FindParm( "-debugwait" ) == 0 );
	
	if ( bForceWait )
	{
		s_bDebuggerAttached = false;
	}

	if ( !s_bDebuggerAttached )
	{
		Msg( "Waiting for right Ctrl+Alt+Shift to continue..." );
		while ( !s_bDebuggerAttached )
		{
			Msg( "." );
			Sleep(1000);

			if ( short( GetAsyncKeyState( VK_RCONTROL ) ) < 0 &&
				short( GetAsyncKeyState( VK_RSHIFT ) ) < 0 &&
				short( GetAsyncKeyState( VK_RMENU ) ) < 0 )
			{
				s_bDebuggerAttached = true;
			}
		}
		Msg( " ok.\n" );
	}
}

// same as "system", but doesn't pop up a window
void MySystem( char const * const pCommand, CmdSink::IResponse **ppResponse )
{
	// Trap the command in InterceptFxc
	if ( InterceptFxc::TryExecuteCommand( pCommand, ppResponse ) )
	{
		Sleep( 0 );
		return;
	}

	unlink( "shader.o" );

	FILE *batFp = fopen( "temp.bat", "w" );
	fprintf( batFp, "%s\n", pCommand );
	fclose( batFp );
	
	STARTUPINFO si;
	PROCESS_INFORMATION pi;
	
	ZeroMemory( &si, sizeof(si) );
	si.cb = sizeof(si);
	ZeroMemory( &pi, sizeof(pi) );
	
	// Start the child process. 
	if( !CreateProcess( NULL, // No module name (use command line). 
		"temp.bat", // Command line. 
		NULL,             // Process handle not inheritable. 
		NULL,             // Thread handle not inheritable. 
		FALSE,            // Set handle inheritance to FALSE. 
		IDLE_PRIORITY_CLASS | CREATE_NO_WINDOW,                // No creation flags. 
		NULL,             // Use parent's environment block. 
		g_WorkerTempPath, // Use parent's starting directory. 
		&si,              // Pointer to STARTUPINFO structure.
		&pi )             // Pointer to PROCESS_INFORMATION structure.
		) 
	{
		Error( "CreateProcess failed." );
		Assert( 0 );
	}
	
	// Wait until child process exits.
	WaitForSingleObject( pi.hProcess, INFINITE );
	
	// Close process and thread handles. 
	CloseHandle( pi.hProcess );
	CloseHandle( pi.hThread );
}

// Assemble a reply package to the master from the compiled bytecode
// return the length of the package.
size_t AssembleWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
								   MessageBuffer *pBuf )
{
	GLOBAL_DATA_MTX_LOCK();
	CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
	StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ];
	GLOBAL_DATA_MTX_UNLOCK();

	size_t nBytesWritten = 0;

	if ( pStComboRec && pStComboRec->m_DynamicCombos.Count() )
	{
		CUtlBuffer ubDynamicComboBuffer;
		ubDynamicComboBuffer.SetBigEndian( g_bIsX360 );

		pStComboRec->SortDynamicCombos();
		// iterate over all dynamic combos. 
		for(int i = 0 ; i < pStComboRec->m_DynamicCombos.Count(); i++ )
		{
			CByteCodeBlock *pCode = pStComboRec->m_DynamicCombos[i];
			// check if we have already output an identical combo
			bool bDup = false;
#if 0
			// check for duplicate bytecode. actually doesn't save much because bzip does a good
			// job compressing dupes.
			for( int j = 0; j < i; j++ )
			{
				if (
					( pCode->m_nCRC32 == pStComboRec->m_DynamicCombos[j]->m_nCRC32 ) &&
					( pCode->m_nCodeSize == pStComboRec->m_DynamicCombos[j]->m_nCodeSize ) &&
					( memcmp( pCode->m_ByteCode, pStComboRec->m_DynamicCombos[i]->m_ByteCode, pCode->m_nCodeSize ) == 0 )
					)										// identical bytecode?
				{
					bDup = true;
					OutputDynamicComboDup( &nBytesWritten, &ubDynamicComboBuffer,
										   pBuf, pCode->m_nComboID,
										   pStComboRec->m_DynamicCombos[j]->m_nComboID );
				}
			}
#endif
			if ( ! bDup )
				OutputDynamicCombo( &nBytesWritten, &ubDynamicComboBuffer,
									pBuf, pCode->m_nComboID,
									pCode->m_nCodeSize, pCode->m_ByteCode );
		}
		FlushCombos( &nBytesWritten, &ubDynamicComboBuffer, pBuf );
	}

	// Time to limit amount of prints
	static float s_fLastInfoTime = 0;
	float fCurTime = ( float ) Plat_FloatTime();

	GLOBAL_DATA_MTX_LOCK();
	if ( pStComboRec )
		pByteCodeArray->DeleteByKey( nComboOfEntry );
	if( fabs( fCurTime - s_fLastInfoTime ) > 1.f )
	{
		Msg( "\rCompiling  %s  [ %2llu remaining ] ...         \r",
			 pEntry->m_szName, nComboOfEntry );
		s_fLastInfoTime = fCurTime;
	}
	GLOBAL_DATA_MTX_UNLOCK();

	return nBytesWritten;
}

// Copy a reply package to the master from the compiled bytecode
// return the length of the data copied.
size_t CopyWorkerReplyPackage( CfgProcessor::CfgEntryInfo const *pEntry, uint64 nComboOfEntry,
							   MessageBuffer *pBuf, int nSkipsSoFar )
{
	GLOBAL_DATA_MTX_LOCK();
		CStaticCombo *pStComboRec = StaticComboFromDict( pEntry->m_szName, nComboOfEntry );
		StaticComboNodeHash_t *pByteCodeArray = g_ShaderByteCode[ pEntry->m_szName ]; // Get a static combo pointer
	GLOBAL_DATA_MTX_UNLOCK();

	int len = pStComboRec ? pStComboRec->m_abPackedCode.GetLength() : NULL;

	if ( len )
	{
		if ( nSkipsSoFar )
		{
			pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
		}

		pBuf->write( &len, sizeof( len ) );
		if ( len )
			pBuf->write( pStComboRec->m_abPackedCode.GetData(), len );
	
	}
	
	if ( pStComboRec )
	{
		GLOBAL_DATA_MTX_LOCK();
			pByteCodeArray->DeleteByKey( nComboOfEntry );
		GLOBAL_DATA_MTX_UNLOCK();
	}

	return size_t( len );
}



template < typename TMutexType >
class CWorkerAccumState : public CParallelProcessorBase < CWorkerAccumState < TMutexType > >
{
	friend ThisParallelProcessorBase_t;

private:
	static bool & DisconnectState() { static bool sb = false; return sb; }
	static void Special_DisconnectHandler( int procID, const char *pReason ) { DisconnectState() = true; }

public:
	explicit CWorkerAccumState( TMutexType *pMutex ) :
		m_pMutex( pMutex ), m_iFirstCommand( 0 ), m_iNextCommand( 0 ),
		m_iEndCommand( 0 ), m_iLastFinished( 0 ),
		m_hCombo( NULL ),
		m_fnOldDisconnectHandler( g_fnDisconnectHandler ),
		m_autoRestoreDisconnectHandler( g_fnDisconnectHandler )
		{
			DisconnectState() = false;
		}
	~CWorkerAccumState() { QuitSubs(); }

	void RangeBegin( uint64 iFirstCommand, uint64 iEndCommand );
	void RangeFinished( void );

	void ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo );
	void ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo );
	void HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse );

public:
	using ThisParallelProcessorBase_t::Run;

public:
	bool OnProcess();
	bool OnProcessST();

protected:
	TMutexType *m_pMutex;

protected:
	struct SubProcess
	{
		DWORD dwIndex;
		DWORD dwSvcThreadId;
		uint64 iRunningCommand;
		PROCESS_INFORMATION pi;
		SubProcessKernelObjects *pCommObjs;
	};
	CThreadLocal < SubProcess * > m_lpSubProcessInfo;
	CUtlVector < SubProcess * > m_arrSubProcessInfos;
	uint64 m_iFirstCommand;
	uint64 m_iNextCommand;
	uint64 m_iEndCommand;

	uint64 m_iLastFinished;

	CfgProcessor::ComboHandle m_hCombo;

	DisconnectHandlerFn_t m_fnOldDisconnectHandler;
	CAutoPushPop< DisconnectHandlerFn_t > m_autoRestoreDisconnectHandler;
	
	void QuitSubs( void );
	void TryToPackageData( uint64 iCommandNumber );
	void PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs );
};

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeBegin( uint64 iFirstCommand, uint64 iEndCommand )
{
	m_iFirstCommand = iFirstCommand;
	m_iNextCommand = iFirstCommand;
	m_iEndCommand = iEndCommand;
	m_iLastFinished = iFirstCommand;
	m_hCombo = NULL;
	CfgProcessor::Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
	
	g_fnDisconnectHandler = Special_DisconnectHandler;

	// Notify all connected sub-processes that the master is still alive
	for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
	{
		if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
		{
			SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
			if ( void *pvMemory = shrmem.Lock() )
			{
				strcpy( ( char * ) pvMemory, "keepalive" );
				shrmem.Unlock();
			}
		}
	}
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::RangeFinished( void )
{
	if( !DisconnectState() )
	{
		// Finish packaging data
		TryToPackageData( m_iEndCommand - 1 );
	}
	else
	{
		// Master disconnected
		QuitSubs();
	}

	g_fnDisconnectHandler = m_fnOldDisconnectHandler;
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::QuitSubs( void )
{
	CUtlVector < HANDLE > m_arrWait;
	m_arrWait.EnsureCapacity( m_arrSubProcessInfos.Count() );

	for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
	{
		if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
		{
			SubProcessKernelObjects_Memory shrmem( pSp->pCommObjs );
			if ( void *pvMemory = shrmem.Lock() )
			{
				strcpy( ( char * ) pvMemory, "quit" );
				shrmem.Unlock();
			}

			m_arrWait.AddToTail( pSp->pi.hProcess );
		}
	}

	if ( m_arrWait.Count() )
	{
		DWORD dwWait = WaitForMultipleObjects( m_arrWait.Count(), m_arrWait.Base(), TRUE, 2 * 1000 );
		if ( WAIT_TIMEOUT == dwWait )
		{
			Warning( "Timed out while waiting for sub-processes to shut down!\n" );
		}
	}

	for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
	{
		if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
		{
			CloseHandle( pSp->pi.hThread );
			CloseHandle( pSp->pi.hProcess );

			delete pSp->pCommObjs;
			delete pSp;
		}
	}

	if ( DisconnectState() )
		Vmpi_Worker_DefaultDisconnectHandler( 0, "Master disconnected during compilation." );
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::PrepareSubProcess( SubProcess **ppSp, SubProcessKernelObjects **ppCommObjs )
{
	SubProcess *pSp = m_lpSubProcessInfo.Get();
	SubProcessKernelObjects *pCommObjs = NULL;

	if ( pSp )
	{
		pCommObjs = pSp->pCommObjs;
	}
	else
	{
		pSp = new SubProcess;
		m_lpSubProcessInfo.Set( pSp );

		pSp->dwSvcThreadId = ThreadGetCurrentId();

		char chBaseNameBuffer[0x30];
		sprintf( chBaseNameBuffer, "SHCMPL_SUB_%08X_%08llX_%08X", pSp->dwSvcThreadId, (long long)time( NULL ), GetCurrentProcessId() );
		pCommObjs = pSp->pCommObjs = new SubProcessKernelObjects_Create( chBaseNameBuffer );

		ZeroMemory( &pSp->pi, sizeof( pSp->pi ) );

		STARTUPINFO si;
		ZeroMemory( &si, sizeof( si ) );
		si.cb = sizeof( si );

		char chCommandLine[0x100];
		sprintf( chCommandLine, "\"%s\\shadercompile.exe\" -subprocess %s", g_WorkerTempPath, chBaseNameBuffer );
#ifdef _DEBUG
		V_strncat( chCommandLine, " -allowdebug", sizeof( chCommandLine ) );
#endif
		BOOL bCreateResult = CreateProcess( NULL, chCommandLine, NULL, NULL, FALSE, CREATE_NO_WINDOW, NULL, g_WorkerTempPath, &si, &pSp->pi );
		( void ) bCreateResult;
		Assert( bCreateResult && "CreateProcess failed?" );

		m_pMutex->Lock();
		pSp->dwIndex = m_arrSubProcessInfos.AddToTail( pSp );
		m_pMutex->Unlock();
	}

	if ( ppSp ) *ppSp = pSp;
	if ( ppCommObjs ) *ppCommObjs = pCommObjs;
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommandThreaded( CfgProcessor::ComboHandle hCombo )
{
	// DebugOut( "threaded: running: \"%s\"\n", szCommand );

	SubProcessKernelObjects *pCommObjs = NULL;
	PrepareSubProcess( NULL, &pCommObjs );

	// Execute the command
	SubProcessKernelObjects_Memory shrmem( pCommObjs );

	{
		void *pvMemory = shrmem.Lock();
		Assert( pvMemory );
		
		Combo_FormatCommand( hCombo, ( char * ) pvMemory );

		shrmem.Unlock();
	}

	// Obtain the command response
	{
		void const *pvMemory = shrmem.Lock();
		Assert( pvMemory );

		// TODO: Vitaliy :: TEMP fix:
		// Usually what happens if we fail to lock here is
		// when our subprocess dies and to recover we will
		// attempt to restart on another worker.
		if ( !pvMemory )
			// ::RaiseException( GetLastError(), EXCEPTION_NONCONTINUABLE, 0, NULL );
			TerminateProcess( GetCurrentProcess(), 1 );

		CmdSink::IResponse *pResponse;
		if ( pvMemory ) 
			pResponse = new CSubProcessResponse( pvMemory );
		else
			pResponse = new CmdSink::CResponseError;

		HandleCommandResponse( hCombo, pResponse );

		shrmem.Unlock();
	}
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::ExecuteCompileCommand( CfgProcessor::ComboHandle hCombo )
{
	CmdSink::IResponse *pResponse = NULL;
	
	{
		char chBuffer[ 4096 ];
		Combo_FormatCommand( hCombo, chBuffer );

		DebugOut( "running: \"%s\"\n", chBuffer );

		MySystem( chBuffer, &pResponse );
	}

	HandleCommandResponse( hCombo, pResponse );
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::HandleCommandResponse( CfgProcessor::ComboHandle hCombo, CmdSink::IResponse *pResponse )
{
	VMPI_HandleSocketErrors();

	if ( !pResponse )
		pResponse = new CmdSink::CResponseFiles( "shader.o", "output.txt" );

	// Command info
	CfgProcessor::CfgEntryInfo const *pEntryInfo = Combo_GetEntryInfo( hCombo );
	uint64 iComboIndex = Combo_GetComboNum( hCombo );
	uint64 iCommandNumber = Combo_GetCommandNum( hCombo );

	if ( pResponse->Succeeded() )
	{
		GLOBAL_DATA_MTX_LOCK();
		uint64 nStComboIdx = iComboIndex / pEntryInfo->m_numDynamicCombos;
		uint64 nDyComboIdx = iComboIndex - ( nStComboIdx * pEntryInfo->m_numDynamicCombos );
		StaticComboFromDictAdd( pEntryInfo->m_szName, nStComboIdx )->AddDynamicCombo( nDyComboIdx , pResponse->GetResultBuffer(), pResponse->GetResultBufferLen() );
		GLOBAL_DATA_MTX_UNLOCK();
		
	}

	// Tell the master that this shader failed
	if ( !pResponse->Succeeded() )
	{
		GLOBAL_DATA_MTX_LOCK();
			ShaderHadErrorDispatchInt( pEntryInfo->m_szName );
		GLOBAL_DATA_MTX_UNLOCK();
	}

	// Process listing even if the shader succeeds for warnings
	char const *szListing = pResponse->GetListing();
	if ( ( !g_bSuppressWarnings && szListing ) || !pResponse->Succeeded() )
	{
		char chCommandNumber[50];
		sprintf( chCommandNumber, "%I64u", iCommandNumber );

		char chUnreportedListing[0xFF];
		if ( !szListing )
		{
			sprintf( chUnreportedListing, "(0): error 0000: Compiler failed without error description, latest version of fxc.exe might give a description." );
			szListing = chUnreportedListing;
		}

		// Send the listing for dispatch
		CUtlBinaryBlock errMsg;
		errMsg.SetLength(
			strlen( chCommandNumber ) + 1 +			// command + newline
			strlen( szListing ) + 1 +				// listing + newline
			1										// null-terminator
			);
		sprintf( ( char * ) errMsg.Get(), "%s\n%s\n", chCommandNumber, szListing );

		GLOBAL_DATA_MTX_LOCK();
		ErrMsgDispatchInt( ( char * ) errMsg.Get(), pEntryInfo->m_szShaderFileName );
		GLOBAL_DATA_MTX_UNLOCK();
	}

	// Maybe zip things up
	TryToPackageData( iCommandNumber );
}

template < typename TMutexType >
void CWorkerAccumState < TMutexType > ::TryToPackageData( uint64 iCommandNumber )
{
	m_pMutex->Lock();

	uint64 iFinishedByNow = iCommandNumber + 1;

	// Check if somebody is running an earlier command
	for ( int k = 0; k < m_arrSubProcessInfos.Count(); ++ k )
	{
		if ( SubProcess *pSp = m_arrSubProcessInfos[ k ] )
		{
			if ( pSp->iRunningCommand < iCommandNumber )
			{
				iFinishedByNow = 0;
				break;
			}
		}
	}

	uint64 iLastFinished = m_iLastFinished;
	if ( iFinishedByNow > m_iLastFinished )
	{
		m_iLastFinished = iFinishedByNow;
		m_pMutex->Unlock();
	}
	else
	{
		m_pMutex->Unlock();
		return;
	}

	CfgProcessor::ComboHandle hChBegin = CfgProcessor::Combo_GetCombo( iLastFinished );
	CfgProcessor::ComboHandle hChEnd = CfgProcessor::Combo_GetCombo( iFinishedByNow );

	Assert( hChBegin && hChEnd );

	CfgProcessor::CfgEntryInfo const *pInfoBegin = Combo_GetEntryInfo( hChBegin );
	CfgProcessor::CfgEntryInfo const *pInfoEnd = Combo_GetEntryInfo( hChEnd );

	uint64 nComboBegin = Combo_GetComboNum( hChBegin ) / pInfoBegin->m_numDynamicCombos;
	uint64 nComboEnd = Combo_GetComboNum( hChEnd ) / pInfoEnd->m_numDynamicCombos;

	for ( ; pInfoBegin && (
	      ( pInfoBegin->m_iCommandStart < pInfoEnd->m_iCommandStart ) ||
	      ( nComboBegin > nComboEnd ) ); )
	{
		// Zip this combo
		MessageBuffer mbPacked;
		size_t nPackedLength = AssembleWorkerReplyPackage( pInfoBegin, nComboBegin, &mbPacked );

		if ( nPackedLength )
		{
			// Packed buffer
			GLOBAL_DATA_MTX_LOCK();
			uint8 *pCodeBuffer = StaticComboFromDictAdd( pInfoBegin->m_szName,
														 nComboBegin )->AllocPackedCodeBlock( nPackedLength );
			GLOBAL_DATA_MTX_UNLOCK();

			if ( pCodeBuffer )
				mbPacked.read( pCodeBuffer, nPackedLength );
		}

		// Next iteration
		if ( ! nComboBegin -- )
		{
			Combo_Free( hChBegin );
			if ( ( hChBegin = CfgProcessor::Combo_GetCombo( pInfoBegin->m_iCommandEnd ) ) != NULL )
			{
				pInfoBegin = Combo_GetEntryInfo( hChBegin );
				nComboBegin = pInfoBegin->m_numStaticCombos - 1;
			}
		}
	}

	Combo_Free( hChBegin );
	Combo_Free( hChEnd );
}


template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcess()
{
	m_pMutex->Lock();
		CfgProcessor::ComboHandle hThreadCombo = m_hCombo ? Combo_Alloc( m_hCombo ) : NULL;
	m_pMutex->Unlock();
	
	uint64 iThreadCommand = ~uint64(0);

	SubProcess *pSp = NULL;
	PrepareSubProcess( &pSp, NULL );

	for ( ; ; )
	{
		m_pMutex->Lock();
			if ( DisconnectState() )
				Combo_Free( m_hCombo );

			if ( m_hCombo )
			{
				Combo_Assign( hThreadCombo, m_hCombo );
				pSp->iRunningCommand = Combo_GetCommandNum( hThreadCombo );
				Combo_GetNext( iThreadCommand, m_hCombo, m_iEndCommand );
			}
			else
			{
				Combo_Free( hThreadCombo );
				iThreadCommand = ~uint64(0);
				pSp->iRunningCommand = ~uint64(0);
			}
		m_pMutex->Unlock();

		if ( hThreadCombo )
		{
			ExecuteCompileCommandThreaded( hThreadCombo );
		}
		else
			break;
	}

	Combo_Free( hThreadCombo );
	return false;
}

template < typename TMutexType >
bool CWorkerAccumState < TMutexType > ::OnProcessST()
{
	while ( m_hCombo )
	{
		ExecuteCompileCommand( m_hCombo );
		
		Combo_GetNext( m_iNextCommand, m_hCombo, m_iEndCommand );
	}
	return false;
}

//
// Worker_ProcessCommandRange_Singleton
//
class Worker_ProcessCommandRange_Singleton
{
public:
	static Worker_ProcessCommandRange_Singleton *& Instance() { static Worker_ProcessCommandRange_Singleton *s_ptr = NULL; return s_ptr; }
	static Worker_ProcessCommandRange_Singleton * GetInstance() { Worker_ProcessCommandRange_Singleton *p = Instance(); Assert( p ); return p; }

public:
	Worker_ProcessCommandRange_Singleton() { Assert( !Instance() ); Instance() = this; Startup(); }
	~Worker_ProcessCommandRange_Singleton() { Assert( Instance() == this ); Instance() = NULL; Shutdown(); }

public:
	void ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd );

protected:
	void Startup( void );
	void Shutdown( void );

	//
	// Multi-threaded section
protected:
	struct MT {
		MT() : pWorkerObj( NULL ), pThreadPool( NULL ) {}

		typedef CThreadFastMutex MultiThreadMutex_t;
		MultiThreadMutex_t mtx;
		
		typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
		WorkerClass_t *pWorkerObj;

		IThreadPool *pThreadPool;
		ThreadPoolStartParams_t tpsp;
	} m_MT;

	//
	// Single-threaded section
protected:
	struct ST {
		ST() : pWorkerObj( NULL ) {}

		typedef CThreadNullMutex MultiThreadMutex_t;
		MultiThreadMutex_t mtx;

		typedef CWorkerAccumState < MultiThreadMutex_t > WorkerClass_t;
		WorkerClass_t *pWorkerObj;
	} m_ST;
};

void Worker_ProcessCommandRange_Singleton::Startup( void )
{
	bool bInitializedThreadPool = false;
	CPUInformation const &cpu = *GetCPUInformation();

	if ( cpu.m_nLogicalProcessors > 1 )
	{
		// Attempt to initialize thread pool
		m_MT.pThreadPool = g_pThreadPool;
		if ( m_MT.pThreadPool )
		{
			m_MT.tpsp.bIOThreads = false;
			m_MT.tpsp.nThreads = cpu.m_nLogicalProcessors - 1;

			if ( m_MT.pThreadPool->Start( m_MT.tpsp ) )
			{
				if ( m_MT.pThreadPool->NumThreads() >= 1 )
				{
					// Make sure that our mutex is in multi-threaded mode
					Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );

					m_MT.pWorkerObj = new MT::WorkerClass_t( &m_MT.mtx );

					bInitializedThreadPool = true;
				}
				else
				{
					m_MT.pThreadPool->Stop();
				}
			}

			if ( !bInitializedThreadPool )
				m_MT.pThreadPool = NULL;
		}
	}

	// Otherwise initialize single-threaded mode
	if ( !bInitializedThreadPool )
	{
		m_ST.pWorkerObj = new ST::WorkerClass_t( &m_ST.mtx );
	}
}

void Worker_ProcessCommandRange_Singleton::Shutdown( void )
{
	if ( m_MT.pThreadPool )
	{
		if( m_MT.pWorkerObj )
			delete m_MT.pWorkerObj;

		m_MT.pThreadPool->Stop();
		m_MT.pThreadPool = NULL;
	}
	else
	{
		if ( m_ST.pWorkerObj )
			delete m_ST.pWorkerObj;
	}
}

void Worker_ProcessCommandRange_Singleton::ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
	if ( m_MT.pThreadPool )
	{
		MT::WorkerClass_t *pWorkerObj = m_MT.pWorkerObj;

		pWorkerObj->RangeBegin( shaderStart, shaderEnd );
		pWorkerObj->Run();
		pWorkerObj->RangeFinished();
	}
	else
	{
		ST::WorkerClass_t *pWorkerObj = m_ST.pWorkerObj;

		pWorkerObj->RangeBegin( shaderStart, shaderEnd );
		pWorkerObj->OnProcessST();
		pWorkerObj->RangeFinished();
	}
}



// You must process the work unit range.
void Worker_ProcessCommandRange( uint64 shaderStart, uint64 shaderEnd )
{
	Worker_ProcessCommandRange_Singleton::GetInstance()->ProcessCommandRange( shaderStart, shaderEnd );
}

// You must append data to pBuf with the work unit results.
void Worker_ProcessWorkUnitFn( int iThread, uint64 iWorkUnit, MessageBuffer *pBuf )
{
	uint64 comboStart = iWorkUnit * g_nStaticCombosPerWorkUnit;
	uint64 comboEnd = comboStart + g_nStaticCombosPerWorkUnit;
	comboEnd = min( g_numStaticCombos, comboEnd );

	// Determine the commands required to be executed:
	uint64 nComboOfTheEntry = 0;
	CfgProcessor::CfgEntryInfo const *pEntry = NULL;

	pEntry = GetEntryByStaticComboNum( comboEnd, &nComboOfTheEntry );
	uint64 commandEnd = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;
	Assert( commandEnd <= g_numCompileCommands );

	pEntry = GetEntryByStaticComboNum( comboStart, &nComboOfTheEntry );
	uint64 commandStart = pEntry->m_iCommandStart + nComboOfTheEntry * pEntry->m_numDynamicCombos;

	// Compile all the shader combos
	Worker_ProcessCommandRange( commandStart, commandEnd );
	nComboOfTheEntry = pEntry->m_numStaticCombos - 1 - nComboOfTheEntry;

	// Copy off the reply packages
	int nSkipsSoFar = 0;
	for ( uint64 kCombo = comboStart; kCombo < comboEnd; ++ kCombo )
	{
		size_t nCpBytes = CopyWorkerReplyPackage( pEntry, nComboOfTheEntry, pBuf, nSkipsSoFar );
		if ( nCpBytes )
			nSkipsSoFar = 0;
		else
			-- nSkipsSoFar;
		if ( nComboOfTheEntry == 0 )
		{
			++pEntry;
			nComboOfTheEntry = pEntry->m_numStaticCombos;
		}
		nComboOfTheEntry--;
	}
	if ( nSkipsSoFar )
	{
		pBuf->write( &nSkipsSoFar, sizeof( nSkipsSoFar ) );
	}

	//////////////////////////////////////////////////////////////////////////
	//
	// Now deliver all our accumulated spew to the master
	//
	//////////////////////////////////////////////////////////////////////////

	// Failed shaders
	for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
	{
		char const *szShaderName = g_Master_ShaderHadError.String( k );
		if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
			continue;

		int const len = strlen( szShaderName );
		
		CUtlBinaryBlock bb;
		bb.SetLength( 1 + len + 1 );
		sprintf( ( char * ) bb.Get(), "%c%s", SHADERHADERROR_PACKETID, szShaderName );

		VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
		VMPI_HandleSocketErrors();
	}

	// Compiler spew
	for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
	{
		char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
		CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];

		char const * const szFirstCmd = cmi.GetFirstCommand();
		int const numReported = cmi.GetNumTimesReported();

		char chNumReported[0x40];
		sprintf( chNumReported, "%d", numReported );

		CUtlBinaryBlock bb;
		bb.SetLength( 1 + strlen(szMsg) + 1 + strlen( szFirstCmd ) + 1 + strlen( chNumReported ) + 1 + 1 );
		sprintf( ( char * ) bb.Get(), "%c%s\n%s\n%s\n", ERRMSG_PACKETID, szMsg, szFirstCmd, chNumReported );

		VMPI_SendData( bb.Get(), bb.Length(), VMPI_MASTER_ID );
		VMPI_HandleSocketErrors();
	}

	// Clean all reported msgs
	g_Master_CompilerMsgInfo.Purge();
}


void Shader_ParseShaderInfoFromCompileCommands( CfgProcessor::CfgEntryInfo const *pEntry, ShaderInfo_t &shaderInfo )
{
	if ( CfgProcessor::ComboHandle hCombo = CfgProcessor::Combo_GetCombo( pEntry->m_iCommandStart ) )
	{
		char cmd[ 4096 ];
		Combo_FormatCommand( hCombo, cmd );
		
		{
			memset( &shaderInfo, 0, sizeof( ShaderInfo_t ) );

			const char *pCentroidMask = strstr( cmd, "/DCENTROIDMASK=" );
			const char *pFlags = strstr( cmd, "/DFLAGS=0x" );
			const char *pShaderModel = strstr( cmd, "/DSHADER_MODEL_" );

			if( !pCentroidMask || !pFlags || !pShaderModel )
			{
				Assert( !"!pCentroidMask || !pFlags || !pShaderModel" );
				return;
			}

			sscanf( pCentroidMask + strlen( "/DCENTROIDMASK=" ), "%u", &shaderInfo.m_CentroidMask );
			sscanf( pFlags + strlen( "/DFLAGS=0x" ), "%x", &shaderInfo.m_Flags );

			// Copy shader model
			pShaderModel += strlen( "/DSHADER_MODEL_" );
			for ( char *pszSm = shaderInfo.m_szShaderModel, * const pszEnd = pszSm + sizeof( shaderInfo.m_szShaderModel ) - 1;
				pszSm < pszEnd ; ++ pszSm )
			{
				char &rchLastChar = (*pszSm = *pShaderModel ++);
				if ( !rchLastChar ||
					V_isspace( rchLastChar ) ||
					'=' == rchLastChar )
				{
					rchLastChar = 0;
					break;
				}
			}

			shaderInfo.m_nShaderCombo = 0;
			shaderInfo.m_nTotalShaderCombos = pEntry->m_numCombos;
			shaderInfo.m_nDynamicCombos = pEntry->m_numDynamicCombos;
			shaderInfo.m_nStaticCombo = 0;

			shaderInfo.m_pShaderName = pEntry->m_szName;
			shaderInfo.m_pShaderSrc = pEntry->m_szShaderFileName;
		}

		Combo_Free( hCombo );
	}
}




void Worker_GetLocalCopyOfShaders( void )
{
	// Create virtual files for all of the stuff that we need to compile the shader
	// make sure and prefix the file name so that it doesn't find it locally.

	char filename[1024];
	sprintf( filename, "%s\\uniquefilestocopy.txt", g_pShaderPath );

	CUtlInplaceBuffer bffr( 0, 0, CUtlBuffer::TEXT_BUFFER );
	if( !g_pFileSystem->ReadFile( filename, NULL, bffr ) )
	{
		fprintf( stderr, "Can't open uniquefilestocopy.txt!\n" );
		exit( -1 );
	}

	while( char *pszLineToCopy = bffr.InplaceGetLinePtr() )
	{
		V_MakeAbsolutePath( filename, sizeof( filename ), pszLineToCopy, g_pShaderPath );
		
		if ( g_bVerbose )
			printf( "getting local copy of shader: \"%s\" (\"%s\")\n", pszLineToCopy, filename );

		CUtlBuffer fileBuf;
		if ( !g_pFileSystem->ReadFile( filename, NULL, fileBuf ) )
		{
			Warning( "Can't find \"%s\"\n", filename );
			continue;
		}

		// Grab just the filename.
		char justFilename[MAX_PATH];
		char *pLastSlash = max( strrchr( pszLineToCopy, '/' ), strrchr( pszLineToCopy, '\\' ) );
		if ( pLastSlash )
			Q_strncpy( justFilename, pLastSlash + 1, sizeof( justFilename ) );
		else
			Q_strncpy( justFilename, pszLineToCopy, sizeof( justFilename ) );

		sprintf( filename, "%s%s", g_WorkerTempPath, justFilename );
		if ( g_bVerbose )
			printf( "creating \"%s\"\n", filename );
		
		FILE *fp3 = fopen( filename, "wb" );
		if ( !fp3 )
		{
			Error( "Can't open '%s' for writing.", pszLineToCopy );
			continue;
		}

		fwrite( fileBuf.Base(), 1, fileBuf.GetBytesRemaining(), fp3 );
		fclose( fp3 );

		// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
		// while we're running since they're in the temp path.

		static CUtlVector< FILE * > s_arrHackedFiles;
		static struct X_s_arrHackedFiles { ~X_s_arrHackedFiles() {
			for ( int k = 0; k < s_arrHackedFiles.Count(); ++ k )
				fclose( s_arrHackedFiles[k] );
		 } } s_autoCloseHackedFiles;

		/* THIS IS THE EVIL LINE ----> */ FILE *fHack = fopen( filename, "r" );
		s_arrHackedFiles.AddToTail( fHack );
		// -- END of EVIL
	}
}

void Worker_GetLocalCopyOfBinary( const char *pFilename )
{
	CUtlBuffer fileBuf;
	char tmpFilename[MAX_PATH];
	sprintf( tmpFilename, "%s\\%s", g_ExeDir, pFilename );
	if ( g_bVerbose )
		printf( "trying to open: %s\n", tmpFilename );
	
	FILE *fp = fopen( tmpFilename, "rb" );
	if( !fp )
	{
		Assert( 0 );
		fprintf( stderr, "Can't open %s!\n", pFilename );
		exit( -1 );
	}
	fseek( fp, 0, SEEK_END );
	int fileLen = ftell( fp );
	fseek( fp, 0, SEEK_SET );
	fileBuf.EnsureCapacity( fileLen );
	int nBytesRead = fread( fileBuf.Base(), 1, fileLen, fp );
	fclose( fp );
	fileBuf.SeekPut( CUtlBuffer::SEEK_HEAD, nBytesRead );

	char newFilename[MAX_PATH];
	sprintf( newFilename, "%s%s", g_WorkerTempPath, pFilename );
	
	FILE *fp2 = fopen( newFilename, "wb" );
	if( !fp2 )
	{
		Assert( 0 );
		fprintf( stderr, "Can't open %s!\n", newFilename );
		exit( -1 );
	}
	fwrite( fileBuf.Base(), 1, fileLen, fp2 );
	fclose( fp2 );

	// SUPER EVIL, but if we don't do this, Windows will randomly nuke files of ours
	// while we're running since they're in the temp path.
	fopen( newFilename, "r" );
}

void Worker_GetLocalCopyOfBinaries( void )
{
	Worker_GetLocalCopyOfBinary( "mysql_wrapper.dll" ); // This is necessary so VMPI doesn't run in SDK mode.
	Worker_GetLocalCopyOfBinary( "vstdlib.dll" );
	Worker_GetLocalCopyOfBinary( "tier0.dll" );
}

void Shared_ParseListOfCompileCommands( void )
{
//	double tt_start = Plat_FloatTime();

	char fileListFileName[1024];
	sprintf( fileListFileName, "%s\\filelist.txt", g_pShaderPath );

	CUtlInplaceBuffer bffr( 0, 0, CUtlInplaceBuffer::TEXT_BUFFER );
	if( !g_pFileSystem->ReadFile( fileListFileName, NULL, bffr) )
	{
		DebugOut( "Can't open %s!\n", fileListFileName );
		fprintf( stderr, "Can't open %s!\n", fileListFileName );
		exit( -1 );
	}

	CfgProcessor::ReadConfiguration( &bffr );
	CfgProcessor::DescribeConfiguration( g_arrCompileEntries );

	for ( CfgProcessor::CfgEntryInfo const *pInfo = g_arrCompileEntries.Get();
		  pInfo && pInfo->m_szName; ++ pInfo )
	{
		++ g_numShaders;
		g_numStaticCombos += pInfo->m_numStaticCombos;
		g_numCompileCommands = pInfo->m_iCommandEnd;
	}

//	double tt_end = Plat_FloatTime();
	
	Msg( "\rCompiling %s commands.         \r", PrettyPrintNumber( g_numCompileCommands ) );
}

void SetupExeDir( int argc, char **argv )
{
	strcpy( g_ExeDir, argv[0] );
	Q_StripFilename( g_ExeDir );

	if ( g_ExeDir[0] == 0 )
	{
		Q_strncpy( g_ExeDir, ".\\", sizeof( g_ExeDir ) );
	}

	Q_FixSlashes( g_ExeDir );
}

void SetupPaths( int argc, char **argv )
{
	GetTempPath( sizeof( g_WorkerTempPath ), g_WorkerTempPath );

	strcat( g_WorkerTempPath, "shadercompiletemp\\" );
	char tmp[MAX_PATH];
	sprintf( tmp, "rd /s /q \"%s\"", g_WorkerTempPath );
	system( tmp );
	_mkdir( g_WorkerTempPath );
//	printf( "g_WorkerTempPath: \"%s\"\n", g_WorkerTempPath );

	CommandLine()->CreateCmdLine( argc, argv );
	g_pShaderPath = CommandLine()->ParmValue( "-shaderpath", "" );

	g_bVerbose = CommandLine()->FindParm("-verbose") != 0;
}

void SetupDebugFile( void )
{
#ifdef DEBUGFP
	const char *pComputerName = getenv( "COMPUTERNAME" );
	char filename[MAX_PATH];
	sprintf( filename, "\\\\fileserver\\user\\gary\\debug\\%s.txt", pComputerName );
	g_WorkerDebugFp = fopen( filename, "w" );
	Assert( g_WorkerDebugFp );
	DebugOut( "opened debug file\n" );
#endif
}

void CompileShaders_NoVMPI()
{
	Worker_ProcessCommandRange_Singleton pcr;

	//
	// We will iterate on the cfg entries and process them
	//
	for ( CfgProcessor::CfgEntryInfo const *pEntry = g_arrCompileEntries.Get();
		  pEntry && pEntry->m_szName; ++ pEntry )
	{
		//
		// Stick the shader info
		//
		ShaderInfo_t siLastShaderInfo;
		memset( &siLastShaderInfo, 0, sizeof( siLastShaderInfo ) );

		Shader_ParseShaderInfoFromCompileCommands( pEntry, siLastShaderInfo );

		g_ShaderToShaderInfo[ pEntry->m_szName ] = siLastShaderInfo;

		//
		// Compile stuff
		//
		Worker_ProcessCommandRange( pEntry->m_iCommandStart, pEntry->m_iCommandEnd );

		//
		// Now when the whole shader is finished we can write it
		//
		char const *szShaderToWrite = pEntry->m_szName;
		g_numCommandsCompleted = g_numCompileCommands;
		WriteShaderFiles( szShaderToWrite );
		g_numCommandsCompleted = pEntry->m_iCommandEnd;
	}

	Msg( "\r                                                  \r" );
}


class CDistributeShaderCompileMaster : public IWorkUnitDistributorCallbacks
{
public:
	CDistributeShaderCompileMaster( void );
	~CDistributeShaderCompileMaster( void );

public:
	virtual void OnWorkUnitsCompleted( uint64 numWorkUnits );

private:
	void ThreadProc( void );
	friend DWORD WINAPI CDistributeShaderCompileMaster::ThreadProcAdapter( LPVOID pvArg );
	static DWORD WINAPI ThreadProcAdapter( LPVOID pvArg ) { reinterpret_cast< CDistributeShaderCompileMaster * >( pvArg )->ThreadProc(); return 0; }
	
private:
	HANDLE m_hThread;
	HANDLE m_hEvent;
	CThreadFastMutex m_mtx;
	BOOL m_bRunning;

private:
	CfgProcessor::CfgEntryInfo const *m_pAnalyzeShaders;
	CUtlVector< char const * > m_arrShaderNamesToWrite;
};

CDistributeShaderCompileMaster::CDistributeShaderCompileMaster( void ) :
	m_hThread( NULL ),
	m_hEvent( NULL ),
	m_bRunning( TRUE )
{
	m_hEvent = CreateEvent( NULL, FALSE, FALSE, NULL );
	m_hThread = CreateThread( NULL, 0, ThreadProcAdapter, reinterpret_cast< LPVOID >(this), 0, NULL );

	m_pAnalyzeShaders = g_arrCompileEntries.Get();
}

CDistributeShaderCompileMaster::~CDistributeShaderCompileMaster( void )
{
	m_bRunning = FALSE;
	
	SetEvent( m_hEvent );
	WaitForSingleObject( m_hThread, INFINITE );
	
	CloseHandle( m_hThread );
	CloseHandle( m_hEvent );
}

void CDistributeShaderCompileMaster::OnWorkUnitsCompleted( uint64 numWorkUnits )
{
	// Make sure that our mutex is in multi-threaded mode
	Threading::g_mtxGlobal.SetThreadedMode( Threading::eMultiThreaded );

	// Figure out how many commands have completed based on work units
	g_numCompletedStaticCombos = numWorkUnits * g_nStaticCombosPerWorkUnit;
	uint64 numStaticCombosOfTheEntry = 0;
	CfgProcessor::CfgEntryInfo const *pEntry = GetEntryByStaticComboNum( g_numCompletedStaticCombos, &numStaticCombosOfTheEntry );
	g_numCommandsCompleted = pEntry->m_iCommandStart + numStaticCombosOfTheEntry * pEntry->m_numDynamicCombos;

	// Iterate over the shaders yet to be written and see if we can queue them
	for ( ; m_pAnalyzeShaders->m_szName &&
		    m_pAnalyzeShaders->m_iCommandEnd <= g_numCommandsCompleted;
			++ m_pAnalyzeShaders
		)
	{
		m_mtx.Lock();
		m_arrShaderNamesToWrite.AddToTail( m_pAnalyzeShaders->m_szName );
		SetEvent( m_hEvent );
		m_mtx.Unlock();
	}
}

void CDistributeShaderCompileMaster::ThreadProc( void )
{
	for ( ; m_bRunning; )
	{
		WaitForSingleObject( m_hEvent, INFINITE );
		
		// Do a pump of shaders to write
		for ( int numShadersWritten = 0; /* forever */ ; ++ numShadersWritten )
		{
			m_mtx.Lock();
			char const * szShaderToWrite = NULL;
			if ( m_arrShaderNamesToWrite.Count() > numShadersWritten )
				szShaderToWrite = m_arrShaderNamesToWrite[ numShadersWritten ];
			else
				m_arrShaderNamesToWrite.RemoveAll();
			m_mtx.Unlock();

			if ( !szShaderToWrite )
				break;

			// We have the shader to write asynchronously
			WriteShaderFiles( szShaderToWrite );
		}
	}
}

int ShaderCompile_Main( int argc, char* argv[] )
{
	InstallSpewFunction();
	g_bSuppressPrintfOutput = false;
	g_flStartTime = Plat_FloatTime();

	SetupDebugFile();
	numthreads = 1; // managed specifically in Worker_ProcessCommandRange_Singleton::Startup

	/*
	Special section of code implementing "-subprocess" flag
	*/
	if ( int iSubprocess = CommandLine()->FindParm( "-subprocess" ) )
	{
		char const *szSubProcessData = CommandLine()->GetParm( 1 + iSubprocess );
		return ShaderCompile_Subprocess_Main( szSubProcessData );
	}

	// This needs to get called before VMPI is setup because in SDK mode, VMPI will change the args around.
	SetupExeDir( argc, argv );

	g_bIsX360 = CommandLine()->FindParm( "-x360" ) != 0;
	// g_bSuppressWarnings = g_bIsX360;

	bool bShouldUseVMPI = ( CommandLine()->FindParm( "-nompi" ) == 0 );
	if ( bShouldUseVMPI )
	{	
		// Master, start accepting connections.
		// Worker, make a connection.
		DebugOut( "Before VMPI_Init\n" );
		g_bSuppressPrintfOutput = true;
		VMPIRunMode mode = VMPI_RUN_NETWORKED;
		if ( !VMPI_Init( argc, argv, "dependency_info_shadercompile.txt", MyDisconnectHandler, mode ) )
		{
			g_bSuppressPrintfOutput = false;
			DebugOut( "MPI_Init failed.\n" );
			Error( "MPI_Init failed." );
		}

		extern void VMPI_SetWorkUnitsPartitionSize( int numWusToDeal );
		VMPI_SetWorkUnitsPartitionSize( 32 );
	}

	SetupPaths( argc, argv );

	g_bSuppressPrintfOutput = false;
	DebugOut( "After VMPI_Init\n" );

	// Setting up the minidump handlers
	if ( bShouldUseVMPI && !g_bMPIMaster )
		SetupToolsMinidumpHandler( VMPI_ExceptionFilter );
	else
		SetupDefaultToolsMinidumpHandler();

	if ( CommandLine()->FindParm( "-game" ) == 0 )
	{
		// Used with filesystem_stdio.dll
		FileSystem_Init( NULL, 0, FS_INIT_COMPATIBILITY_MODE );
	}
	else
	{
		// SDK uses this since it only has filesystem_steam.dll.
		FileSystem_Init( NULL, 0, FS_INIT_FULL );
	}
	
	DebugOut( "After VMPI_FileSystem_Init\n" );
	Shared_ParseListOfCompileCommands();
	DebugOut( "After Shared_ParseListOfCompileCommands\n" );

	if ( bShouldUseVMPI )
	{
		// Partition combos
		g_nStaticCombosPerWorkUnit = 0;
		if ( g_numStaticCombos )
		{
			if ( g_numStaticCombos <= 1024 )
				g_nStaticCombosPerWorkUnit = 1;
			else if ( g_numStaticCombos > 1024 * 10 )
				g_nStaticCombosPerWorkUnit = 10;
			else
				g_nStaticCombosPerWorkUnit = g_numStaticCombos / 1024;
		}

		uint64 nWorkUnits;
		if( g_nStaticCombosPerWorkUnit == 0 )
		{
			nWorkUnits = 1;
			g_nStaticCombosPerWorkUnit = g_numStaticCombos;
		}
		else
		{
			nWorkUnits = g_numStaticCombos / g_nStaticCombosPerWorkUnit + 1;
		}

		DebugOut( "Before conditional\n" );
		if ( g_bMPIMaster )
		{
			// Send all of the workers the complete list of work to do.
			DebugOut( "Before STARTWORK_PACKETID\n" );

			char packetID = STARTWORK_PACKETID;
			VMPI_SendData( &packetID, sizeof( packetID ), VMPI_PERSISTENT );

			// Compile master distribution tracker
			CDistributeShaderCompileMaster dscm;
			g_pDistributeWorkCallbacks = &dscm;

			{
				char chCommands[50], chStaticCombos[50], chNumWorkUnits[50];
				sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
				sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
				sprintf( chNumWorkUnits, "%s", PrettyPrintNumber( nWorkUnits ) );
				Msg( "\rCompiling %s commands in %s work units.\n", chCommands, chNumWorkUnits );
			}

			// nWorkUnits is how many work units. . .1000 is good.
			// The work unit number impies which combo to do.
			DebugOut( "Before DistributeWork\n" );
			DistributeWork( nWorkUnits, WORKUNIT_PACKETID, NULL, Master_ReceiveWorkUnitFn );

			g_pDistributeWorkCallbacks = NULL;
		}
		else
		{
			// wait until we get a packet from the master to start doing stuff.
			MessageBuffer buf;
			DebugOut( "Before VMPI_DispatchUntil\n" );
			while ( !g_bGotStartWorkPacket )
			{
				VMPI_DispatchNextMessage();
			}
			DebugOut( "after VMPI_DispatchUntil\n" );

			DebugOut( "Before Worker_GetLocalCopyOfShaders\n" );
			Worker_GetLocalCopyOfShaders();
			DebugOut( "Before Worker_GetLocalCopyOfBinaries\n" );
			Worker_GetLocalCopyOfBinaries();

			DebugOut( "Before _chdir\n" );
			_chdir( g_WorkerTempPath );

			// nWorkUnits is how many work units. . .1000 is good.
			// The work unit number impies which combo to do.
			DebugOut( "Before DistributeWork\n" );

			// Allows calling into ProcessCommandRange inside the worker function
			{
				Worker_ProcessCommandRange_Singleton pcr;
				DistributeWork( nWorkUnits, WORKUNIT_PACKETID, Worker_ProcessWorkUnitFn, NULL );
			}
		}

		g_bSuppressPrintfOutput = true;
		g_bSuppressPrintfOutput = false;
	}
	else // no VMPI
	{
		Worker_GetLocalCopyOfShaders();
		Worker_GetLocalCopyOfBinaries();
		_chdir( g_WorkerTempPath );

		{
			char chCommands[50], chStaticCombos[50];
			sprintf( chCommands, "%s", PrettyPrintNumber( g_numCompileCommands ) );
			sprintf( chStaticCombos, "%s", PrettyPrintNumber( g_numStaticCombos ) );
			Msg( "\rCompiling %s commands in %s static combos.\n", chCommands, chStaticCombos );
		}
		CompileShaders_NoVMPI();
	}

	Msg( "\r                                                                \r" );
	if ( g_bMPIMaster || !bShouldUseVMPI )
	{
		char str[ 4096 ];

		// Write everything that succeeded
		int nStrings = g_ShaderByteCode.GetNumStrings();
		for( int i = 0; i < nStrings; i++ )
		{
			WriteShaderFiles( g_ShaderByteCode.String(i) );
		}

		// Write all the errors
		//////////////////////////////////////////////////////////////////////////
		//
		// Now deliver all our accumulated spew to the output
		//
		//////////////////////////////////////////////////////////////////////////

		bool bValveVerboseComboErrors = ( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) &&
			atoi( getenv( "VALVE_VERBOSE_COMBO_ERRORS" ) ) ) ? true : false;

		// Compiler spew
		for ( int k = 0, kEnd = g_Master_CompilerMsgInfo.GetNumStrings(); k < kEnd; ++ k )
		{
			char const * const szMsg = g_Master_CompilerMsgInfo.String( k );
			CompilerMsgInfo const &cmi = g_Master_CompilerMsgInfo[ int_as_symid( k ) ];

			char const * const szFirstCmd = cmi.GetFirstCommand();
			int const numReported = cmi.GetNumTimesReported();

			uint64 iFirstCommand = _strtoui64( szFirstCmd, NULL, 10 );
			CfgProcessor::ComboHandle hCombo = NULL;
			CfgProcessor::CfgEntryInfo const *pComboEntryInfo = NULL;
			if ( CfgProcessor::Combo_GetNext( iFirstCommand, hCombo, g_numCompileCommands ) )
			{
				Combo_FormatCommand( hCombo, str );
				pComboEntryInfo = Combo_GetEntryInfo( hCombo );
				Combo_Free( hCombo );
			}
			else
			{
				sprintf( str, "cmd # %s", szFirstCmd );
			}


			Msg( "\n%s\n", szMsg );
			Msg( "    Reported %d time(s), example command:\n", numReported);

			if ( bValveVerboseComboErrors )
			{
				Msg( "    Verbose Description:\n" );
				if ( pComboEntryInfo )
				{
					Msg( "        Src File: %s\n", pComboEntryInfo->m_szShaderFileName );
					Msg( "        Tgt File: %s\n", pComboEntryInfo->m_szName );
				}

				// Between     /DSHADERCOMBO=   and    /Dmain
				char const *pBegin = strstr( str, "/DSHADERCOMBO=" );
				char const *pEnd = strstr( str, "/Dmain" );
				if ( pBegin )
				{
					pBegin += strlen( "/DSHADERCOMBO=" ) ;
					char const *pSpace = strchr( pBegin, ' ' );
					if ( pSpace )
						Msg( "        Combo # : %.*s\n", ( pSpace - pBegin ), pBegin );
				}

				if ( !pEnd )
					pEnd = str + strlen( str );
				while ( pBegin && *pBegin && !V_isspace( *pBegin ) )
					++ pBegin;
				while ( pBegin && *pBegin && V_isspace( *pBegin ) )
					++ pBegin;

				// Now parse all combo defines in [pBegin, pEnd]
				while ( pBegin && *pBegin && ( pBegin < pEnd ) )
				{
					char const *pDefine = strstr( pBegin, "/D" );
					if ( !pDefine || pDefine >= pEnd )
						break;

					char const *pEqSign = strchr( pDefine, '=' );
					if ( !pEqSign || pEqSign >= pEnd )
						break;

					char const *pSpace = strchr( pEqSign, ' ' );
					if ( !pSpace || pSpace >= pEnd )
						pSpace = pEnd;

					pBegin = pSpace;

					Msg( "                  %.*s %.*s\n",
						( pSpace - pEqSign - 1 ), pEqSign + 1,
						( pEqSign - pDefine - 2 ), pDefine + 2 );
				}
			}
			Msg( "    %s\n", str );
		}

		// Failed shaders summary
		for ( int k = 0, kEnd = g_Master_ShaderHadError.GetNumStrings(); k < kEnd; ++ k )
		{
			char const *szShaderName = g_Master_ShaderHadError.String( k );
			if ( !g_Master_ShaderHadError[ int_as_symid( k ) ] )
				continue;

			Msg( "FAILED:    %s\n", szShaderName );
		}

		//
		// End
		//
		double end = Plat_FloatTime();
		
		GetHourMinuteSecondsString( (int)( end - g_flStartTime ), str, sizeof( str ) );
		DebugOut( "%s elapsed\n", str );
		DebugOut( "Precise timing = %.5f\n", ( end - g_flStartTime ) );

		if ( bShouldUseVMPI )
		{
			VMPI_FileSystem_Term();
			DebugOut( "Before VMPI_Finalize\n" );
			VMPI_Finalize();
		}
	}
	
	return g_Master_ShaderHadError.GetNumStrings();
}

class CShaderCompileDLL : public IShaderCompileDLL
{
	int main( int argc, char **argv );
};

int CShaderCompileDLL::main( int argc, char **argv )
{
	return ShaderCompile_Main( argc, argv );
}

EXPOSE_SINGLE_INTERFACE( CShaderCompileDLL, IShaderCompileDLL, SHADER_COMPILE_INTERFACE_VERSION );


class CLaunchableDLL : public ILaunchableDLL
{
	int main( int argc, char **argv )
	{
		return ShaderCompile_Main( argc, argv );
	}
};

EXPOSE_SINGLE_INTERFACE( CLaunchableDLL, ILaunchableDLL, LAUNCHABLE_DLL_INTERFACE_VERSION );