|
|
//===== Copyright � 1996-2005, Valve Corporation, All rights reserved. ======//
//
// Purpose:
//
//===========================================================================//
#include <windows.h>
#include "vmpi.h"
#include "cmdlib.h"
#include "vmpi_tools_shared.h"
#include "tier1/strtools.h"
#include "mpi_stats.h"
#include "iphelpers.h"
#include "tier0/minidump.h"
// ----------------------------------------------------------------------------- //
// Globals.
// ----------------------------------------------------------------------------- //
static bool g_bReceivedDirectoryInfo = false; // Have we gotten the qdir info yet?
static bool g_bReceivedDBInfo = false; static CDBInfo g_DBInfo; static unsigned long g_JobPrimaryID;
static int g_nDisconnects = 0; // Tracks how many remote processes have disconnected ungracefully.
// ----------------------------------------------------------------------------- //
// Shared dispatch code.
// ----------------------------------------------------------------------------- //
bool SharedDispatch( MessageBuffer *pBuf, int iSource, int iPacketID ) { char *pInPos = &pBuf->data[2];
switch ( pBuf->data[1] ) { case VMPI_SUBPACKETID_DIRECTORIES: { Q_strncpy( gamedir, pInPos, sizeof( gamedir ) ); pInPos += strlen( pInPos ) + 1;
Q_strncpy( qdir, pInPos, sizeof( qdir ) ); g_bReceivedDirectoryInfo = true; } return true;
case VMPI_SUBPACKETID_DBINFO: { g_DBInfo = *((CDBInfo*)pInPos); pInPos += sizeof( CDBInfo ); g_JobPrimaryID = *((unsigned long*)pInPos);
g_bReceivedDBInfo = true; } return true;
case VMPI_SUBPACKETID_CRASH: { char const chCrashInfoType = *pInPos; pInPos += 2; switch ( chCrashInfoType ) { case 't': Warning( "\nWorker '%s' dead: %s\n", VMPI_GetMachineName( iSource ), pInPos ); break; case 'f': { int iFileSize = * reinterpret_cast< int const * >( pInPos ); pInPos += sizeof( iFileSize );
// Temp folder
char const *szFolder = NULL; if ( !szFolder ) szFolder = getenv( "TEMP" ); if ( !szFolder ) szFolder = getenv( "TMP" ); if ( !szFolder ) szFolder = "c:";
// Base module name
char chModuleName[_MAX_PATH], *pModuleName = chModuleName; ::GetModuleFileName( NULL, chModuleName, sizeof( chModuleName ) / sizeof( chModuleName[0] ) );
if ( char *pch = strrchr( chModuleName, '.' ) ) *pch = 0; if ( char *pch = strrchr( chModuleName, '\\' ) ) *pch = 0, pModuleName = pch + 1;
// Current time
struct tm curTime; Plat_GetLocalTime( &curTime );
// Number of minidumps this run
static int s_numMiniDumps = 0; ++ s_numMiniDumps;
// Prepare the filename
char chSaveFileName[ 2 * _MAX_PATH ] = { 0 }; sprintf( chSaveFileName, "%s\\vmpi_%s_on_%s_%d%.2d%2d%.2d%.2d%.2d_%d.mdmp", szFolder, pModuleName, VMPI_GetMachineName( iSource ), curTime.tm_year + 1900, /* Year less 2000 */ curTime.tm_mon + 1, /* month (0 - 11 : 0 = January) */ curTime.tm_mday, /* day of month (1 - 31) */ curTime.tm_hour, /* hour (0 - 23) */ curTime.tm_min, /* minutes (0 - 59) */ curTime.tm_sec, /* seconds (0 - 59) */ s_numMiniDumps );
if ( FILE *fDump = fopen( chSaveFileName, "wb" ) ) { fwrite( pInPos, 1, iFileSize, fDump ); fclose( fDump );
Warning( "\nSaved worker crash minidump '%s', size %d byte(s).\n", chSaveFileName, iFileSize ); } else { Warning( "\nReceived worker crash minidump size %d byte(s), failed to save.\n", iFileSize ); } } break; } } return true; }
return false; }
CDispatchReg g_SharedDispatchReg( VMPI_SHARED_PACKET_ID, SharedDispatch );
VMPI_REGISTER_PACKET_ID( VMPI_SHARED_PACKET_ID ); VMPI_REGISTER_SUBPACKET_ID( VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DIRECTORIES ); VMPI_REGISTER_SUBPACKET_ID( VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DBINFO ); VMPI_REGISTER_SUBPACKET_ID( VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_CRASH ); VMPI_REGISTER_SUBPACKET_ID( VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_MULTICAST_ADDR );
// ----------------------------------------------------------------------------- //
// Module interfaces.
// ----------------------------------------------------------------------------- //
void SendQDirInfo() { char cPacketID[2] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DIRECTORIES };
MessageBuffer mb; mb.write( cPacketID, 2 ); mb.write( gamedir, strlen( gamedir ) + 1 ); mb.write( qdir, strlen( qdir ) + 1 );
VMPI_SendData( mb.data, mb.getLen(), VMPI_PERSISTENT ); }
void RecvQDirInfo() { while ( !g_bReceivedDirectoryInfo ) VMPI_DispatchNextMessage(); }
void SendDBInfo( const CDBInfo *pInfo, unsigned long jobPrimaryID ) { char cPacketInfo[2] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_DBINFO }; const void *pChunks[] = { cPacketInfo, pInfo, &jobPrimaryID }; int chunkLengths[] = { 2, sizeof( CDBInfo ), sizeof( jobPrimaryID ) }; VMPI_SendChunks( pChunks, chunkLengths, ARRAYSIZE( pChunks ), VMPI_PERSISTENT ); }
void RecvDBInfo( CDBInfo *pInfo, unsigned long *pJobPrimaryID ) { while ( !g_bReceivedDBInfo ) VMPI_DispatchNextMessage();
*pInfo = g_DBInfo; *pJobPrimaryID = g_JobPrimaryID; }
// If the file is successfully opened, read and sent returns the size of the file in bytes
// otherwise returns 0 and nothing is sent
int VMPI_SendFileChunk( const void *pvChunkPrefix, int lenPrefix, tchar const *ptchFileName ) { HANDLE hFile = NULL; HANDLE hMapping = NULL; void const *pvMappedData = NULL; int iResult = 0;
hFile = ::CreateFile( ptchFileName, GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL ); if ( !hFile || ( hFile == INVALID_HANDLE_VALUE ) ) goto done;
hMapping = ::CreateFileMapping( hFile, NULL, PAGE_READONLY, 0, 0, NULL ); if ( !hMapping || ( hMapping == INVALID_HANDLE_VALUE ) ) goto done;
pvMappedData = ::MapViewOfFile( hMapping, FILE_MAP_READ, 0, 0, 0 ); if ( !pvMappedData ) goto done;
int iMappedFileSize = ::GetFileSize( hFile, NULL ); if ( INVALID_FILE_SIZE == iMappedFileSize ) goto done;
// Send the data over VMPI
if ( VMPI_Send3Chunks( pvChunkPrefix, lenPrefix, &iMappedFileSize, sizeof( iMappedFileSize ), pvMappedData, iMappedFileSize, VMPI_MASTER_ID ) ) iResult = iMappedFileSize;
// Fall-through for cleanup code to execute
done: if ( pvMappedData ) ::UnmapViewOfFile( pvMappedData );
if ( hMapping && ( hMapping != INVALID_HANDLE_VALUE ) ) ::CloseHandle( hMapping );
if ( hFile && ( hFile != INVALID_HANDLE_VALUE ) ) ::CloseHandle( hFile );
return iResult; }
void VMPI_HandleCrash( const char *pMessage, uint uCode, void *pvExceptionInfo, bool bAssert ) { static LONG crashHandlerCount = 0; if ( InterlockedIncrement( &crashHandlerCount ) == 1 ) { Msg( "\nFAILURE: '%s' (assert: %d)\n", pMessage, bAssert );
// Send a message to the master.
char crashMsg[4] = { VMPI_SHARED_PACKET_ID, VMPI_SUBPACKETID_CRASH, 't', ':' };
VMPI_Send2Chunks( crashMsg, sizeof( crashMsg ), pMessage, strlen( pMessage ) + 1, VMPI_MASTER_ID );
// Now attempt to create a minidump with the given exception information
if ( pvExceptionInfo ) { tchar tchMinidumpFileName[_MAX_PATH] = { 0 }; bool bSucceededWritingMinidump = WriteMiniDumpUsingExceptionInfo( uCode, (ExceptionInfo_t*)pvExceptionInfo, MINIDUMP_WithDataSegs | MINIDUMP_WithIndirectlyReferencedMemory | MINIDUMP_WithProcessThreadData, // MINIDUMP_WithDataSegs | MINIDUMP_WithFullMemory | MINIDUMP_WithHandleData | MINIDUMP_WithUnloadedModules | MINIDUMP_WithIndirectlyReferencedMemory | MINIDUMP_WithProcessThreadData | MINIDUMP_WithPrivateReadWriteMemory,
// MINIDUMP_Normal,
tchMinidumpFileName ); if ( bSucceededWritingMinidump ) { crashMsg[2] = 'f'; VMPI_SendFileChunk( crashMsg, sizeof( crashMsg ), tchMinidumpFileName ); ::DeleteFile( tchMinidumpFileName ); } }
// Let the messages go out.
Sleep( 500 ); }
InterlockedDecrement( &crashHandlerCount ); }
// This is called if we crash inside our crash handler. It just terminates the process immediately.
LONG __stdcall VMPI_SecondExceptionFilter( struct _EXCEPTION_POINTERS *ExceptionInfo ) { TerminateProcess( GetCurrentProcess(), 2 ); return EXCEPTION_EXECUTE_HANDLER; // (never gets here anyway)
}
void VMPI_ExceptionFilter( unsigned long uCode, void *pvExceptionInfo ) { // This is called if we crash inside our crash handler. It just terminates the process immediately.
SetUnhandledExceptionFilter( VMPI_SecondExceptionFilter );
//DWORD code = ExceptionInfo->ExceptionRecord->ExceptionCode;
#define ERR_RECORD( name ) { name, #name }
struct { int code; char *pReason; } errors[] = { ERR_RECORD( EXCEPTION_ACCESS_VIOLATION ), ERR_RECORD( EXCEPTION_ARRAY_BOUNDS_EXCEEDED ), ERR_RECORD( EXCEPTION_BREAKPOINT ), ERR_RECORD( EXCEPTION_DATATYPE_MISALIGNMENT ), ERR_RECORD( EXCEPTION_FLT_DENORMAL_OPERAND ), ERR_RECORD( EXCEPTION_FLT_DIVIDE_BY_ZERO ), ERR_RECORD( EXCEPTION_FLT_INEXACT_RESULT ), ERR_RECORD( EXCEPTION_FLT_INVALID_OPERATION ), ERR_RECORD( EXCEPTION_FLT_OVERFLOW ), ERR_RECORD( EXCEPTION_FLT_STACK_CHECK ), ERR_RECORD( EXCEPTION_FLT_UNDERFLOW ), ERR_RECORD( EXCEPTION_ILLEGAL_INSTRUCTION ), ERR_RECORD( EXCEPTION_IN_PAGE_ERROR ), ERR_RECORD( EXCEPTION_INT_DIVIDE_BY_ZERO ), ERR_RECORD( EXCEPTION_INT_OVERFLOW ), ERR_RECORD( EXCEPTION_INVALID_DISPOSITION ), ERR_RECORD( EXCEPTION_NONCONTINUABLE_EXCEPTION ), ERR_RECORD( EXCEPTION_PRIV_INSTRUCTION ), ERR_RECORD( EXCEPTION_SINGLE_STEP ), ERR_RECORD( EXCEPTION_STACK_OVERFLOW ), ERR_RECORD( EXCEPTION_ACCESS_VIOLATION ), };
int nErrors = sizeof( errors ) / sizeof( errors[0] ); int i=0; char *pchReason = NULL; char chUnknownBuffer[32]; for ( i; ( i < nErrors ) && !pchReason; i++ ) { if ( errors[i].code == uCode ) pchReason = errors[i].pReason; }
if ( i == nErrors ) { sprintf( chUnknownBuffer, "Error code 0x%08X", uCode ); pchReason = chUnknownBuffer; } VMPI_HandleCrash( pchReason, uCode, pvExceptionInfo, true );
TerminateProcess( GetCurrentProcess(), 1 ); }
void HandleMPIDisconnect( int procID, const char *pReason ) { int nLiveWorkers = VMPI_GetCurrentNumberOfConnections() - g_nDisconnects - 1;
// We ran into the size limit before and it wasn't readily apparent that the size limit had
// been breached, so make sure to show errors about invalid packet sizes..
bool bOldSuppress = g_bSuppressPrintfOutput; g_bSuppressPrintfOutput = ( Q_stristr( pReason, "invalid packet size" ) == 0 );
Warning( "\n\n--- WARNING: lost connection to '%s' (%s).\n", VMPI_GetMachineName( procID ), pReason ); if ( g_bMPIMaster ) { Warning( "%d workers remain.\n\n", nLiveWorkers );
++g_nDisconnects; /*
if ( VMPI_GetCurrentNumberOfConnections() - g_nDisconnects <= 1 ) { Error( "All machines disconnected!" ); } */ } else { VMPI_HandleAutoRestart(); Error( "Worker quitting." ); } g_bSuppressPrintfOutput = bOldSuppress; }
|