archive
/
csgo
mirror of https://github.com/sr2echa/CSGO-Source-Code
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
Counter Strike : Global Offensive Source Code
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
13 Commits
1 Branch
0 Tags
504 MiB
 
 
 
 
 
 
Branch: master
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'master'
${ noResults }
csgo/cstrike15_src/devtools/memlog/memlog.cpp

2517 lines
79 KiB
Raw Permalink Blame History

// memlog.cpp : mines memory data from vxconsole logs (see game/client/c_memorylog.cpp)
#include "utlbuffer.h"
#include <conio.h>
#include <math.h>
#include <iostream>
#include <tchar.h>
#include <assert.h>
#include <fstream>
#include <sstream>
#include <vector>
#include <map>
#define _SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS
#include <hash_map>
#include <algorithm>
#include "windows.h"
using namespace std;
using namespace stdext;
// We write this to output files on conversion, and test it on read to see if files need re-converting:
static const int MEMLOG_VERSION = 4;
enum DVDHosted_t
{
// Is the game running FULLY off the DVD? (i.e. it reads no data from the HDD - no texture streaming)
// If so, it will use more memory for audio/anim/texture data.
DVDHOSTED_UNKNOWN = 0,
DVDHOSTED_YES,
DVDHOSTED_NO
};
enum Platform_t
{
PLATFORM_UNKNOWN = 0,
PLATFORM_360 = 1,
PLATFORM_PS3 = 2,
PLATFORM_PC = 3
};
const char *gPlatformNames[] = { "unknown", "360", "PS3", "PC" };
enum TrackStat_t
{
TRACKSTAT_UNKNOWN = 0,
TRACKSTAT_MINFREE_CPU = 1,
TRACKSTAT_MINFREE_GPU = 2
};
// TODO: these will eventually vary by platform (certainly, these numbers are not useful on PC)
#define CPU_MEM_SIZE 512
#define GPU_MEM_SIZE 256
static const int MAX_LANG_LEN = 64;
struct CHeaderInfo
{
CHeaderInfo() : memlogVersion( 0 ), dvdHosted( DVDHOSTED_UNKNOWN ), buildNumber(-1), platform( PLATFORM_UNKNOWN ) { language[0] = 0; }
int memlogVersion;
DVDHosted_t dvdHosted;
int buildNumber;
Platform_t platform;
char language[MAX_LANG_LEN];
};
// Bitfield type used to filter logs w.r.t subsets of maps/players
class FilterBitfield
{
public:
FilterBitfield( void ) { for ( int i = 0; i < NUM_INT64S; i++ ) bits[i] = 0; }
void SetAll( void ) { for ( int i = 0; i < NUM_INT64S; i++ ) bits[i] = -1; }
void Set( int bit ) { bits[bit/64] |= ((__int64)1) << (bit&63); }
bool IsSet( int bit ) { return !!( bits[bit/64] & ((__int64)1) << (bit&63) ); }
bool Intersects( const FilterBitfield &other ) { __int64 result = 0; for ( int i = 0; i < NUM_INT64S; i++ ) result |= ( bits[i] & other.bits[i] ); return !!result; }
bool operator ! ( void ) const { __int64 result = 0; for ( int i = 0; i < NUM_INT64S; i++ ) result |= bits[i]; return !result; }
static const int NUM_INT64S = 4;
__int64 bits[NUM_INT64S];
};
class CStringList
{
public:
static const int MAX_STRINGLIST_ENTRIES = 64*FilterBitfield::NUM_INT64S;
~CStringList( void )
{
for ( int i = 0; i < m_Strings.Count(); i++ ) delete m_Strings[ i ];
}
int AddString( const char *string )
{
int existingIndex = StringToInt( string );
if ( existingIndex != -1 )
return existingIndex;
if ( m_Strings.Count() == MAX_STRINGLIST_ENTRIES )
{
Assert( 0 );
printf( "----ERROR: more than %d strings added to stringlist (e.g. '%s'), have to increase the size of FilterBitfield.bits!\n\n", MAX_STRINGLIST_ENTRIES, m_Strings[0] );
return -1;
}
return m_Strings.AddToTail( strdup( string ) );
}
int StringToInt( const char *string )
{
// TODO: make this search faster (BUT we still need to be able to index into m_Strings and FilterBitfields... storing CUtlSymbolTable indices in m_String could work)
for ( int i = 0; i < m_Strings.Count(); i++ )
{
if ( !stricmp( string, m_Strings[ i ] ) )
return i;
}
return -1;
}
FilterBitfield SubstringToBitfield( const char *subString )
{
FilterBitfield bitField;
if ( !subString || !subString[0] )
{
bitField.SetAll();
}
else
{
for ( int i = 0; i < m_Strings.Count(); i++ )
{
if ( V_stristr( m_Strings[ i ], subString ) )
bitField.Set( i );
}
}
return bitField;
}
private:
CUtlVector< const char *>m_Strings;
};
struct MapMin_t // See CLogFile::badMaps
{
int map; // Map number
float minMem; // Minimum memory observed in this map
};
struct CItem
{
int time;
float freeMem;
float gpuFree;
int globalMapIndex;
int logMapIndex;
int numBots;
int numPlayers;
int numLocalPlayers;
int numSpecators;
bool isServer;
FilterBitfield playerBitfield; // Bits represent items in CLogFile::playerList
};
struct CLogFile
{
public:
CLogFile( void ) {}
CUtlVector<CItem> entries;
string memoryLog; // Path of the source memorylog file
string consoleLog; // Path of the source vxconsole file
ULONGLONG modifyTime; // Last modification time of the source vxconsole file
bool isActive; // Last time we checked, the vxconsole file was still being written
float minFreeMem; // Minimum free memory in any entry in this log
float minGPUFreeMem; // Minimum free GPU memory in any entry in this log
bool isServer; // True if the local box is the Server for any entry in this log
bool isSplitscreen; // True if the local box is running splitscreen (>= 2 local players) for any entry in this log
int maxPlayers; // Maximum players in any entry in this log
CStringList mapList; // The maps present in this log
CStringList playerList; // The players present in this log
float memorylogTick; // Seconds between entries in this log
CHeaderInfo headerInfo; // Various global data about the log (stuff that can't be computed from 'entries')
CUtlVector<MapMin_t>badMaps; // A temp list that accumulates maps in this log which pass the current filters
private:
CLogFile( const CLogFile &other ) {}
};
struct CLogStats
{
CLogStats( void )
{
for ( int i = 0; i < CStringList::MAX_STRINGLIST_ENTRIES; i++ )
{
mapMin[ i ] = CPU_MEM_SIZE;
mapGPUMin[ i ] = GPU_MEM_SIZE;
mapAverage[ i ] = 0.0f;
mapSeconds[ i ] = 0;
mapSamples[ i ] = 0;
}
}
float mapMin[ CStringList::MAX_STRINGLIST_ENTRIES ];
float mapGPUMin[ CStringList::MAX_STRINGLIST_ENTRIES ];
float mapAverage[ CStringList::MAX_STRINGLIST_ENTRIES ];
float mapSeconds[ CStringList::MAX_STRINGLIST_ENTRIES ];
unsigned int mapSamples[ CStringList::MAX_STRINGLIST_ENTRIES ];
};
static const char *gKnownMaps[] = { "none",
/*"credits", */
/* L4D1
"l4d_hospital01_apartment",
"l4d_hospital02_subway",
"l4d_hospital03_sewers",
"l4d_hospital04_interior",
"l4d_hospital05_rooftop",
"l4d_airport01_greenhouse",
"l4d_airport02_offices",
"l4d_airport03_garage",
"l4d_airport04_terminal",
"l4d_airport05_runway",
"l4d_farm01_hilltop",
"l4d_farm02_traintunnel",
"l4d_farm03_bridge",
"l4d_farm04_barn",
"l4d_farm05_cornfield",
"l4d_smalltown01_caves",
"l4d_smalltown02_drainage",
"l4d_smalltown03_ranchhouse",
"l4d_smalltown04_mainstreet",
"l4d_smalltown05_houseboat",
"l4d_vs_hospital01_apartment",
"l4d_vs_hospital02_subway",
"l4d_vs_hospital03_sewers",
"l4d_vs_hospital04_interior",
"l4d_vs_hospital05_rooftop",
"l4d_vs_airport01_greenhouse",
"l4d_vs_airport02_offices",
"l4d_vs_airport03_garage",
"l4d_vs_airport04_terminal",
"l4d_vs_airport05_runway",
"l4d_vs_farm01_hilltop",
"l4d_vs_farm02_traintunnel",
"l4d_vs_farm03_bridge",
"l4d_vs_farm04_barn",
"l4d_vs_farm05_cornfield",
"l4d_vs_smalltown01_caves",
"l4d_vs_smalltown02_drainage",
"l4d_vs_smalltown03_ranchhouse",
"l4d_vs_smalltown04_mainstreet",
"l4d_vs_smalltown05_houseboat",
"backgroundstreet", */
/* L4D2
"c1m1_hotel",
"c1m2_streets",
"c1m3_mall",
"c1m4_atrium",
"c2m1_highway",
"c2m2_fairgrounds",
"c2m3_coaster",
"c2m4_barns",
"c2m5_concert",
"c3m1_plankcountry",
"c3m2_swamp",
"c3m3_shantytown",
"c3m4_plantation",
"c4m1_milltown_a",
"c4m2_sugarmill_a",
"c4m3_sugarmill_b",
"c4m4_milltown_b",
"c4m5_milltown_escape",
"c5m1_waterfront",
"c5m2_park",
"c5m3_cemetery",
"c5m4_quarter",
"c5m5_bridge", */
/* // Portal 2 SP
"sp_a1_intro1",
"sp_a1_intro2",
"sp_a1_intro3",
"sp_a1_intro4",
"sp_a1_intro5",
"sp_a1_intro6",
"sp_a1_intro7",
"sp_a1_wakeup",
"sp_a2_intro",
"sp_a2_laser_intro",
"sp_a2_laser_stairs",
"sp_a2_dual_lasers",
"sp_a2_laser_over_goo",
"sp_a2_catapult_intro",
"sp_a2_trust_fling",
"sp_a2_pit_flings",
"sp_a2_fizzler_intro",
"sp_a2_sphere_peek",
"sp_a2_ricochet",
"sp_a2_bridge_intro",
"sp_a2_bridge_the_gap",
"sp_a2_turret_intro",
"sp_a2_laser_relays",
"sp_a2_turret_blocker",
"sp_a2_laser_vs_turret",
"sp_a2_pull_the_rug",
"sp_a2_column_blocker",
"sp_a2_laser_chaining",
"sp_a2_turret_tower",
"sp_a2_triple_laser",
"sp_a2_bts1",
"sp_a2_bts2",
"sp_a2_bts3",
"sp_a2_bts4",
"sp_a2_bts5",
"sp_a2_bts6",
"sp_a2_core",
"sp_a3_00",
"sp_a3_01",
"sp_a3_03",
"sp_a3_jump_intro",
"sp_a3_bomb_flings",
"sp_a3_crazy_box",
"sp_a3_transition01",
"sp_a3_speed_ramp",
"sp_a3_speed_flings",
"sp_a3_portal_intro",
"sp_a3_end",
"sp_a4_intro",
"sp_a4_tb_intro",
"sp_a4_tb_trust_drop",
"sp_a4_tb_wall_button",
"sp_a4_tb_polarity",
"sp_a4_tb_catch",
"sp_a4_stop_the_box",
"sp_a4_laser_catapult",
"sp_a4_laser_platform",
"sp_a4_speed_tb_catch",
"sp_a4_jump_polarity",
"sp_a4_finale1",
"sp_a4_finale2",
"sp_a4_finale3",
"sp_a4_finale4",
// Portal 2 Co-op
"mp_coop_start",
"mp_coop_lobby_2",
"mp_coop_doors",
"mp_coop_race_2",
"mp_coop_laser_2",
"mp_coop_rat_maze",
"mp_coop_laser_crusher",
"mp_coop_teambts",
"mp_coop_fling_3",
"mp_coop_infinifling_train",
"mp_coop_come_along",
"mp_coop_fling_1",
"mp_coop_catapult_1",
"mp_coop_multifling_1",
"mp_coop_fling_crushers",
"mp_coop_fan",
"mp_coop_wall_intro",
"mp_coop_wall_2",
"mp_coop_catapult_wall_intro",
"mp_coop_wall_block",
"mp_coop_catapult_2",
"mp_coop_turret_walls",
"mp_coop_turret_ball",
"mp_coop_wall_5",
"mp_coop_tbeam_redirect",
"mp_coop_tbeam_drill",
"mp_coop_tbeam_catch_grind_1",
"mp_coop_tbeam_laser_1",
"mp_coop_tbeam_polarity",
"mp_coop_tbeam_polarity2",
"mp_coop_tbeam_polarity3",
"mp_coop_tbeam_maze",
"mp_coop_tbeam_end",
"mp_coop_paint_come_along",
"mp_coop_paint_redirect",
"mp_coop_paint_bridge",
"mp_coop_paint_walljumps",
"mp_coop_paint_speed_fling",
"mp_coop_paint_red_racer",
"mp_coop_paint_speed_catch",
"mp_coop_paint_longjump_intro",
"mp_coop_separation_1",
"mp_coop_rocket_block",
"mp_coop_race_3",
"mp_coop_laser_1",
"mp_coop_wall_1",
"mp_coop_2guns_longjump_intro",
"mp_coop_paint_crazy_box",
"mp_coop_wall_6",
"mp_coop_button_tower",
"mp_coop_tbeam_fling_float_1",
// Portal 2 demo
"demo_intro",
"demo_paint",
"demo_underground", */
// CSGO
"cs_italy",
"cs_office",
"de_aztec",
"de_dust2",
"de_dust",
"de_inferno",
"de_nuke",
"de_lake",
"de_safehouse",
"de_shorttrain"
"de_sugarcane",
"de_stmarc",
"de_bank",
"ar_shoots",
"ar_baggage",
"de_train",
"training1",
};
const int gNumKnownMaps = ARRAYSIZE( gKnownMaps );
static const char *gIgnoreMaps[] = {"devtest",
/*"test_box",
"test_box2",
"nav_test",
"transition_test01",
"transition_test02",
"c2m4_concert",
"c5m2_cemetery",
"c5m3_quarter",
"c5m4_bridge"*/ };
const int gNumIgnoreMaps = ARRAYSIZE( gIgnoreMaps );
CUtlVector< const char * > gMapNames; // List of encountered map names
typedef hash_map<string, int> CMapHash; // For quickly determining is a string is in gMapNames
CMapHash gMapHash;
typedef hash_map<string, CLogFile*> CLogFiles;
const int FILTER_SIZE = 64;
struct Config
{
char sourcePath[ _MAX_PATH ]; // Where we're getting logs from
char prevCommandLine[ _MAX_PATH ]; // Previously entered command
bool recurse; // Recurse the tree under 'sourcePath'
bool update; // Re-convert vxconsole logs, if they're newer than their corresponding memorylogs
bool updateActive; // Re-convert vxconsole logs,
bool forceUpdate; // Re-convert vxconsole logs, even if they've been converted before
// Console mode:
bool consoleMode; // Accept commands from the user 'till they quit
bool load; // Load new logs from 'sourcePath'
bool unload; // Unload all logs from 'sourcePath'
bool unloadAll; // Unload all logs
bool quitting; // We're done, exit the app
bool help; // User wants to see the help text
// Memory tracking (CSV files to plot memory stats over time)
char trackFile[MAX_PATH]; // Tracking file to update from the current filter set
char trackColumn[32]; // New column to add to the tracking file
TrackStat_t trackStat; // Stat to track (min free CPU memory, min free GPU memory...)
// Data analysis filters (per-log-entry)
float dangerLimit; // Spew log entries in which memory drops below this limit (in MB)
int minPlayers; // Spew log entries with at least this many concurrent players
int maxPlayers; // Spew log entries with at most this many concurrent players
bool isSplitscreen; // Spew log entries in which the machine has more than one local player
bool isSinglescreen; // Spew log entries in which the machine has AT MOST one local player
char mapFilter[FILTER_SIZE]; // Spew log entries with map names which contain this substring
char playerFilter[FILTER_SIZE]; // Spew log entries with player names which contain this substring
// Data analysis filters (per-log-file)
int dangerTime; // Spew logs in which memory drops below the danger limit within this many seconds
int duration; // Spew logs in which the timer reaches this many seconds
int minAge; // Spew logs updated at least this many seconds ago
int maxAge; // Spew logs updated at most this many seconds ago
bool isServer; // Spew logs in which the machine is a listen server at least once
bool isClient; // Spew logs in which the machine is NEVER a listen server
bool isActive; // Spew logs currently being written
DVDHosted_t dvdHosted; // Spew logs matching this DVD hosted state (UNKNOWN means accept all)
char languageFilter[FILTER_SIZE]; // Spew logs with a language which contain this substring
Platform_t platform; // Spew logs generated from the specified platform (UNKNOWN means accept all)
};
Config gConfig;
// 10 million 100-nanosecond intervals per second in FILETIME
const ULONGLONG ONE_SECOND = 10000000LL;
int AddNewMapName( const char *name )
{
char *nameCopy = strdup( name ); // Leak
strlwr( nameCopy );
if ( gMapHash.find( string( nameCopy ) ) != gMapHash.end() )
{
printf( "ERROR: duplicate map name in gMapNames!!!! (%s)\n", nameCopy );
printf( "Aborting... press any key to exit\n" );
DebuggerBreakIfDebugging();
getchar();
exit( 1 );
}
int index = gMapNames.AddToTail( nameCopy );
gMapHash[ string( nameCopy ) ] = index;
return index;
}
bool ParseLineForDVDHostedInfo( string &line, CHeaderInfo &headerInfo )
{
// Game spews text like this on startup:
//
// Xbox Launched From DVD.
// Install Status:
// Version: 746360 (english) (Xbox|PS3|PC)
// DVD Hosted: Enabled <---- this is the important line
// Progress: 0/1200 MB
// Active Image: 746360
//
// We use that to determine if we are running SOLELY off the DVD (no HDD access, no texture streaming),
// in which case textures/audio/anims take more memory, so the minimum free memory watermark is lower.
// UPDATE: these memory effects apply to L4D/L4D2 but not PORTAL2 (it doesn't do texture streaming)
if ( headerInfo.dvdHosted == DVDHOSTED_UNKNOWN )
{
const char *cursor = line.c_str();
if ( strstr( cursor, "Device\\Harddisk" ) )
{
// Running from the HDD (this test works for old images without the "DVD Hosted:" spew)
headerInfo.dvdHosted = DVDHOSTED_NO;
return true;
}
else if ( strstr( cursor, "DVD Hosted:" ) )
{
headerInfo.dvdHosted = strstr( cursor, "Enabled" ) ? DVDHOSTED_YES : DVDHOSTED_NO;
return true;
}
}
return false;
}
bool ParseLineForBuildNumberLanguageAndPlatformInfo( string &line, CHeaderInfo &headerInfo )
{
// Game spews text like this on startup:
//
// Xbox Launched From DVD.
// Install Status:
// Version: 746360 (english) (Xbox|PS3|PC) <---- this is the important line
// DVD Hosted: Enabled
// Progress: 0/1200 MB
// Active Image: 746360
//
// We use that to determine current build number and language.
if ( headerInfo.buildNumber == -1 )
{
const char *cursor = line.c_str();
cursor = strstr( cursor, "Version:" );
if ( cursor )
{
static char language[1024], platform[1024];
if ( 3 == sscanf( cursor, "Version: %d (%s (%s", &headerInfo.buildNumber, &(language[0]), &(platform[0]) ) )
{
strncpy( &(headerInfo.language[0]), &(language[0]), MAX_LANG_LEN );
char *closingBrace = strstr( headerInfo.language, ")" );
if ( closingBrace ) *closingBrace = 0;
strlwr( &( headerInfo.language[0] ) );
closingBrace = strstr( platform, ")" );
if ( closingBrace ) *closingBrace = 0;
if ( !stricmp( platform, "Xbox" ) )
headerInfo.platform = PLATFORM_360;
else if ( !stricmp( platform, "PS3" ) )
headerInfo.platform = PLATFORM_PS3;
else if ( !stricmp( platform, "PC" ) )
headerInfo.platform = PLATFORM_PC;
return true;
}
headerInfo.buildNumber = -1;
headerInfo.language[0] = 0;
headerInfo.platform = PLATFORM_UNKNOWN;
}
}
return false;
}
bool ParseLineForHeaderInfo( string &line, CHeaderInfo &headerInfo )
{
if ( ParseLineForDVDHostedInfo( line, headerInfo ) )
return true;
if ( ParseLineForBuildNumberLanguageAndPlatformInfo( line, headerInfo ) )
return true;
return false;
}
int ConvertItem( string &line, CItem &result, const CItem &prevItem, CHeaderInfo *headerInfo, const char *fileName, int lineNum, vector<int> *truncated, CStringList *mapList, CStringList *playerList, bool skipIgnored )
{
// Accumulate header lines as we go through the log
if ( headerInfo && ParseLineForHeaderInfo( line, *headerInfo ) )
return -1;
// Special-case 'out of memory' lines:
const char *start = strstr( line.c_str(), "*** OUT OF MEMORY!" );
if ( start )
{
// Replace this with a fake "zero memory free" line
static char fakeLine[1024];
_snprintf( fakeLine, sizeof( fakeLine ), "[MEMORYLOG] Time:%6d | Free:%6.2f | GPU:%6.2f | %s | Map: %-32s | Bots:%2d | Players: 0",
prevItem.time, 0.0f, 0.0f, prevItem.isServer ? "Server" : "Client",
( ( prevItem.globalMapIndex < 0 ) ? "none" : gMapNames[ prevItem.globalMapIndex ] ), prevItem.numBots );
printf( "---- OUT OF MEMORY on line %d, in %s\n\n", lineNum, fileName );
line = fakeLine;
}
start = strstr( line.c_str(), "[MEMORYLOG] " );
if ( !start )
return -1;
// Get rid of line endings returned by GetLine()
while ( ( line[ line.size() - 1 ] == '\r' ) || ( line[ line.size() - 1 ] == '\n' ) )
line.resize( line.size() - 1 );
int time;
float freeMem;
float GPUFree;
char hostType[33];
char mapName[33];
int numBots;
int numPlayers;
CMapHash::iterator it;
int numRead = sscanf( start,
"[MEMORYLOG] Time:%d | Free: %f | GPU: %f | %s | Map: %s | Bots: %d | Players: %d",
&time,
&freeMem,
&GPUFree,
hostType,
mapName,
&numBots,
&numPlayers );
if ( numRead != 7 )
goto error;
bool isServer;
if ( !stricmp( hostType, "Server" ) )
isServer = true;
else if ( !stricmp( hostType, "Client" ) )
isServer = false;
else
goto error;
int globalMapIndex = -1;
strlwr( mapName );
it = gMapHash.find( mapName );
if ( it != gMapHash.end() )
{
globalMapIndex = it->second;
if ( skipIgnored && ( globalMapIndex < gNumIgnoreMaps ) )
return -1;
}
else
{
// Add the new name to gMapNames and gMapHash
globalMapIndex = AddNewMapName( mapName );
// This message notifies us when new maps appear on the horizon (e.g. 'credits'!)
printf( "----WARNING: unrecognised map name '%s' on line %d, in %s\n\n", mapName, lineNum, fileName );
}
// Also add the map name to a per-logfile list
int logMapIndex = mapList ? mapList->AddString( mapName ) : -1;
// Iterate over players
int numLocalPlayers = 0, numSpectators = 0, numNamedBots = 0;
const char *playerStart = start;
for ( int i = 0; i < numPlayers; i++ )
{
playerStart = strstr( playerStart, ", " );
if ( !playerStart )
goto playerError;
playerStart += 2;
const char *playerEnd = playerStart;
while ( playerEnd[0] && ( playerEnd[0] != '|' ) && ( playerEnd[0] != ',' ) ) playerEnd++;
if ( playerList )
{
// Build a vector of player name references
string playerName( playerStart, ( playerEnd - playerStart ) );
strlwr( (char*)playerName.c_str() );
int playerIndex = playerList->AddString( playerName.c_str() );
result.playerBitfield.Set( playerIndex );
}
// Track how many local/spectator players there are
while ( playerEnd[0] == '|' )
{
playerEnd++;
if ( !strncmp( playerEnd, "LOCAL", 5 ) )
{
numLocalPlayers++;
playerEnd += 5;
}
else if ( !strncmp( playerEnd, "SPEC", 4 ) )
{
numSpectators++;
playerEnd += 4;
}
else if ( !strncmp( playerEnd, "BOT", 3 ) )
{
numNamedBots++;
playerEnd += 3;
}
else
goto playerError;
if ( playerEnd[0] && ( playerEnd[0] != ',' ) && ( playerEnd[0] != '|' ) && ( playerEnd[0] != ' ' ) )
goto playerError;
}
}
Assert( !numNamedBots || ( numNamedBots == numBots ) );
if ( numNamedBots && ( numNamedBots != numBots ) )
goto playerError;
if ( false )
{
playerError:
// Don't quit on errors in the player list, keep the data we managed to get (usually truncated lines in versus games)
if ( strlen( line.c_str() ) == 264 )
{
// TODO: Somewhere, lines are getting capped at 264 chars (don't think it's in VXConsole, it's max command len is 4096)
if ( truncated ) truncated->push_back( lineNum );
}
else
{
printf( "----ERROR: unrecognised [MEMORYLOG] syntax on line %d, in %s\n", lineNum, fileName );
printf( " \"%s\"\n\n", line.c_str() );
}
}
result.freeMem = freeMem;
result.gpuFree = GPUFree;
result.time = time;
result.globalMapIndex = globalMapIndex;
result.logMapIndex = logMapIndex;
result.numBots = numBots;
result.numPlayers = numPlayers;
result.numLocalPlayers = numLocalPlayers;
result.numSpecators = numSpectators;
result.isServer = isServer;
return ( start - line.c_str() );
error:
printf( "----ERROR: unrecognised [MEMORYLOG] syntax on line %d, in %s\n", lineNum, fileName );
printf( " \"%s\"\n\n", line.c_str() );
return -1;
}
bool ReadFile( const string &fileName, CUtlBuffer &buffer )
{
FILE *fp = fopen( fileName.c_str(), "rb" );
if (!fp)
return false;
fseek( fp, 0, SEEK_END );
int nFileLength = ftell( fp );
fseek( fp, 0, SEEK_SET );
buffer.EnsureCapacity( nFileLength );
int nBytesRead = fread( buffer.Base(), 1, nFileLength, fp );
fclose( fp );
buffer.SeekPut( CUtlBuffer::SEEK_HEAD, nBytesRead );
return true;
}
bool WriteFile( const string &fileName, CUtlBuffer &buffer, CUtlBuffer *header )
{
FILE *fp = fopen( fileName.c_str(), "wb" );
if ( !fp )
return false;
if ( header )
fwrite( header->Base(), 1, header->TellPut(), fp );
fwrite( buffer.Base(), 1, buffer.TellPut(), fp );
fclose( fp );
return true;
}
char *SpewHeaderSummary( char *buffer, int bufferSize, const CHeaderInfo &headerInfo )
{
const char *dvdHosted = ( headerInfo.dvdHosted == DVDHOSTED_UNKNOWN ) ? " - " : ( ( headerInfo.dvdHosted == DVDHOSTED_YES ) ? "DVD" : "HDD" );
bool languageKnown = !!strcmp( headerInfo.language, "unknown" );
_snprintf( buffer, bufferSize, "%7d %4s %10s ", headerInfo.buildNumber, dvdHosted, languageKnown ? headerInfo.language : " -- " );
return buffer;
}
void WriteHeaderInfoToBuffer( CUtlBuffer &buffer, CHeaderInfo &headerInfo )
{
buffer.Clear();
buffer.Printf( "[HDR]:version=%d\n", MEMLOG_VERSION );
buffer.Printf( "[HDR]:dvdhosted=%s\n", ( headerInfo.dvdHosted == DVDHOSTED_UNKNOWN ) ? "unknown" : ( ( headerInfo.dvdHosted == DVDHOSTED_YES ) ? "yes" : "no" ) );
buffer.Printf( "[HDR]:build=%d\n", headerInfo.buildNumber );
buffer.Printf( "[HDR]:language=%s\n", ( headerInfo.language && headerInfo.language[0] ) ? headerInfo.language : "unknown" );
buffer.Printf( "[HDR]:platform=%s\n", gPlatformNames[ headerInfo.platform ] );
}
void InitItem( CItem &item )
{
item.time = 0;
item.freeMem = CPU_MEM_SIZE;
item.gpuFree = GPU_MEM_SIZE;
item.globalMapIndex = gMapHash[ string( "backgroundstreet" ) ];
item.logMapIndex = -1;
item.numBots = 0;
item.numPlayers = 0;
item.numLocalPlayers = 0;
item.numSpecators = 0;
item.isServer = false;
}
bool ConvertVXConsoleLog( string &consoleLog, string &memoryLog )
{
CUtlBuffer iBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER|CUtlBuffer::CONTAINS_CRLF );
CUtlBuffer oBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER|CUtlBuffer::CONTAINS_CRLF );
CUtlBuffer ohBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER|CUtlBuffer::CONTAINS_CRLF );
if ( !ReadFile( consoleLog, iBuffer ) )
{
printf( "----ERROR: Failed to read file %s\n\n", consoleLog.c_str() );
return false;
}
bool isEmpty = true;
int lineNum = 1;
vector<int> truncated;
CHeaderInfo headerInfo;
CItem prevItem;
InitItem( prevItem );
while ( iBuffer.IsValid() )
{
static char lineBuf[ 10000 ];
iBuffer.GetLine( lineBuf, sizeof( lineBuf ) );
// CUtlBuffer.GetLine() returns TWO lines for lines ending in '\r\n' :o/
if ( lineBuf[0] == '\n' ) continue;
if ( !lineBuf[0] && ( iBuffer.TellGet() != iBuffer.TellPut() ) )
{
// CUtlBuffer.GetLine() can fail if there are non-ASCII characters in the log. If that happens
// (which it does - vxconsole bug?), we'll get a zero-length line, so unstick the buffer:
iBuffer.GetChar();
continue;
}
CItem item;
string line = lineBuf;
int start = ConvertItem( line, item, prevItem, &headerInfo, consoleLog.c_str(), lineNum, &truncated, NULL, NULL, false );
if ( start >= 0 )
{
// If the line is valid, copy it to the output file
oBuffer.Put( ( line.c_str() + start ), ( line.size() - start ) );
oBuffer.Put( "\n", 1 );
isEmpty = false;
prevItem = item;
}
lineNum++;
}
// Write the headerInfo to a buffer
WriteHeaderInfoToBuffer( ohBuffer, headerInfo );
if ( truncated.size() )
{
printf( "----WARNING: [MEMORYLOG] text truncated in %s\n", consoleLog.c_str() );
printf( "---- Truncated lines: %d", truncated[0] );
for ( unsigned int i = 1; i < truncated.size(); i++ ) printf( ", %d", truncated[i] );
printf( "\n" );
}
// Write the output file, with zero or more items in it
if ( !WriteFile( memoryLog, oBuffer, &ohBuffer ) )
{
printf( "----ERROR: Failed to write file %s\n\n", memoryLog.c_str() );
return false;
}
return !isEmpty;
}
CLogFile &InitLogFile( CLogFiles &results, const string &memoryLog, const string &consoleLog, const ULONGLONG &modifyTime, bool isActive )
{
CLogFile *result = new CLogFile();
pair<CLogFiles::iterator,bool> insertion = results.insert( make_pair( memoryLog, result ) );
if ( !insertion.second || ( insertion.first == results.end() ) )
{
printf( "----ERROR: hash_map insertion failed...?\n\n" );
insertion.first = results.find( memoryLog );
if ( insertion.first == results.end() )
printf( "----ERROR: hash_map totally b0rked, gonna crash...\n\n" );
}
result->memoryLog = memoryLog;
result->consoleLog = consoleLog;
result->modifyTime = modifyTime;
result->isActive = isActive;
result->minFreeMem = CPU_MEM_SIZE;
result->minGPUFreeMem = GPU_MEM_SIZE;
result->isServer = false;
result->isSplitscreen = false;
result->maxPlayers = 0;
return *result;
}
bool StringToPlatformName( const char *name, Platform_t &platform )
{
for ( int i = 0; i < ARRAYSIZE( gPlatformNames ); i++ )
{
if ( !strnicmp( name, gPlatformNames[ i ], strlen( gPlatformNames[ i ] ) ) )
{
platform = (Platform_t)i;
return true;
}
}
return false;
}
bool ReadHeaderLine( const char *lineBuf, CHeaderInfo &headerInfo )
{
static const char HEADER_PREFIX[] = "[HDR]:";
static const int HEADER_PREFIX_LEN = sizeof( HEADER_PREFIX ) - 1;
static const char KEY_VERSION[] = "version=";
static const int KEY_VERSION_LEN = sizeof( KEY_VERSION ) - 1;
static const char KEY_DVD_HOSTED[] = "dvdhosted=";
static const int KEY_DVD_HOSTED_LEN = sizeof( KEY_DVD_HOSTED ) - 1;
static const char KEY_BUILDNUM[] = "build=";
static const int KEY_BUILDNUM_LEN = sizeof( KEY_BUILDNUM ) - 1;
static const char KEY_LANGUAGE[] = "language=";
static const int KEY_LANGUAGE_LEN = sizeof( KEY_LANGUAGE ) - 1;
static const char KEY_PLATFORM[] = "platform=";
static const int KEY_PLATFORM_LEN = sizeof( KEY_PLATFORM ) - 1;
const char *cursor = lineBuf;
if ( strncmp( cursor, HEADER_PREFIX, HEADER_PREFIX_LEN ) )
return false;
cursor += HEADER_PREFIX_LEN;
if ( !strncmp( cursor, KEY_VERSION, KEY_VERSION_LEN ) )
{
cursor += KEY_VERSION_LEN;
headerInfo.memlogVersion = atoi( cursor );
return true;
}
else if ( !strncmp( cursor, KEY_DVD_HOSTED, KEY_DVD_HOSTED_LEN ) )
{
cursor += KEY_DVD_HOSTED_LEN;
if ( !strncmp( cursor, "yes", 3 ) )
{
headerInfo.dvdHosted = DVDHOSTED_YES;
}
else if ( !strncmp( cursor, "no", 2 ) )
{
headerInfo.dvdHosted = DVDHOSTED_NO;
}
return true;
}
else if ( !strncmp( cursor, KEY_BUILDNUM, KEY_BUILDNUM_LEN ) )
{
cursor += KEY_BUILDNUM_LEN;
headerInfo.buildNumber = atoi( cursor );
return true;
}
else if ( !strncmp( cursor, KEY_LANGUAGE, KEY_LANGUAGE_LEN ) )
{
cursor += KEY_LANGUAGE_LEN;
strncpy( headerInfo.language, cursor, MAX_LANG_LEN );
char *eol = strstr( headerInfo.language, "\n" );
if ( eol ) *eol = 0;
return true;
}
else if ( !strncmp( cursor, KEY_PLATFORM, KEY_PLATFORM_LEN ) )
{
cursor += KEY_PLATFORM_LEN;
if ( StringToPlatformName( cursor, headerInfo.platform ) )
return true;
}
printf( "----ERROR: Unrecognised header line: (%s)\n\n", lineBuf );
return true;
}
int ReadMemoryLog( string &memoryLog, string &consoleLog, CLogFiles & results, const ULONGLONG &modifyTime, bool isActive, bool checkVersion )
{
CUtlBuffer iBuffer( 0, 0, CUtlBuffer::TEXT_BUFFER|CUtlBuffer::CONTAINS_CRLF );
if ( !ReadFile( memoryLog.c_str(), iBuffer ) )
{
printf( "----ERROR: Failed to open input file %s\n\n", memoryLog.c_str() );
return 0;
}
CLogFile &result = InitLogFile( results, memoryLog, consoleLog, modifyTime, isActive );
bool bOrderError = false;
bool bReallyEmpty = true;
bool bDoneHeader = false;
int lineNum = 1;
int lastTime = 0;
CItem prevItem;
InitItem( prevItem );
while ( iBuffer.IsValid() )
{
// create+add an item
static char lineBuf[ 10000 ];
iBuffer.GetLine( lineBuf, sizeof( lineBuf ) );
// CUtlBuffer.GetLine() returns TWO lines for lines ending in '\r\n' :o/
if ( lineBuf[0] == '\n' ) continue;
if ( !bDoneHeader )
{
// Read one or more header lines at the top of the memorylog file
if ( ReadHeaderLine( lineBuf, result.headerInfo ) )
{
lineNum++;
continue;
}
bDoneHeader = true;
// Check file version
if ( ( result.headerInfo.memlogVersion < MEMLOG_VERSION ) && checkVersion )
{
printf( "----Log is from an old version, re-converting...\n" );
CLogFile *oldLog = results.find( memoryLog )->second;
if ( oldLog )
{
delete oldLog;
results.erase( memoryLog );
}
return -1;
}
else if ( result.headerInfo.memlogVersion > MEMLOG_VERSION )
{
printf( "----ERROR: memory log file is newer than this version of memlog.exe!\n\n" );
}
}
CItem item;
string line = lineBuf;
if ( ConvertItem( line, item, prevItem, NULL, memoryLog.c_str(), lineNum, NULL, &result.mapList, &result.playerList, true ) >= 0 )
{
bReallyEmpty = false;
if ( item.globalMapIndex >= 0 )
{
result.entries.AddToTail( item );
prevItem = item;
}
bOrderError = bOrderError || ( item.time < lastTime );
lastTime = item.time;
}
lineNum++;
}
// VXConsole may have concatenated logs from multiple runs - DO NOT WANT!
if ( bOrderError )
printf( "----ERROR: '[MEMORYLOG]' timestamps are out-of-order! Log file may contain multiple runs of the game...\n\n" );
// It's ok to load logs with [MEMORYLOG] lines only for invalid maps, but
// logs with no [MEMORYLOG] lines should be zero size and not loaded at all.
// UPDATE: the header info might still be useful, so we load 'em anyway...
if ( bReallyEmpty )
printf( "----Empty memorylog file.\n" );
// Compute a few aggregate stats in order to speed up log filtering:
int noneMap = gMapHash[ string( "none" ) ];
int menuMap = gMapHash[ string( "backgroundstreet" ) ];
int creditsMap = gMapHash[ string( "credits" ) ];
float memorylogTick = 0;
for ( int i = 0; i < result.entries.Count(); i++ )
{
CItem &item = result.entries[ i ];
result.minFreeMem = min( result.minFreeMem, item.freeMem );
result.minGPUFreeMem = min( result.minGPUFreeMem, item.gpuFree );
result.maxPlayers = max( result.maxPlayers, item.numPlayers );
if ( item.numLocalPlayers > 1 )
result.isSplitscreen = true;
if ( item.isServer && ( item.globalMapIndex != noneMap ) && ( item.globalMapIndex != menuMap ) && ( item.globalMapIndex != creditsMap ) )
result.isServer = true;
if ( i > 0 ) // Average the intervals between memorylog entries
memorylogTick += ( item.time - result.entries[ i - 1 ].time );
}
if ( result.entries.Count() > 1 )
memorylogTick /= ( result.entries.Count() - 1 );
result.memorylogTick = memorylogTick;
return 1;
}
ULONGLONG WriteTime( WIN32_FIND_DATA &fileData )
{
return ( ((ULONGLONG)fileData.ftLastWriteTime.dwHighDateTime ) << 32 ) | (ULONGLONG)fileData.ftLastWriteTime.dwLowDateTime;
}
/*ULONGLONG CreateTime( WIN32_FIND_DATA &fileData )
{
return ( ((ULONGLONG)fileData.ftCreationTime.dwHighDateTime ) << 32 ) | (ULONGLONG)fileData.ftCreationTime.dwLowDateTime;
}*/
ULONGLONG SystemTime( void )
{
SYSTEMTIME LocalSysTime;
FILETIME LocalFileTime;
FILETIME UTCFileTime;
BOOL success = TRUE;
GetLocalTime( &LocalSysTime );
success = SystemTimeToFileTime( &LocalSysTime, &LocalFileTime );
success = ( LocalFileTimeToFileTime( &LocalFileTime, &UTCFileTime ) || success );
if ( !success )
{
static int errCount = 0;
if ( !errCount++ )
{
printf( "----ERROR: error generating system time... all logs will be considered 'active'\n\n" );
DebuggerBreak();
}
return 0;
}
return ( ((ULONGLONG)UTCFileTime.dwHighDateTime ) << 32 ) | (ULONGLONG)UTCFileTime.dwLowDateTime;
}
bool IsActive( ULONGLONG lastWriteTime )
{
return ( SystemTime() < ( lastWriteTime + 60*ONE_SECOND ) );
}
void Tick( void )
{
// Let the user know we're still working (recursing through fileserver is sloooow)
SYSTEMTIME systemTime;
GetLocalTime( &systemTime );
static int lastTime = -10;
if ( ( ( systemTime.wSecond % 10 ) == 0 ) && ( systemTime.wSecond != lastTime ) )
{
printf( "%02d:%02d:%02d--------------------------------------------------------------------------------------------\n", systemTime.wHour, systemTime.wMinute, systemTime.wSecond );
lastTime = systemTime.wSecond;
}
}
void _ProcessLogFiles( const char *path, CLogFiles &results,
int &numLoaded, int &numUnloaded, int &numConverted, int &numUpdated, int &numActive )
{
WIN32_FIND_DATA fileData;
HANDLE handle;
Tick();
if ( gConfig.load )
{
// Convert 'vxconsole_*.log' files
static char pathBuf[ _MAX_PATH ]; // No recursion in this loop
_snprintf( pathBuf, sizeof( pathBuf ), "%svxconsole_*.log", path );
handle = FindFirstFile( pathBuf, &fileData );
if ( handle != INVALID_HANDLE_VALUE )
{
do
{
_snprintf( pathBuf, sizeof( pathBuf ) , "%s%s", path, fileData.cFileName );
string consoleLog = pathBuf;
// Does this 'vxconsole_*.log' have a corresponding 'memorylog_*.log'?
static char prefix[] = "vxconsole_";
char *suffix = fileData.cFileName + sizeof( prefix ) - 1;
_snprintf( pathBuf, sizeof( pathBuf ) , "%smemorylog_%s", path, suffix );
string memoryLog = pathBuf;
WIN32_FIND_DATA fileData2;
HANDLE handle2 = FindFirstFile( memoryLog.c_str(), &fileData2 );
// Convert all logs if asked to do a forced update
bool bNeedsConverting = gConfig.forceUpdate;
bool isActive = ( handle2 != INVALID_HANDLE_VALUE ) && IsActive( WriteTime( fileData ) );
if ( !bNeedsConverting )
{
if ( isActive )
{
// Only convert currently-active logs if that is specifically requested
bNeedsConverting = gConfig.updateActive;
if ( !gConfig.updateActive )
printf( "--Skipping update of ACTIVE log %s\n", consoleLog.c_str() );
numActive++;
}
else if ( handle2 == INVALID_HANDLE_VALUE )
{
// Convert logs we haven't converted before
bNeedsConverting = true;
}
else if ( gConfig.update )
{
// We have been asked to reconvert logs that have been updated
bNeedsConverting = WriteTime( fileData ) > WriteTime( fileData2 );
}
}
if ( bNeedsConverting )
{
// Create/update the memory log
bool bUpdating = ( handle2 != INVALID_HANDLE_VALUE );
printf( "--%s %s\n", ( bUpdating ? "Updating" : "Converting" ), consoleLog.c_str() );
if ( isActive )
printf( "----Log is ACTIVE\n" );
ConvertVXConsoleLog( consoleLog, memoryLog );
numConverted += bUpdating ? 0 : 1;
numUpdated += bUpdating ? 1 : 0;
}
FindClose( handle2 );
Tick();
}
while( FindNextFile( handle, &fileData ) );
FindClose( handle );
}
}
// Note that update/updateActive/forceUpdate imply 'load' (it's confusing otherwise)
if ( gConfig.load || gConfig.unload )
{
// Read in 'memorylog_*.log' files
static char pathBuf[ _MAX_PATH ]; // No recursion in this loop
_snprintf( pathBuf, sizeof( pathBuf ), "%smemorylog_*.log", path );
handle = FindFirstFile( pathBuf, &fileData );
if ( handle != INVALID_HANDLE_VALUE )
{
do
{
// Compute full memorylog_*/vxconsole_* paths
string memoryLog = path;
memoryLog += fileData.cFileName;
strlwr( (char *)memoryLog.c_str() );
// Is this memory log already in memory?
CLogFiles::iterator it = results.find( memoryLog );
bool bLoaded = ( it != results.end() );
// Re-load active logs if requested
if ( bLoaded && it->second->isActive && gConfig.updateActive )
{
CLogFile *oldLog = it->second;
if ( oldLog ) delete oldLog;
results.erase( it );
bLoaded = false;
}
if ( gConfig.load && !bLoaded )
{
// Get the timestamp for the corresponding vxconsole log (if there is one)
char *suffix = fileData.cFileName + strlen( "memorylog_" );
_snprintf( pathBuf, sizeof( pathBuf ) , "%svxconsole_%s", path, suffix );
string consoleLog = pathBuf;
strlwr( (char *)consoleLog.c_str() );
WIN32_FIND_DATA fileData2;
HANDLE handle2 = FindFirstFile( consoleLog.c_str(), &fileData2 );
ULONGLONG timeStamp = ( handle2 != INVALID_HANDLE_VALUE ) ? WriteTime( fileData2 ) : WriteTime( fileData );
bool isActive = ( handle2 != INVALID_HANDLE_VALUE ) ? IsActive( WriteTime( fileData2 ) ) : false;
if ( handle2 == INVALID_HANDLE_VALUE ) consoleLog = "";
FindClose( handle2 );
// Load each memorylog file, and add it to the results
printf( "--Loading %s\n", memoryLog.c_str() );
if ( isActive )
printf( "----Log is ACTIVE, %s\n", ( gConfig.updateActive | gConfig.forceUpdate ) ? "loaded up-to-date memorylog data" : "loading old memorylog data" );
bool checkVersion = true, ignoreVersion = false;
int retVal = ReadMemoryLog( memoryLog, consoleLog, results, timeStamp, isActive, checkVersion );
if ( retVal == -1 )
{
// Memorylog is out of date, re-convert it
numUpdated += ConvertVXConsoleLog( consoleLog, memoryLog ) ? 1 : 0;
printf( "----Loading re-converted log\n", memoryLog.c_str() );
retVal = ReadMemoryLog( memoryLog, consoleLog, results, timeStamp, isActive, ignoreVersion );
}
if ( retVal > 0 )
numLoaded++;
}
else if ( gConfig.unload && bLoaded )
{
// Unload this memorylog file
printf( "--Unloading %s\n", memoryLog.c_str() );
results.erase( memoryLog );
numUnloaded++;
}
Tick();
}
while( FindNextFile( handle, &fileData ) );
FindClose( handle );
}
}
if ( gConfig.recurse )
{
// Recurse to subdirectories
char *searchPath = new char[ _MAX_PATH ]; // Avoid blowing the stack due to recursion
_snprintf( searchPath, _MAX_PATH, "%s*.*", path );
handle = FindFirstFile( searchPath, &fileData );
if ( handle != INVALID_HANDLE_VALUE )
{
do
{
if ( ( fileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) &&
( fileData.cFileName[ 0 ] != '.' ) )
{
char *subDir = new char[ _MAX_PATH ]; // Avoid blowing the stack due to recursion
_snprintf( subDir, _MAX_PATH, "%s%s\\", path, fileData.cFileName );
_ProcessLogFiles( subDir, results, numLoaded, numUnloaded, numConverted, numUpdated, numActive );
delete[] subDir;
}
}
while( FindNextFile( handle, &fileData ) );
}
FindClose( handle );
delete[] searchPath;
}
}
void ProcessLogFiles( const char *path, CLogFiles &results )
{
if ( gConfig.unloadAll )
{
CLogFiles::iterator it;
for ( it = results.begin(); it != results.end(); it++ )
{
printf( "--Unloading %s\n", it->first.c_str() );
delete it->second;
}
printf( "\n" );
printf( "%d logs unloaded\n", results.size() );
printf( "\n" );
results.clear();
return;
}
if ( !path || !path[0] )
return;
if ( gConfig.load || gConfig.unload )
{
int numLoaded = 0, numUnloaded = 0, numConverted = 0, numUpdated = 0, numEmpty = 0, numActive = 0;
_ProcessLogFiles( gConfig.sourcePath, results, numLoaded, numUnloaded, numConverted, numUpdated, numActive );
for ( CLogFiles::iterator it = results.begin(); it != results.end(); it++ )
{
numEmpty += it->second->entries.Count() ? 0 : 1;
}
printf( "\n" );
/* TODO:
* numLoaded is confusing... not sure what it means
* I updated a bunch of logs from fileserver and saw this:
* 2 logs loaded
* 26 new logs converted
* 12 logs re-converted
* 7 active logs (converted)
* (1435 empty log files)
* in light of that, I have no idea what "logs loaded" means...
* hmm, it seems to occur for empty logs!
* then I ran "r u a" a second time (without copying over any more files) and it found another empty log... and listed logs loaded as 1
* then I did it again and it loaded no logs...
* weird
*/
if ( numLoaded ) printf( "%d logs loaded\n", numLoaded );
if ( numUnloaded ) printf( "%d logs unloaded\n", numUnloaded );
if ( numConverted ) printf( "%d new logs converted\n", numConverted );
if ( numUpdated ) printf( "%d logs re-converted\n", numUpdated );
if ( numActive ) printf( "%d active logs %s\n", numActive, gConfig.updateActive ? "(converted)" : "(not converted)" );
if ( numEmpty ) printf( "(%d empty log files)\n", numEmpty );
printf( "\n" );
}
}
int trim( string &s )
{
// Remove whitespace from the head+tail of the string
size_t oldlen = s.length(), head = 0, tail = 0;
while( ( head < oldlen ) && V_isspace( s[head] ) ) head++;
while( ( tail < oldlen ) && V_isspace( s[oldlen-1-tail] ) ) tail++;
size_t numRemoved = head + tail;
if ( !numRemoved )
return 0;
s = s.substr( head, MAX(0,oldlen-numRemoved) ); // The 'MAX' fixes the case where the string is ALL whitespace :)
return numRemoved;
}
struct TrackSort_t
{
int line;
float minFree;
};
int TrackSortFunc( const void *a, const void *b )
{
TrackSort_t *A = (TrackSort_t *)a, *B = (TrackSort_t *)b;
return ( A->minFree < B->minFree ) ? -1 : +1;
}
void NormalizeCSVRowLengths( vector< vector< string > > &lines )
{
unsigned int nMaxLineLength = 0;
for ( unsigned int i = 0; i < lines.size(); i++ )
{
nMaxLineLength = max( nMaxLineLength, lines[i].size() );
}
for ( unsigned int i = 0; i < lines.size(); i++ )
{
string empty = i ? "" : "unknown";
vector<string> &line = lines[i];
while ( line.size() < nMaxLineLength )
{
line.push_back( empty );
}
}
}
bool UpdateTrackFile( const char *trackFile, const char *trackColumn,
unsigned int *mapSamples, float *mapMin, float *mapGPUMin )
{
// Make a backup before we start
CUtlBuffer fileBuffer;
string fileName = trackFile;
if ( ReadFile( fileName, fileBuffer ) )
{
int nCharsToExtension = V_stristr( trackFile, ".csv" ) - trackFile;
Assert( nCharsToExtension > 0 );
string backupName = fileName.substr( 0, nCharsToExtension ) + "_bak.csv";
if ( !WriteFile( backupName, fileBuffer, NULL ) )
{
printf( "ERROR: could not write backup tracking .CSV file (%s), aborting!\n\n", backupName.c_str() );
return false;
}
}
// Now read the .CSV into a vector of vectors of strings
const string MAP_NAME_HEADER = "Map Name";
unsigned int nMaxRowLen = 0;
vector<vector<string>> lines;
CMapHash mapsInCSV;
ifstream file;
file.open( trackFile );
if ( file.is_open() )
{
while ( !file.eof() )
{
string lineString;
getline( file, lineString );
if ( !lineString.size() )
break;
// Each line is a vector of strings:
string item;
vector<string> line;
stringstream lineStream( lineString );
while ( !lineStream.eof() )
{
getline( lineStream, item, ',' );
trim( item ); // avoid duplicates differing only by whitespace
line.push_back( item );
}
nMaxRowLen = max( nMaxRowLen, line.size() );
lines.push_back( line );
}
file.close();
}
// Init an empty .CSV file:
if ( !nMaxRowLen || !lines.size() )
{
vector<string> headerLine;
headerLine.push_back( MAP_NAME_HEADER );
lines.clear();
lines.push_back( headerLine );
}
// Validate the .CSV file
for ( unsigned int i = 0; i < lines.size(); i++ )
{
string &firstItem = lines[ i ][ 0 ];
if ( ( ( i == 0 ) && ( firstItem != MAP_NAME_HEADER ) ) || !firstItem.size() )
{
printf( "ERROR: first item on line %d in tracking .CSV file (%s) is invalid, aborting!\n\n", i+1, trackFile );
return false;
}
// Make a note of all the maps found in the file
if ( i > 0 )
{
if ( mapsInCSV.find( firstItem ) != mapsInCSV.end() )
{
printf( "ERROR: map '%s' occurs more than once in tracking .CSV file (%s), aborting!\n\n", firstItem.c_str(), trackFile );
return false;
}
mapsInCSV[ firstItem ] = i;
}
}
// Normalize row lengths
NormalizeCSVRowLengths( lines );
// Add a new column
vector<string> &headerLine = lines[0];
if ( trackColumn[0] )
{
headerLine.push_back( string( trackColumn ) );
}
else
{
ostringstream columnName;
columnName << "column_";
columnName << headerLine.size();
headerLine.push_back( columnName.str() );
}
// Add data to the .CSV file
bool trackGPUMem = ( gConfig.trackStat == TRACKSTAT_MINFREE_GPU );
unsigned int numMaps = gMapNames.Count();
for ( unsigned int i = gNumIgnoreMaps; i < numMaps; i++ )
{
string empty = "", mapName = gMapNames[ i ];
if ( mapsInCSV.find( mapName ) == mapsInCSV.end() )
{
// This map is not present in the .CSV file, so add a new row
vector<string> newLine;
newLine.push_back( mapName );
while ( newLine.size() < nMaxRowLen )
{
newLine.push_back( empty );
}
lines.push_back( newLine );
mapsInCSV[ mapName ] = lines.size() - 1;
}
// Add the new data sample to the end of this line
int nMapLine = mapsInCSV[ mapName ];
vector<string> &line = lines[ nMapLine ];
if ( mapSamples[ i ] )
{
// Add the worst case sample for this map
ostringstream minFree;
minFree << ( trackGPUMem ? mapGPUMin[ i ] : mapMin[ i ] );
line.push_back( minFree.str() );
}
else
{
// Add an empty entry for this map
// (it makes diffing/sorting/combining/comparing these CSVs easier if they all contain the same set of maps)
line.push_back( empty );
}
}
// Normalize line lengths again, in case some maps weren't updated:
NormalizeCSVRowLengths( lines );
// Sort the .CSV file so the maps with the 'worst' most recent data point are at the top
// (maps missing recent data points are sorted to the top, 'older' maps being biased higher)
vector< TrackSort_t > sortedLines;
for ( unsigned int i = 1; i < lines.size(); i++ )
{
int memSize = trackGPUMem ? GPU_MEM_SIZE : CPU_MEM_SIZE;
TrackSort_t trackSort = { i, memSize };
vector<string> &line = lines[ i ];
for ( unsigned int j = line.size()-1; j > 0; j-- )
{
if ( 1 == sscanf( line[j].c_str(), "%f", &trackSort.minFree ) )
{
trackSort.minFree -= memSize*(line.size()-(j+1)); // The older the data, the higher up the list it goes
sortedLines.push_back( trackSort );
break;
}
if ( j == 1 )
{
//printf( "ERROR: no valid datapoints for map '%s' in tracking .CSV file (%s)\n\n", line[0].c_str(), trackFile );
sortedLines.push_back( trackSort );
}
}
}
qsort( &sortedLines[0], sortedLines.size(), sizeof(TrackSort_t), TrackSortFunc );
// Write out the updated file
ofstream output;
output.open( trackFile );
if ( !output.is_open() )
{
printf( "ERROR: could write to tracking .CSV file (%s), aborting!\n\n", trackFile );
return false;
}
for ( unsigned int i = 0; i < lines.size(); i++ )
{
int nLine = ( i == 0 ) ? 0 : sortedLines[i-1].line;
vector<string> &line = lines[ nLine ];
for ( unsigned int j = 0; j < line.size(); j++ )
{
if ( j > 0 ) output.write( ",", 1 );
output.write( line[j].c_str(), line[j].size() );
}
if ( i < (lines.size()-1) ) output.write( "\n", 1 );
}
output.close();
return true;
}
bool FilterLogFile( CLogFile &logFile, CLogStats &filteredLogStats )
{
// TODO: these filters are very inconsistent, rework it to allow more arbitrary filter specification
// like "include:numplayers:1-4" or "exclude:numplayers:5-8"
// Filter based on aggregate log properties
if ( gConfig.dangerLimit && ( logFile.minFreeMem > gConfig.dangerLimit ) && ( logFile.minGPUFreeMem > gConfig.dangerLimit ) )
return false; // Log contains no entries with memory below the danger limit
if ( gConfig.isServer && !logFile.isServer )
return false; // Log doesn't contain any entries where the local machine is the Server
if ( gConfig.isClient && logFile.isServer )
return false; // Log contains entries where the local machine is the Server
if ( gConfig.isSplitscreen && !logFile.isSplitscreen )
return false; // Log doesn't contain any entries where the local machine is running Splitscreen
if ( gConfig.minPlayers && ( logFile.maxPlayers < gConfig.minPlayers ) )
return false; // Log contains no entries with the requisite number of players
ULONGLONG systemTime = SystemTime();
if ( gConfig.minAge && ( logFile.modifyTime > ( systemTime - gConfig.minAge*ONE_SECOND ) ) )
return false; // Too recent
if ( gConfig.maxAge && ( logFile.modifyTime < ( systemTime - gConfig.maxAge*ONE_SECOND ) ) )
return false; // Too old
if ( gConfig.isActive && !logFile.isActive )
return false; // Not active (not currently being written)
if ( ( gConfig.dvdHosted == DVDHOSTED_YES ) && ( logFile.headerInfo.dvdHosted != DVDHOSTED_YES ) )
return false; // Using the HDD
if ( ( gConfig.dvdHosted == DVDHOSTED_NO ) && ( logFile.headerInfo.dvdHosted != DVDHOSTED_NO ) )
return false; // Only using the DVD
if ( gConfig.languageFilter[0] )
{
if ( !strstr( logFile.headerInfo.language, gConfig.languageFilter ) )
return false; // Did not play in the specified language
}
if ( gConfig.platform != PLATFORM_UNKNOWN )
{
if ( logFile.headerInfo.platform != gConfig.platform )
return false; // Log file not for the specified platform
}
FilterBitfield mapFilter = logFile.mapList.SubstringToBitfield( gConfig.mapFilter );
if ( !mapFilter )
return false; // Log doesn't contain the specified map(s)
FilterBitfield playerFilter = logFile.playerList.SubstringToBitfield( gConfig.playerFilter );
if ( !playerFilter )
return false; // Log doesn't contain the specified player(s)
// The log's aggregate properties passed the filters, so now filter individual log entries
int dangerTime = -1;
int numPassingEntries = 0;
int duration = 0;
for ( int i = 0; i < logFile.entries.Count(); i++ )
{
CItem &item = logFile.entries[ i ];
if ( gConfig.dangerLimit && ( item.freeMem > gConfig.dangerLimit ) )
continue; // Filter out entries above the danger limit
if ( dangerTime == -1 )
dangerTime = item.time;
if ( gConfig.dangerTime && ( dangerTime > gConfig.dangerTime ) )
return false; // Log fails if no entries pass the danger limit soon enough
// NOTE: we don't filter individual entries by isServer, since we want to see the effects of playing maps AFTER being a server
if ( gConfig.isSplitscreen && ( item.numLocalPlayers <= 1 ) )
continue; // Filter out single-screen entries
if ( gConfig.isSinglescreen && ( item.numLocalPlayers >= 2 ) )
continue; // Filter out splitscreen entries
if ( gConfig.mapFilter[0] && !( mapFilter.IsSet( item.logMapIndex ) ) )
continue; // Filter out entries not on the specified map(s)
if ( gConfig.minPlayers && ( item.numPlayers < gConfig.minPlayers ) )
continue; // Filter out entries with too few players
if ( gConfig.maxPlayers && ( item.numPlayers > gConfig.maxPlayers ) )
continue; // Filter out entries with too many players
if ( gConfig.playerFilter[0] && !( playerFilter.Intersects( item.playerBitfield ) ) )
continue; // Filter out entries not containing the specified player(s)
// Ok, this entry passes all filters, so update the filtered aggregate stats
filteredLogStats.mapMin[ item.logMapIndex ] = min( item.freeMem, filteredLogStats.mapMin[ item.logMapIndex ] );
filteredLogStats.mapGPUMin[ item.logMapIndex ] = min( item.gpuFree, filteredLogStats.mapGPUMin[ item.logMapIndex ] );
filteredLogStats.mapAverage[ item.logMapIndex ] += item.freeMem;
filteredLogStats.mapSeconds[ item.logMapIndex ] += logFile.memorylogTick;
filteredLogStats.mapSamples[ item.logMapIndex ] ++;
duration = max( duration, item.time ); // Only update this for entries passing the filters
numPassingEntries++;
}
if ( numPassingEntries == 0 )
return false; // Log contains no lines passing the filters
if ( gConfig.duration && ( duration < gConfig.duration ) )
return false; // App run time too short (at filter-passing entries)
for ( int i = 0; i < CStringList::MAX_STRINGLIST_ENTRIES; i++ )
{
// Compute final per-map averages (for passing entries)
if ( filteredLogStats.mapSamples[ i ] > 0 )
filteredLogStats.mapAverage[ i ] /= filteredLogStats.mapSamples[ i ];
}
// We have a winner!
return true;
}
bool HasFilter( void )
{
// Are we filtering the logs in any meaningful way?
return ( gConfig.dangerLimit || gConfig.dangerTime || gConfig.duration || gConfig.minAge || gConfig.maxAge || gConfig.minPlayers || gConfig.maxPlayers ||
gConfig.isServer || gConfig.isClient || gConfig.isSplitscreen || gConfig.isSinglescreen || gConfig.isActive || ( gConfig.dvdHosted != DVDHOSTED_UNKNOWN ) ||
gConfig.mapFilter[0] || gConfig.playerFilter[0] || gConfig.languageFilter[0] || ( gConfig.platform != PLATFORM_UNKNOWN ) );
}
void PrintStats( CLogFiles &results )
{
int numMaps = gMapNames.Count();
if ( ( results.size() == 0 ) || ( numMaps == 0 ) )
{
printf( "Aggregate stats\n" );
printf( "---------------\n" );
printf( "No log files loaded\n" );
return;
}
if ( gConfig.isActive )
{
// Update which of our loaded logs are still active
for ( CLogFiles::iterator logIt = results.begin(); logIt != results.end(); logIt++ )
{
CLogFile &logFile = *logIt->second;
if ( logFile.isActive )
{
// If it was active when we loaded it, is it still?
WIN32_FIND_DATA fileData;
HANDLE handle = FindFirstFile( logFile.consoleLog.c_str(), &fileData );
if ( ( handle == INVALID_HANDLE_VALUE ) || !IsActive( WriteTime( fileData ) ) )
{
logFile.isActive = false;
}
}
}
}
typedef multimap<ULONGLONG, const CLogFile *> CFilteredLogs;
CFilteredLogs filteredLogs;
float *mapMin = new float[ numMaps ];
float *mapGPUMin = new float[ numMaps ];
float *mapAverage = new float[ numMaps ];
float *mapSeconds = new float[ numMaps ];
unsigned int *mapSamples = new unsigned int[ numMaps ];
unsigned int *mapLogs = new unsigned int[ numMaps ];
for ( int i = 0; i < numMaps; i++ )
{
mapMin[i] = CPU_MEM_SIZE;
mapGPUMin[i] = GPU_MEM_SIZE;
mapAverage[i] = 0.0f;
mapSeconds[i] = 0;
mapSamples[i] = 0;
mapLogs[i] = 0;
}
for ( CLogFiles::iterator logIt = results.begin(); logIt != results.end(); logIt++ )
{
CLogFile &logFile = *logIt->second;
CLogStats filteredLogStats;
// Run the log file through our filters
logFile.badMaps.Purge();
if ( FilterLogFile( logFile, filteredLogStats ) )
{
// Score! Use the data from this log to update per-map aggregate stats:
for ( int i = 0; i < numMaps; i++ )
{
int index = logFile.mapList.StringToInt( gMapNames[ i ] ); // Index into this logfile's map list
if ( ( index >= 0 ) && ( filteredLogStats.mapSamples[ index ] > 0 ) )
{
mapMin[ i ] = min( mapMin[ i ], filteredLogStats.mapMin[ index ] );
mapGPUMin[ i ] = min( mapGPUMin[ i ], filteredLogStats.mapGPUMin[ index ] );
mapAverage[ i ] += filteredLogStats.mapAverage[ index ]*filteredLogStats.mapSamples[ index ];
mapSeconds[ i ] += filteredLogStats.mapSeconds[ index ];
mapSamples[ i ] += filteredLogStats.mapSamples[ index ];
mapLogs[ i ]++;
// Build a list of the 'bad' maps in this log file, for spewage below:
MapMin_t mapMin = { i, filteredLogStats.mapMin[ index ] };
logFile.badMaps.AddToTail( mapMin );
}
}
// Add this to the list of logs to spew:
filteredLogs.insert( make_pair( logFile.modifyTime, &logFile ) );
}
}
if ( gConfig.trackFile[0] )
{
UpdateTrackFile( gConfig.trackFile, gConfig.trackColumn, mapSamples, mapMin, mapGPUMin );
}
// Spew the names of logs passing our filters (multimap-sorted by log last-modified time):
// TODO: will this spew scroll faster if we print multiple lines at a time?
if ( HasFilter() )
{
printf( "\n\nLog files which pass filters ( command-line \"%s\" )\n", gConfig.prevCommandLine );
printf( "----------------------------\n" );
CFilteredLogs::iterator it;
for ( it = filteredLogs.begin(); it != filteredLogs.end(); it++ )
{
static char headerString[1024];
SpewHeaderSummary( headerString, ARRAYSIZE( headerString ), it->second->headerInfo );
printf( " [%s] %s\n", headerString, it->second->consoleLog.c_str()[0] ? it->second->consoleLog.c_str() : it->second->memoryLog.c_str() );
// Spew a little summary of all offending maps in this log file (makes associating map issues w/ logs much faster)
printf( " " );
const CUtlVector<MapMin_t> &badMaps = it->second->badMaps;
for ( int i = 0; i < badMaps.Count(); i++ ) printf( " %4.1fMB:%-28s|", badMaps[i].minMem, gMapNames[ badMaps[i].map ] );
printf( "\n" );
}
printf( "\n" );
}
// NOTE: empty log files will always fail filters, unless you set "danger:512" (TODO: not sure this workaround works any more...)
printf( "Aggregate stats ( command-line \"%s\" )\n", gConfig.prevCommandLine );
printf( "--------------- (%d logs pass filters, of %d loaded)\n", filteredLogs.size(), results.size() );
int totalSeconds = 0;
if ( filteredLogs.size() )
{
for ( int i = 0; i < numMaps; i++ )
{
if ( mapSamples[ i ] )
{
mapAverage[ i ] /= mapSamples[ i ]; // Compute the final per-map average
totalSeconds += mapSeconds[ i ];
printf( "Map %-32s %4d logs, %6d samples, min %6.2fMB, average %6.2fMB\n",
gMapNames[ i ], mapLogs[ i ], mapSamples[ i ], mapMin[ i ], mapAverage[ i ] );
}
}
}
int days = (int)( totalSeconds / 86400 );
totalSeconds -= days*86400;
int hours = (int)( totalSeconds / 3600 );
totalSeconds -= hours*3600;
int mins = (int)( totalSeconds / 60 );
printf( " Total play time represented by these samples: %dd:%2dh:%2dm\n", days, hours, mins );
delete mapMin;
delete mapGPUMin;
delete mapAverage;
delete mapSamples;
delete mapSeconds;
delete mapLogs;
}
void InitMapHash( void )
{
for ( int i = 0; i < gNumIgnoreMaps; i++ ) AddNewMapName( gIgnoreMaps[ i ] );
for ( int i = 0; i < gNumKnownMaps; i++ ) AddNewMapName( gKnownMaps[ i ] );
}
const char *CleanPath( const char *path )
{
strncpy( gConfig.sourcePath, path, sizeof( gConfig.sourcePath ) );
strlwr( gConfig.sourcePath );
int pathLen = (int)strlen( gConfig.sourcePath );
if ( pathLen && ( gConfig.sourcePath[ pathLen - 1 ] != '\\' ) && ( gConfig.sourcePath[ pathLen - 1 ] != '/' ) )
{
strncat( gConfig.sourcePath, "\\", sizeof( gConfig.sourcePath ) );
}
return gConfig.sourcePath;
}
void Usage()
{
printf( "\n" );
printf( "memlog mines vxconsole logs for memory data\n" );
printf( "\n" );
printf( " USAGE: memlog [options] <folder>\n" );
printf( "\n" );
printf( "Input is a folder. memlog will convert all vxconsole logs in that folder into\n" );
printf( "memlog files (prefix 'memorylog_'). It will then output aggregate memory data for\n" );
printf( "those files.\n" );
printf( "\n" );
printf( "options:\n" );
printf( "[-c|console] run in console mode (see below)\n" );
printf( "[-r|recurse] recurse the input folder tree\n" );
printf( "[-u|update] reconvert vxconsole logs that have been updated\n" );
printf( "[-a|updateactive] reconvert vxconsole logs that are still being updated\n" );
printf( "[-f|force] reconvert all vxconsole logs\n" );
printf( "\n" );
printf( "tracking:\n" );
printf( "[-track:<file>] update a CSV file which tracks memory stats over time\n" );
printf( " the filename must end with one of these suffices:\n" );
printf( " '_MinFreeCPU' - track minimium free CPU memory\n" );
printf( " '_MinFreeGPU' - track minimium free GPU memory\n" );
printf( "[-trackcol:<name>] 'name' is the new column to add (rows are maps)\n" );
printf( "\n" );
printf( "analysis: (spews data passing these filters - all default to off)\n" );
printf( " ----the following options are applied per log entry----\n" );
printf( "[-danger:N] pass log entries in which memory is below N kilobytes\n" );
printf( "[-minplayers:N] pass log entries with at least this many concurrent players\n" );
printf( "[-maxplayers:N] pass log entries with at most this many concurrent players\n" );
printf( "[-issinglescreen] pass log entries in which the local box has 1 player\n" );
printf( "[-issplitscreen] pass log entries in which the local box has 2 players\n" );
printf( "[-map:<name>] pass log entries with map names containing this substring\n" );
printf( "[-player:<name>] pass log entries with player names containing this substring\n" );
printf( " ----the following options are applied per log file----\n" );
printf( "[-dangertime:N] pass logs dropping below 'danger' within this many minutes\n" );
printf( "[-duration:X] pass logs in which the timer reaches this many minutes\n" );
printf( "[-minage:N] pass logs updated at least this many hours ago\n" );
printf( "[-maxage:N] pass logs updated at most this many hours ago\n" );
printf( "[-isserver] pass logs in which the local box is a listen server at least once\n" );
printf( "[-isclient] pass logs in which the local box is NEVER a listen server\n" );
printf( "[-isactive] pass logs that are still being updated\n" );
printf( "[-isdvdonly] pass logs in which the local box is fully DVD hosted\n" );
printf( "[-isusinghdd] pass logs in which the local box is using the HDD\n" );
printf( "[-language:<name>] pass logs with the language containing this substring\n" );
printf( "[-platform:<name>] pass logs generated on this platform ('360', 'PS3', 'PC')\n" );
printf( "\n" );
printf( "\n" );
printf( "Console mode:\n" );
printf( "\n" );
printf( " USAGE: [options] [folder]\n" );
printf( "\n" );
printf( "In console mode, memlog keeps running until told to quit, allowing the user to\n" );
printf( "perform any number of log mining operations without having to re-load all the logs\n" );
printf( "each time. Commands in console mode are the same as the above command-line\n" );
printf( "options (without the leading '-'). Enter a set of commands, in any order (followed\n" );
printf( "by a folder path if appropriate), and hit enter. If you don't specify a path, the\n" );
printf( "most recently entered path will be used.\n" );
printf( "\n" );
printf( "console-mode-specific commands:\n" );
printf( "[load] load more memory logs, from the specified folder\n" );
printf( "[unload] unload memory logs in the specified folder\n" );
printf( "[unloadall] unload all memory logs\n" );
printf( "\n" );
}
void InitConfig( bool bCommandLine = false )
{
if ( bCommandLine )
{
// These persist after the first set of commands:
gConfig.sourcePath[0] = 0;
gConfig.consoleMode = false;
}
gConfig.recurse = false;
gConfig.update = false;
gConfig.updateActive = false;
gConfig.forceUpdate = false;
gConfig.load = false;
gConfig.unload = false;
gConfig.unloadAll = false;
gConfig.quitting = false;
gConfig.help = false;
// 'track' updates a memory tracking file, but only do it when asked
gConfig.trackFile[0] = 0;
gConfig.trackColumn[0] = 0;
gConfig.trackStat = TRACKSTAT_UNKNOWN;
// Default all filters off (no analysis)
gConfig.dangerLimit = 0;
gConfig.dangerTime = 0;
gConfig.duration = 0;
gConfig.minAge = 0;
gConfig.maxAge = 0;
gConfig.minPlayers = 0;
gConfig.maxPlayers = 0;
gConfig.isServer = false;
gConfig.isClient = false;
gConfig.isSplitscreen = false;
gConfig.isSinglescreen = false;
gConfig.isActive = false;
gConfig.dvdHosted = DVDHOSTED_UNKNOWN;
gConfig.mapFilter[0] = 0;
gConfig.playerFilter[0] = 0;
gConfig.languageFilter[0] = 0;
gConfig.platform = PLATFORM_UNKNOWN;
}
bool ParseOption( const char *option )
{
// Make everything lower-case for simplicity
strlwr( (char *)option );
if ( option[0] == '-' )
option++;
// Console mode
{
if ( !strcmp( option, "c" ) || !strcmp( option, "console" ) )
{
gConfig.consoleMode = true;
return true;
}
if ( !strcmp( option, "quit" ) || !strcmp( option, "exit" ) )
{
gConfig.quitting = true;
return true;
}
if ( !strcmp( option, "help" ) || !strcmp( option, "h" ) || !strcmp( option, "usage" ) || !strcmp( option, "?" ) )
{
gConfig.help = true;
return true;
}
}
// Data analysis (filters)
{
int dangerLimit;
if ( sscanf( option, "danger:%d", &dangerLimit ) == 1 )
{
if ( dangerLimit > 0 )
{
gConfig.dangerLimit = dangerLimit / 1024.0f; // KB -> MB
return true;
}
printf( "!'danger' must be > 0!\n" );
return false;
}
int dangerTime;
if ( sscanf( option, "dangertime:%d", &dangerTime ) == 1 )
{
if ( dangerTime > 0 )
{
gConfig.dangerTime = dangerTime*60; // Minutes
return true;
}
printf( "!'dangertime' must be > 0!\n" );
return false;
}
int duration;
if ( sscanf( option, "duration:%d", &duration ) == 1 )
{
if ( duration > 0 )
{
gConfig.duration = duration*60; // Minutes
return true;
}
printf( "!'duration' must be > 0!\n" );
return false;
}
int minAge;
if ( sscanf( option, "minage:%d", &minAge ) == 1 )
{
if ( minAge >= 0 )
{
gConfig.minAge = minAge*3600; // Hours
return true;
}
printf( "!'minage' must be >= 0!\n" );
return false;
}
int maxAge;
if ( sscanf( option, "maxage:%d", &maxAge ) == 1 )
{
if ( maxAge >= 0 )
{
gConfig.maxAge = maxAge*3600; // Hours
return true;
}
printf( "!'maxage' must be >= 0!\n" );
return false;
}
int minPlayers;
if ( sscanf( option, "minplayers:%d", &minPlayers ) == 1 )
{
if ( minPlayers >= 1 )
{
gConfig.minPlayers = minPlayers;
return true;
}
printf( "!'minplayers' must be >= 1!\n" );
return false;
}
int maxPlayers;
if ( sscanf( option, "maxplayers:%d", &maxPlayers ) == 1 )
{
if ( maxPlayers >= 1 )
{
gConfig.maxPlayers = maxPlayers;
return true;
}
printf( "!'maxplayers' must be >= 1!\n" );
return false;
}
if ( !strcmp( option, "isclient" ) )
{
if ( gConfig.isServer )
{
printf( "!'isclient' and 'isserver' are mutually exclusive!\n" );
return false;
}
gConfig.isClient = true;
return true;
}
if ( !strcmp( option, "isserver" ) )
{
if ( gConfig.isClient )
{
printf( "!'isclient' and 'isserver' are mutually exclusive!\n" );
return false;
}
gConfig.isServer = true;
return true;
}
if ( !strcmp( option, "issplitscreen" ) )
{
if ( gConfig.isSinglescreen )
{
printf( "!'issinglescreen' and 'issplitscreen' are mutually exclusive!\n" );
return false;
}
gConfig.isSplitscreen = true;
return true;
}
if ( !strcmp( option, "issinglescreen" ) )
{
if ( gConfig.isSplitscreen )
{
printf( "!'issinglescreen' and 'issplitscreen' are mutually exclusive!\n" );
return false;
}
gConfig.isSinglescreen = true;
return true;
}
if ( !strcmp( option, "isactive" ) )
{
gConfig.isActive = true;
return true;
}
if ( !strcmp( option, "isdvdonly" ) )
{
if ( gConfig.dvdHosted == DVDHOSTED_NO )
{
printf( "!'isdvdonly' and 'isusinghdd' are mutually exclusive!\n" );
return false;
}
gConfig.dvdHosted = DVDHOSTED_YES;
return true;
}
if ( !strcmp( option, "isusinghdd" ) )
{
if ( gConfig.dvdHosted == DVDHOSTED_YES )
{
printf( "!'isdvdonly' and 'isusinghdd' are mutually exclusive!\n" );
return false;
}
gConfig.dvdHosted = DVDHOSTED_NO;
return true;
}
char mapFilter[ FILTER_SIZE ] = "";
if ( sscanf( option, "map:%32s", mapFilter ) == 1 )
{
if ( mapFilter[0] )
{
strncpy( gConfig.mapFilter, mapFilter, sizeof( gConfig.mapFilter ) );
return true;
}
return false;
}
char playerFilter[ FILTER_SIZE ] = "";
if ( sscanf( option, "player:%32s", playerFilter ) == 1 )
{
if ( playerFilter[0] )
{
strncpy( gConfig.playerFilter, playerFilter, sizeof( gConfig.playerFilter ) );
return true;
}
return false;
}
char languageFilter[ FILTER_SIZE ] = "";
if ( sscanf( option, "language:%32s", languageFilter ) == 1 )
{
if ( languageFilter[0] )
{
strncpy( gConfig.languageFilter, languageFilter, sizeof( gConfig.languageFilter ) );
return true;
}
return false;
}
char platformFilter[ 16 ] = "";
if ( sscanf( option, "platform:%16s", platformFilter ) == 1 )
{
return StringToPlatformName( platformFilter, gConfig.platform );
}
}
// File processing
{
if ( !strcmp( option, "r" ) || !stricmp( option, "recurse" ) )
{
gConfig.recurse = true;
return true;
}
if ( !strcmp( option, "u" ) || !stricmp( option, "update" ) )
{
if ( gConfig.unload || gConfig.unloadAll )
{
printf( "!'update' is mututally exclusive with 'unload'!\n" );
return false;
}
gConfig.update = true;
gConfig.load = true; // Less confusing if update implies load
return true;
}
if ( !strcmp( option, "a" ) || !stricmp( option, "updateactive" ) )
{
if ( gConfig.unload || gConfig.unloadAll )
{
printf( "!'updateactive' is mututally exclusive with 'unload'!\n" );
return false;
}
gConfig.updateActive = true;
gConfig.update = true; // Less confusing if updateActive implies update
gConfig.load = true; // Less confusing if update implies load
return true;
}
if ( !strcmp( option, "f" ) || !stricmp( option, "force" ) )
{
// TODO: these error checks should really be a post-step - this depends on ordering (i.e. we don't get a message if force appears *before* unload)
if ( gConfig.unload || gConfig.unloadAll )
{
printf( "!'force' is mututally exclusive with 'unload'!\n" );
return false;
}
gConfig.load = true; // Less confusing if update implies load
gConfig.forceUpdate = true;
return true;
}
if ( !strcmp( option, "load" ) )
{
if ( gConfig.unload || gConfig.unloadAll )
{
printf( "!'load' is mututally exclusive with 'unload'!\n" );
return false;
}
gConfig.load = true;
return true;
}
if ( !strcmp( option, "unload" ) )
{
if ( gConfig.load )
{
printf( "!'load' is mututally exclusive with 'unload'!\n" );
return false;
}
gConfig.unload = true;
return true;
}
if ( !strcmp( option, "unloadall" ) )
{
if ( gConfig.load )
{
printf( "!'load' is mututally exclusive with 'unloadall'!\n" );
return false;
}
gConfig.unloadAll = true;
return true;
}
COMPILE_TIME_ASSERT( sizeof( gConfig.trackFile ) == MAX_PATH );
if ( 1 == sscanf( option, "track:%260s", gConfig.trackFile ) )
{
int nStrLen = strlen( gConfig.trackFile );
const char *ext = V_stristr( gConfig.trackFile, ".csv" );
if ( !ext || ( ( ext - &gConfig.trackFile[0] ) != ( nStrLen - 4 ) ) || ( nStrLen <= 4 ) )
{
printf( "!'track' must specify a .csv file!\n" );
return false;
}
// Filename suffix determines the stat tracked in the file
if ( V_stristr( gConfig.trackFile, "_MinFreeCPU" ) )
gConfig.trackStat = TRACKSTAT_MINFREE_CPU;
else if ( V_stristr( gConfig.trackFile, "_MinFreeGPU" ) )
gConfig.trackStat = TRACKSTAT_MINFREE_GPU;
else
{
printf( "!'track' .csv file must end with a valid stat type suffix!\n" );
return false;
}
return true;
}
COMPILE_TIME_ASSERT( sizeof( gConfig.trackColumn ) == 32 );
if ( 1 == sscanf( option, "trackcol:%32s", gConfig.trackColumn ) )
{
return true;
}
}
return false;
}
bool ParseCommandLine( int argc, _TCHAR* argv[], Config &config )
{
bool bCommandLine = true;
InitConfig( bCommandLine );
// Cache off the command-line:
gConfig.prevCommandLine[0] = 0;
for ( int i = 1; i < argc; i++ )
{
strcat( gConfig.prevCommandLine, argv[i] );
strcat( gConfig.prevCommandLine, " " );
}
int numOptions = 1;
while ( argv[numOptions] && argv[numOptions][0] == '-' )
{
if ( !ParseOption( argv[numOptions] ) )
{
printf( "ERROR: invalid command-line option '%s'!\n\n\n", argv[numOptions] );
Usage();
return false;
}
numOptions++;
}
if ( numOptions == argc )
{
if ( !gConfig.consoleMode )
{
printf( "ERROR: no folder path specified!\n\n\n" );
Usage();
return false;
}
CleanPath( "" );
}
else
{
gConfig.load = true;
CleanPath( argv[numOptions] );
}
return true;
}
void ExtractTokens( char *buffer, vector<string> &tokens )
{
char *context = NULL;
char *token = strtok_s( buffer, " ", &context );
while( token )
{
tokens.push_back( string( token ) );
token = strtok_s( NULL, " ", &context );
}
}
bool ParseConsoleCommand( void )
{
if ( !gConfig.consoleMode )
return false;
while( true )
{
// Loop until the user inputs a command without errors
bool bError = false;
// NOTE: this remembers the last folder path used
InitConfig();
printf( "\n\n\n\n" );
printf( "> current folder path: '%s'\n", gConfig.sourcePath[0] ? gConfig.sourcePath : "<none>" );
printf( "> Enter a command to process:\n" );
printf( ">\n" );
printf( "-> " );
char buffer[4*_MAX_PATH] = "";
// TODO: (?while debugging?) this interferes with command prompt cut'n'paste (QuickEdit gets around it):
cin.getline( buffer, sizeof( buffer ) );
strcpy( gConfig.prevCommandLine, buffer );
vector<string> commands;
ExtractTokens( buffer, commands );
if ( commands.size() < 1 )
bError = true;
for ( unsigned int i = 0; i < commands.size(); i++ )
{
string & command = commands[ i ];
if ( !ParseOption( command.c_str() ) )
{
// If parsing files, the last item should be the folder path
if ( ( i == ( commands.size() - 1 ) ) &&
( gConfig.load || gConfig.unload ) )
{
CleanPath( command.c_str() );
}
else
{
printf( "\nInvalid command '%s'\n\n", command.c_str() );
bError = true;
}
}
}
if ( ( gConfig.load || gConfig.unload ) && !gConfig.sourcePath[0] )
{
printf( "\nYou must specify a path in order to use '%s'\n\n", gConfig.load ? "load" : "unload" );
bError = true;
}
if ( gConfig.quitting )
return false;
if ( gConfig.help )
{
Usage();
}
else if ( !bError )
{
printf( ">\n" );
printf( "> current folder path: '%s'\n", gConfig.sourcePath[0] ? gConfig.sourcePath : "<none>" );
printf( "> Processing command...\n" );
printf( "\n" );
return true;
}
}
}
// NOTE: this app doesn't bother with little things like freeing memory - enjoy!
int _tmain(int argc, _TCHAR* argv[])
{
InitMapHash();
// Grab command-line options
if ( !ParseCommandLine( argc, argv, gConfig ) )
return 1;
CLogFiles results;
do
{
// Process log files
// TODO: aggregate error messages and spew them at the end, so it's easier to notice+read them (list of: " 'log' plus all errors for 'log' ", for every 'log' with errors)
ProcessLogFiles( gConfig.sourcePath, results );
// Print stats gathered from the log files
PrintStats( results );
}
// Continue processing commands (console mode) if requested
while( ParseConsoleCommand() );
// TODO: add a new option to filter on the 'memory dip' during a map - i.e. the biggest reduction in free memory during a map (always ignore the first ?2? entries for a given map - find the biggest dips to see if 2 is right... ideally, we want to synch up with charlie's numbers for this to be useful)
// TODO: quantize [MEMORYLOG] timestamps (set 'next time' rather than 'prev time' -> :00, :20, :40) so spew aligns for all clients in a game (well, it wouldn't really be synchronized, depending on how long they sat at their respective menus...)
// TODO: spew aggregate memlog data
// - worst-cases
// - av/min/max per map, per machine, globally
// probably want to ignore early high results, concentrate on longer-term numbers (after X times or minutes on a map)
// maybe spew results for "at least x minutes", for several different values of x (15, 30, 60, 90, 120...)
// - correlate mem with: play time, map loads, numplayers, listen Vs dedicated server,
// player join/quits, team death/restarts, campaign starts/ends, exit to menu...
// o load entries for a log file
// o create aggregates for a log file for various criteria
// o combine aggregates across all files
// TODO: would also like to parse SBH spew
// store values in the header (want maxima, per-alloc-size & per-heap)....
// - detect start of an SBH dump and write a func to process it (take note of Toms recent change to the spew - therell be two types of spew out there)
// think about detecting the main menu by looking for adjacent 'none' lines...
// - post-process the log and convert all reasonable 'none's into 'menu's
// - convert runs with a full minute of 'none' at either end
// - menu items must also be "client" and have zero players
if ( IsDebuggerPresent() )
{
printf( "\n\nPress any key to exit...\n" );
_getche();
}
return 0;
}