//========= Copyright � 1996-2005, Valve Corporation, All rights reserved. ============// // // Purpose: // //=============================================================================// #if defined( _WIN32 ) && !defined( _X360 ) #define WIN_32_LEAN_AND_MEAN #include #define VA_COMMIT_FLAGS MEM_COMMIT #define VA_RESERVE_FLAGS MEM_RESERVE #elif defined( _X360 ) #define VA_COMMIT_FLAGS (MEM_COMMIT|MEM_NOZERO|MEM_LARGE_PAGES) #define VA_RESERVE_FLAGS (MEM_RESERVE|MEM_LARGE_PAGES) #elif defined( _PS3 ) #include "sys/memory.h" #include "sys/mempool.h" #include "sys/process.h" #include #endif #include "tier0/dbg.h" #include "memstack.h" #include "utlmap.h" #include "tier0/memdbgon.h" #ifdef _WIN32 #pragma warning(disable:4073) #pragma init_seg(lib) #endif static volatile bool bSpewAllocations = false; // TODO: Register CMemoryStacks with g_pMemAlloc, so it can spew a summary //----------------------------------------------------------------------------- MEMALLOC_DEFINE_EXTERNAL_TRACKING(CMemoryStack); //----------------------------------------------------------------------------- void PrintStatus( void* p ) { CMemoryStack* pMemoryStack = (CMemoryStack*)p; pMemoryStack->PrintContents(); } CMemoryStack::CMemoryStack() : m_pNextAlloc( NULL ) , m_pCommitLimit( NULL ) , m_pAllocLimit( NULL ) , m_pHighestAllocLimit( NULL ) , m_pBase( NULL ) , m_bRegisteredAllocation( false ) , m_maxSize( 0 ) , m_alignment( 16 ) #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE , m_commitIncrement( 0 ) , m_minCommit( 0 ) #ifdef _PS3 , m_pVirtualMemorySection( NULL ) #endif #endif { AddMemoryInfoCallback( this ); m_pszAllocOwner = strdup( "CMemoryStack unattributed" ); } //------------------------------------- CMemoryStack::~CMemoryStack() { if ( m_pBase ) Term(); RemoveMemoryInfoCallback( this ); free( m_pszAllocOwner ); } //------------------------------------- bool CMemoryStack::Init( const char *pszAllocOwner, unsigned maxSize, unsigned commitIncrement, unsigned initialCommit, unsigned alignment ) { Assert( !m_pBase ); m_bPhysical = false; m_maxSize = maxSize; m_alignment = AlignValue( alignment, 4 ); Assert( m_alignment == alignment ); Assert( m_maxSize > 0 ); SetAllocOwner( pszAllocOwner ); #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE #ifdef _PS3 // Memory can only be committed in page-size increments on PS3 static const unsigned PS3_PAGE_SIZE = 64*1024; if ( commitSize < PS3_PAGE_SIZE ) commitSize = PS3_PAGE_SIZE; #endif if ( commitIncrement != 0 ) { m_commitIncrement = commitIncrement; } unsigned pageSize; #ifdef _PS3 pageSize = PS3_PAGE_SIZE; #elif defined( _X360 ) pageSize = 64 * 1024; #else SYSTEM_INFO sysInfo; GetSystemInfo( &sysInfo ); Assert( !( sysInfo.dwPageSize & (sysInfo.dwPageSize-1)) ); pageSize = sysInfo.dwPageSize; #endif if ( m_commitIncrement == 0 ) { m_commitIncrement = pageSize; } else { m_commitIncrement = AlignValue( m_commitIncrement, pageSize ); } m_maxSize = AlignValue( m_maxSize, m_commitIncrement ); Assert( m_maxSize % pageSize == 0 && m_commitIncrement % pageSize == 0 && m_commitIncrement <= m_maxSize ); #ifdef _WIN32 m_pBase = (unsigned char *)VirtualAlloc( NULL, m_maxSize, VA_RESERVE_FLAGS, PAGE_NOACCESS ); #else m_pVirtualMemorySection = g_pMemAlloc->AllocateVirtualMemorySection( m_maxSize ); if ( !m_pVirtualMemorySection ) { Warning( "AllocateVirtualMemorySection failed( size=%d )\n", m_maxSize ); Assert( 0 ); m_pBase = NULL; } else { m_pBase = ( byte* ) m_pVirtualMemorySection->GetBaseAddress(); } #endif if ( !m_pBase ) { #if !defined( NO_MALLOC_OVERRIDE ) g_pMemAlloc->OutOfMemory(); #endif return false; } m_pCommitLimit = m_pNextAlloc = m_pBase; if ( initialCommit ) { initialCommit = AlignValue( initialCommit, m_commitIncrement ); Assert( initialCommit <= m_maxSize ); bool bInitialCommitSucceeded = false; #ifdef _WIN32 bInitialCommitSucceeded = !!VirtualAlloc( m_pCommitLimit, initialCommit, VA_COMMIT_FLAGS, PAGE_READWRITE ); #else m_pVirtualMemorySection->CommitPages( m_pCommitLimit, initialCommit ); bInitialCommitSucceeded = true; #endif if ( !bInitialCommitSucceeded ) { #if !defined( NO_MALLOC_OVERRIDE ) g_pMemAlloc->OutOfMemory( initialCommit ); #endif return false; } m_minCommit = initialCommit; m_pCommitLimit += initialCommit; RegisterAllocation(); } #else m_pBase = (byte*)MemAlloc_AllocAligned( m_maxSize, alignment ? alignment : 1 ); m_pNextAlloc = m_pBase; m_pCommitLimit = m_pBase + m_maxSize; #endif m_pHighestAllocLimit = m_pNextAlloc; m_pAllocLimit = m_pBase + m_maxSize; return ( m_pBase != NULL ); } //------------------------------------- #ifdef _GAMECONSOLE bool CMemoryStack::InitPhysical( const char *pszAllocOwner, uint size, uint nBaseAddrAlignment, uint alignment, uint32 nFlags ) { m_bPhysical = true; m_maxSize = m_commitIncrement = size; m_alignment = AlignValue( alignment, 4 ); SetAllocOwner( pszAllocOwner ); #ifdef _X360 int flags = PAGE_READWRITE | nFlags; if ( size >= 16*1024*1024 ) { flags |= MEM_16MB_PAGES; } else { flags |= MEM_LARGE_PAGES; } m_pBase = (unsigned char *)XPhysicalAlloc( m_maxSize, MAXULONG_PTR, nBaseAddrAlignment, flags ); #elif defined (_PS3) m_pBase = (byte*)nFlags; m_pBase = (byte*)AlignValue( (uintp)m_pBase, m_alignment ); #else #pragma error #endif Assert( m_pBase ); m_pNextAlloc = m_pBase; m_pCommitLimit = m_pBase + m_maxSize; m_pAllocLimit = m_pBase + m_maxSize; m_pHighestAllocLimit = m_pNextAlloc; RegisterAllocation(); return ( m_pBase != NULL ); } #endif //------------------------------------- void CMemoryStack::Term() { FreeAll(); if ( m_pBase ) { #ifdef _GAMECONSOLE if ( m_bPhysical ) { #if defined( _X360 ) XPhysicalFree( m_pBase ); #elif defined( _PS3 ) #else #pragma error #endif m_pCommitLimit = m_pBase = NULL; m_maxSize = 0; RegisterDeallocation(true); m_bPhysical = false; return; } #endif // _GAMECONSOLE #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE #if defined(_WIN32) VirtualFree( m_pBase, 0, MEM_RELEASE ); #else m_pVirtualMemorySection->Release(); m_pVirtualMemorySection = NULL; #endif #else MemAlloc_FreeAligned( m_pBase ); #endif m_pBase = NULL; // Zero these variables to avoid getting misleading mem_dump // results when m_pBase is NULL. m_pNextAlloc = NULL; m_pCommitLimit = NULL; m_pHighestAllocLimit = NULL; m_maxSize = 0; RegisterDeallocation(true); } } //------------------------------------- int CMemoryStack::GetSize() const { if ( m_bPhysical ) return m_maxSize; #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE return m_pCommitLimit - m_pBase; #else return m_maxSize; #endif } //------------------------------------- bool CMemoryStack::CommitTo( byte *pNextAlloc ) RESTRICT { if ( m_bPhysical ) { return NULL; } #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE unsigned char * pNewCommitLimit = AlignValue( pNextAlloc, m_commitIncrement ); ptrdiff_t commitIncrement = pNewCommitLimit - m_pCommitLimit; if( m_pCommitLimit + commitIncrement > m_pAllocLimit ) { #if !defined( NO_MALLOC_OVERRIDE ) g_pMemAlloc->OutOfMemory( commitIncrement ); #endif return false; } if ( pNewCommitLimit > m_pCommitLimit ) { RegisterDeallocation(false); bool bAllocationSucceeded = false; #ifdef _WIN32 bAllocationSucceeded = !!VirtualAlloc( m_pCommitLimit, commitIncrement, VA_COMMIT_FLAGS, PAGE_READWRITE ); #else bAllocationSucceeded = m_pVirtualMemorySection->CommitPages( m_pCommitLimit, commitIncrement ); #endif if ( !bAllocationSucceeded ) { #if !defined( NO_MALLOC_OVERRIDE ) g_pMemAlloc->OutOfMemory( commitIncrement ); #endif return false; } m_pCommitLimit = pNewCommitLimit; RegisterAllocation(); } else if ( pNewCommitLimit < m_pCommitLimit ) { if ( m_pNextAlloc > pNewCommitLimit ) { Warning( "ATTEMPTED TO DECOMMIT OWNED MEMORY STACK SPACE\n" ); pNewCommitLimit = AlignValue( m_pNextAlloc, m_commitIncrement ); } if ( pNewCommitLimit < m_pCommitLimit ) { RegisterDeallocation(false); ptrdiff_t decommitIncrement = m_pCommitLimit - pNewCommitLimit; #ifdef _WIN32 VirtualFree( pNewCommitLimit, decommitIncrement, MEM_DECOMMIT ); #else m_pVirtualMemorySection->DecommitPages( pNewCommitLimit, decommitIncrement ); #endif m_pCommitLimit = pNewCommitLimit; RegisterAllocation(); } } return true; #else return false; #endif } // Identify the owner of this memory stack's memory void CMemoryStack::SetAllocOwner( const char *pszAllocOwner ) { if ( !pszAllocOwner || !Q_strcmp( m_pszAllocOwner, pszAllocOwner ) ) return; free( m_pszAllocOwner ); m_pszAllocOwner = strdup( pszAllocOwner ); } void CMemoryStack::RegisterAllocation() { // 'physical' allocations on PS3 come from RSX local memory, so we don't count them here: if ( IsPS3() && m_bPhysical ) return; if ( GetSize() ) { if ( m_bRegisteredAllocation ) Warning( "CMemoryStack: ERROR - mismatched RegisterAllocation/RegisterDeallocation!\n" ); // NOTE: we deliberately don't use MemAlloc_RegisterExternalAllocation. CMemoryStack needs to bypass 'GetActualDbgInfo' // due to the way it allocates memory: there's just one representative memory address (m_pBase), it grows at unpredictable // times (in CommitTo, not every Alloc call) and it is freed en-masse (instead of freeing each individual allocation). MemAlloc_RegisterAllocation( m_pszAllocOwner, 0, GetSize(), GetSize(), 0 ); } m_bRegisteredAllocation = true; // Temp memorystack spew: very useful when we crash out of memory if ( IsGameConsole() && bSpewAllocations ) Msg( "CMemoryStack: %4.1fMB (%s)\n", GetSize()/(float)(1024*1024), m_pszAllocOwner ); } void CMemoryStack::RegisterDeallocation( bool bShouldSpewSize ) { // 'physical' allocations on PS3 come from RSX local memory, so we don't count them here: if ( IsPS3() && m_bPhysical ) return; if ( GetSize() ) { if ( !m_bRegisteredAllocation ) Warning( "CMemoryStack: ERROR - mismatched RegisterAllocation/RegisterDeallocation!\n" ); MemAlloc_RegisterDeallocation( m_pszAllocOwner, 0, GetSize(), GetSize(), 0 ); } m_bRegisteredAllocation = false; // Temp memorystack spew: very useful when we crash out of memory if ( bShouldSpewSize && IsGameConsole() && bSpewAllocations ) Msg( "CMemoryStack: %4.1fMB (%s)\n", GetSize()/(float)(1024*1024), m_pszAllocOwner ); } //------------------------------------- void CMemoryStack::FreeToAllocPoint( MemoryStackMark_t mark, bool bDecommit ) { mark = AlignValue( mark, m_alignment ); byte *pAllocPoint = m_pBase + mark; Assert( pAllocPoint >= m_pBase && pAllocPoint <= m_pNextAlloc ); if ( pAllocPoint >= m_pBase && pAllocPoint <= m_pNextAlloc ) { m_pNextAlloc = pAllocPoint; #ifdef MEMSTACK_VIRTUAL_MEMORY_AVAILABLE if ( bDecommit && !m_bPhysical ) { CommitTo( MAX( m_pNextAlloc, (m_pBase + m_minCommit) ) ); } #endif } } //------------------------------------- void CMemoryStack::FreeAll( bool bDecommit ) { if ( m_pBase && ( m_pBase < m_pCommitLimit ) ) { FreeToAllocPoint( 0, bDecommit ); } } //------------------------------------- void CMemoryStack::Access( void **ppRegion, unsigned *pBytes ) { *ppRegion = m_pBase; *pBytes = ( m_pNextAlloc - m_pBase); } const char* CMemoryStack::GetMemoryName() const { return m_pszAllocOwner; } size_t CMemoryStack::GetAllocatedBytes() const { return GetUsed(); } size_t CMemoryStack::GetCommittedBytes() const { return GetSize(); } size_t CMemoryStack::GetReservedBytes() const { return GetMaxSize(); } size_t CMemoryStack::GetHighestBytes() const { size_t highest = m_pHighestAllocLimit - m_pBase; return highest; } //------------------------------------- void CMemoryStack::PrintContents() const { size_t highest = m_pHighestAllocLimit - m_pBase; #ifdef PLATFORM_WINDOWS_PC MEMORY_BASIC_INFORMATION info; char moduleName[260]; strcpy( moduleName, "unknown module" ); // Because this code is statically linked into each DLL, this function and the PrintStatus // function will be in the DLL that constructed the CMemoryStack object. We can then // retrieve the DLL name to give slightly more verbose memory dumps. if ( VirtualQuery( &PrintStatus, &info, sizeof( info ) ) == sizeof( info ) ) { GetModuleFileName( (HMODULE) info.AllocationBase, moduleName, _countof( moduleName ) ); moduleName[ _countof( moduleName )-1 ] = 0; } Msg( "CMemoryStack %s in %s\n", m_pszAllocOwner, moduleName ); #else Msg( "CMemoryStack %s\n", m_pszAllocOwner ); #endif Msg( " Total used memory: %d KB\n", GetUsed() / 1024 ); Msg( " Total committed memory: %d KB\n", GetSize() / 1024 ); Msg( " Max committed memory: %u KB out of %d KB\n", (unsigned)highest / 1024, GetMaxSize() / 1024 ); } #ifdef _X360 //----------------------------------------------------------------------------- // // A memory stack used for allocating physical memory on the 360 (can't commit/decommit) // //----------------------------------------------------------------------------- MEMALLOC_DEFINE_EXTERNAL_TRACKING(CPhysicalMemoryStack); //----------------------------------------------------------------------------- // Constructor, destructor //----------------------------------------------------------------------------- CPhysicalMemoryStack::CPhysicalMemoryStack() : m_nAlignment( 16 ), m_nAdditionalFlags( 0 ), m_nUsage( 0 ), m_nPeakUsage( 0 ), m_pLastAllocedChunk( NULL ), m_nFirstAvailableChunk( 0 ), m_nChunkSizeInBytes( 0 ), m_ExtraChunks( 32, 32 ), m_nFramePeakUsage( 0 ) { m_InitialChunk.m_pBase = NULL; m_InitialChunk.m_pNextAlloc = NULL; m_InitialChunk.m_pAllocLimit = NULL; } CPhysicalMemoryStack::~CPhysicalMemoryStack() { Term(); } //----------------------------------------------------------------------------- // Init, shutdown //----------------------------------------------------------------------------- bool CPhysicalMemoryStack::Init( size_t nChunkSizeInBytes, size_t nAlignment, int nInitialChunkCount, uint32 nAdditionalFlags ) { Assert( !m_InitialChunk.m_pBase ); m_pLastAllocedChunk = NULL; m_nAdditionalFlags = nAdditionalFlags; m_nFirstAvailableChunk = 0; m_nUsage = 0; m_nFramePeakUsage = 0; m_nPeakUsage = 0; m_nAlignment = AlignValue( nAlignment, 4 ); // Chunk size must be aligned to the 360 page size size_t nInitMemorySize = nChunkSizeInBytes * nInitialChunkCount; nChunkSizeInBytes = AlignValue( nChunkSizeInBytes, 64 * 1024 ); m_nChunkSizeInBytes = nChunkSizeInBytes; // Fix up initial chunk count to get at least as much memory as requested // based on changes to the chunk size owing to page alignment issues nInitialChunkCount = ( nInitMemorySize + nChunkSizeInBytes - 1 ) / nChunkSizeInBytes; int nFlags = PAGE_READWRITE | nAdditionalFlags; int nAllocationSize = m_nChunkSizeInBytes * nInitialChunkCount; if ( nAllocationSize >= 16*1024*1024 ) { nFlags |= MEM_16MB_PAGES; } else { nFlags |= MEM_LARGE_PAGES; } m_InitialChunk.m_pBase = (uint8*)XPhysicalAlloc( nAllocationSize, MAXULONG_PTR, 0, nFlags ); if ( !m_InitialChunk.m_pBase ) { m_InitialChunk.m_pNextAlloc = m_InitialChunk.m_pAllocLimit = NULL; g_pMemAlloc->OutOfMemory(); return false; } m_InitialChunk.m_pNextAlloc = m_InitialChunk.m_pBase; m_InitialChunk.m_pAllocLimit = m_InitialChunk.m_pBase + nAllocationSize; MemAlloc_RegisterExternalAllocation( CPhysicalMemoryStack, m_InitialChunk.m_pBase, XPhysicalSize( m_InitialChunk.m_pBase ) ); return true; } void CPhysicalMemoryStack::Term() { FreeAll(); if ( m_InitialChunk.m_pBase ) { MemAlloc_RegisterExternalDeallocation( CPhysicalMemoryStack, m_InitialChunk.m_pBase, XPhysicalSize( m_InitialChunk.m_pBase ) ); XPhysicalFree( m_InitialChunk.m_pBase ); m_InitialChunk.m_pBase = m_InitialChunk.m_pNextAlloc = m_InitialChunk.m_pAllocLimit = NULL; } } //----------------------------------------------------------------------------- // Returns the total allocation size //----------------------------------------------------------------------------- size_t CPhysicalMemoryStack::GetSize() const { size_t nBaseSize = (intp)m_InitialChunk.m_pAllocLimit - (intp)m_InitialChunk.m_pBase; return nBaseSize + m_nChunkSizeInBytes * m_ExtraChunks.Count(); } //----------------------------------------------------------------------------- // Allocate from the 'overflow' buffers, only happens if the initial allocation // isn't good enough //----------------------------------------------------------------------------- void *CPhysicalMemoryStack::AllocFromOverflow( size_t nSizeInBytes ) { // Completely full chunks are moved to the front and skipped int nCount = m_ExtraChunks.Count(); for ( int i = m_nFirstAvailableChunk; i < nCount; ++i ) { PhysicalChunk_t &chunk = m_ExtraChunks[i]; // Here we can check if a chunk is full and move it to the head // of the list. We can't do it immediately *after* allocation // because something may later free up some of the memory if ( chunk.m_pNextAlloc == chunk.m_pAllocLimit ) { if ( i > 0 ) { m_ExtraChunks.FastRemove( i ); m_ExtraChunks.InsertBefore( 0 ); } ++m_nFirstAvailableChunk; continue; } void *pResult = chunk.m_pNextAlloc; uint8 *pNextAlloc = chunk.m_pNextAlloc + nSizeInBytes; if ( pNextAlloc > chunk.m_pAllocLimit ) continue; chunk.m_pNextAlloc = pNextAlloc; m_pLastAllocedChunk = &chunk; return pResult; } // No extra chunks to use; add a new one int i = m_ExtraChunks.AddToTail(); PhysicalChunk_t &chunk = m_ExtraChunks[i]; int nFlags = PAGE_READWRITE | MEM_LARGE_PAGES | m_nAdditionalFlags; chunk.m_pBase = (uint8*)XPhysicalAlloc( m_nChunkSizeInBytes, MAXULONG_PTR, 0, nFlags ); if ( !chunk.m_pBase ) { chunk.m_pNextAlloc = chunk.m_pAllocLimit = NULL; m_pLastAllocedChunk = NULL; g_pMemAlloc->OutOfMemory(); return NULL; } MemAlloc_RegisterExternalAllocation( CPhysicalMemoryStack, chunk.m_pBase, XPhysicalSize( chunk.m_pBase ) ); m_pLastAllocedChunk = &chunk; chunk.m_pNextAlloc = chunk.m_pBase + nSizeInBytes; chunk.m_pAllocLimit = chunk.m_pBase + m_nChunkSizeInBytes; return chunk.m_pBase; } //----------------------------------------------------------------------------- // Allows us to free a portion of the previous allocation //----------------------------------------------------------------------------- void CPhysicalMemoryStack::FreeToAllocPoint( MemoryStackMark_t mark, bool bUnused ) { mark = AlignValue( mark, m_nAlignment ); uint8 *pAllocPoint = m_pLastAllocedChunk->m_pBase + mark; Assert( pAllocPoint >= m_pLastAllocedChunk->m_pBase && pAllocPoint <= m_pLastAllocedChunk->m_pNextAlloc ); if ( pAllocPoint >= m_pLastAllocedChunk->m_pBase && pAllocPoint <= m_pLastAllocedChunk->m_pNextAlloc ) { m_nUsage -= (intp)m_pLastAllocedChunk->m_pNextAlloc - (intp)pAllocPoint; m_pLastAllocedChunk->m_pNextAlloc = pAllocPoint; } } //----------------------------------------------------------------------------- // Free overflow buffers, mark initial buffer as empty //----------------------------------------------------------------------------- void CPhysicalMemoryStack::FreeAll( bool bUnused ) { m_nUsage = 0; m_nFramePeakUsage = 0; m_InitialChunk.m_pNextAlloc = m_InitialChunk.m_pBase; m_pLastAllocedChunk = NULL; m_nFirstAvailableChunk = 0; int nCount = m_ExtraChunks.Count(); for ( int i = 0; i < nCount; ++i ) { PhysicalChunk_t &chunk = m_ExtraChunks[i]; MemAlloc_RegisterExternalDeallocation( CPhysicalMemoryStack, chunk.m_pBase, XPhysicalSize( chunk.m_pBase ) ); XPhysicalFree( chunk.m_pBase ); } m_ExtraChunks.RemoveAll(); } //------------------------------------- void CPhysicalMemoryStack::PrintContents() { Msg( "Total used memory: %8d\n", GetUsed() ); Msg( "Peak used memory: %8d\n", GetPeakUsed() ); Msg( "Total allocated memory: %8d\n", GetSize() ); } #endif // _X360