csgo/cstrike15_src/public/filegroup.h

//========= Copyright � 2006, Electonic Arts(C) 2006 - All Rights Reserved ============//

#ifndef FILEGROUP_H
#define FILEGROUP_H

#include <sys/stat.h>
#include "zlib/zlib.h"

#ifdef _PS3

// #ifndef _RETAIL
// #define TIME_FILE_OPERATIONS
// #endif

//#define MEMCMP_FILE_OPERATIONS
#include "tier0/platform.h"
#include <sys/synchronization.h>
#include <sys/types.h>
#include <sys/ppu_thread.h>
#include <sys/stat.h>
#endif //_PS3

#define FILEGROUP_FORMAT_ID "FGP"
#define PREVIOUS_FILEGROUP_FORMAT_VERSION 1 //TEMP: Used for backwards compatibility
#define FILEGROUP_FORMAT_VERSION 2 //Preload section enhancement
#define FILEGROUP_FORMAT_VERSION_COMPILED_DIRECTORY 3 //Preprocessed directory section
const int MAX_FILE_GROUPS = 7;
const int FILEGROUP_UNCOMPRESSED_BUFFER_SIZE = (8 * 1024);
const int FILEGROUP_READ_CHUNK_SIZE = (64 * 1024);
const int FILEGROUP_READ_STORE_SIZE = FILEGROUP_READ_CHUNK_SIZE * 2;
const int FILEGROUP_READ_SIGNAL_SIZE = FILEGROUP_READ_CHUNK_SIZE * 2;
const int FILEGROUP_READ_THREAD_BUFFER_SIZE = FILEGROUP_READ_CHUNK_SIZE * 4;

#define FILEGROUP_USE_HASH_DIRECTORY
#ifdef FILEGROUP_USE_HASH_DIRECTORY
const int FILEGROUP_DIRECTORY_BUCKETS = 256; //Must be power of 2
const int FILEGROUP_BUCKET_MOD = FILEGROUP_DIRECTORY_BUCKETS-1;
#endif

struct FileIdentifier
{
    char*        mName;
    unsigned int mNameId;
    bool         mNameIdSet;
};

struct DirectoryHeader
{
    char    mId[4];
    int     mVersion;
    int     mNumEntries;
    int     mNumPreloadEntries; //Added for FILEGROUP_FORMAT_VERSION 2
};

struct PreviousDirectoryHeader //TEMP: Used for backwards compatibility
{
    char    mId[4];
    int     mVersion;
    int     mNumEntries;
};

struct DirectoryEntry
{
    char	mName[MAX_PATH];
	size_t	mPosition;
	size_t	mLength;
    size_t  mCompressedLength; //Set to zero if compression not used
};

struct SmallDirectoryEntry
{
//	char	mName[116];
	unsigned int mNameId;
	size_t	mPosition;
	size_t	mLength;
	size_t  mCompressedLength; //Set to zero if compression not used
};

#ifdef _PS3

#ifdef MEMCMP_FILE_OPERATIONS
struct DirectoryEntryExtraInfo
{
    char mFullFileName[MAX_PATH];
};
#endif

struct FileGroupReadBuffer
{
    void                    AllocateBuffer(size_t bufferSize);
    void                    FreeBuffer();
    void                    CloseFile();
    void                    WaitOnDiskEjectRecovery();
    sys_ppu_thread_t        mReadThread;
    unsigned char*          mBuffer;
    size_t                  mBufferSize;
    size_t                  mReadChunkSize;
//    unsigned char           mBuffer[FILEGROUP_READ_THREAD_BUFFER_SIZE];
    int                     mFile;
    char                    mFileName[MAX_PATH]; //Need to store filegroup name so that file can be reopened following disk eject
    uint64_t                mFileBlockSize;
    uint64_t                mFileSectorSize;

    sys_lwmutex_t           mBufferMutex; //Used for all updates to the buffer data
    sys_lwmutex_attribute_t mBufferMutexAttr;
    sys_lwmutex_t           mNextConsumeByteMutex; //Used for synchronization of multiple consumer threads
    sys_lwmutex_attribute_t mNextConsumeByteMutexAttr;
	sys_lwmutex_t           mBufferStartEndPosMutex; //Used for synchronization of buffer start and end positions - don't want to update these while in use by consumer 
	sys_lwmutex_attribute_t mBufferStartEndPosMutexAttr;
    sys_mutex_t             mReadThreadSignalMutex;
    sys_mutex_attribute_t   mReadThreadSignalMutexAttr;
    sys_cond_t              mReadThreadSignal; //Signaled by consumer thread indicating that either:
                                               //- there is space for the read thread to carry on reading
                                               //- or the filegroup has been deleted
    sys_cond_attribute_t    mReadThreadSignalAttr;
    sys_lwmutex_t           mEjectRecoveryMutex; //Used to ensure only one thread performs the disk eject recovery procedure
    sys_lwmutex_attribute_t mEjectRecoveryMutexAttr;

    bool            mFileGroupDeleted;
	size_t          mNextConsumePos; //Position of next byte due for read by the consumer thread
	                                 //Read thread does not overwrite any bytes FILEGROUP_READ_STORE_SIZE bytes before this point.

    //unsigned char*  mNextConsumeByte; //Position of next byte due for read by the consumer thread
    //                                  //Read thread does not overwrite bytes beyond this point 


    size_t          mBufferStartPos; //Position in file corresponding to the beginning of the buffer
    size_t          mCurrentPos; //Current position in file
    unsigned char*  mCurrentByte; //Next byte to be read to in buffer
    size_t          mBufferEndPos; //Position in file corresponding to the end of the buffer (set to zero until initial population of buffer)
    bool            mAwaitingConsumer; //If set, indicates that the read thread is waiting for data to be read by the consumer thread             
#ifdef TIME_FILE_OPERATIONS
    system_time_t m_fReadTime;
    int           m_fReadBytes;
#endif

};

class CFileGroup;

class CFileGroupOpenedFile
{
public:
    CFileGroupOpenedFile();  
    ~CFileGroupOpenedFile();
#ifdef MEMCMP_FILE_OPERATIONS
    void Init(const SmallDirectoryEntry& dirEntry, CFileGroup* parentFileGroup, FileGroupReadBuffer* parentBuffer, char* fullFileName, bool preload);
#else
    void Init(const SmallDirectoryEntry& dirEntry, CFileGroup* parentFileGroup, FileGroupReadBuffer* parentBuffer, bool preload);
#endif
    virtual void FS_fseek( __int64 pos, int seekType );
    virtual long FS_ftell();
    virtual int FS_feof();
    virtual size_t FS_fread( void *dest, size_t destSize, size_t size);
    virtual char *FS_fgets( char *dest, int destSize );
//    const char* GetName() const {return mDirEntry->mName;}
    size_t GetPosition() const {return mDirEntry->mPosition;}
    size_t GetLength() const {return mDirEntry->mLength;}
    size_t GetCompressedLength() const {return mDirEntry->mCompressedLength;}
    bool IsPreloaded() const {return mPreloaded;}
    void FS_fclose();
#ifdef _DEBUG
    static int refCount;
#endif
private:
    void                Rewind(); 
    size_t              ReadFromCompressedData( void *dest, size_t readSize);
    size_t              ReadFromUncompressedData( void *dest, size_t readSize);
    void                TraceMemCopy(void* pDest, const void* pSource, size_t nBytes); 
    SmallDirectoryEntry const * mDirEntry;
    CFileGroup*         mParentFileGroup;
    size_t              mSeekPosIndicator;// This is current position as returned by FS_ftell.
    size_t              mActualPosIndicator;// This is the uncompressed position within the file corresponding to the last read.  
    size_t              mCompressedPosIndicator; //This is the number of compressed bytes which have been read.
    bool                mEof; //Set to true on attempting to read beyond the end of the file

    unsigned char       mUncompressedBuffer[FILEGROUP_UNCOMPRESSED_BUFFER_SIZE];
    //unsigned char*      mRemainingUncompressedBuffer; //Pointer to bytes within the uncompressed buffer which have yet to be read
    //size_t              mNumRemainingUncompressedBytes; //Number of bytes in the compressed buffer which have yet to be read
    int                 mUncompressedBufferStartPos;

    z_stream            mStrm; //Zlib stream
    FileGroupReadBuffer* mParentReadBuffer;
    bool                mPreloaded; 
#ifdef MEMCMP_FILE_OPERATIONS 
    int                 mOrdinaryFile;
#endif

};

class CFileGroupSystem
{
public:
    CFileGroupSystem();
    ~CFileGroupSystem();
    void Init();
    int AddFileGroup(const char* fileGroupName, bool useReadThread = true, int* filegroup_index = NULL, bool usepreload = false, int bufferSize = FILEGROUP_READ_CHUNK_SIZE, int readChunkSize = FILEGROUP_READ_CHUNK_SIZE); 
    void DeleteFileGroup(); 
    void DeleteFileGroup(int ID); 
    void DeleteFileGroup(CFileGroup* fileGroup); 
    void DeleteFileGroup(CFileGroup* fileGroup, int groupIndex); 
    int CurrentFileGroup();
    CFileGroupOpenedFile* FS_fopen( FileIdentifier *filename, const char *options, unsigned int flags, __int64 *size );
    CFileGroupOpenedFile* FS_fopen( int filegroup_index, FileIdentifier *filename, const char *options, unsigned int flags, __int64 *size );
    int FS_stat( FileIdentifier *path, struct stat *buf );
    int FS_stat( FileIdentifier *path, CellFsStat *buf );
    int FS_stat( int filegroup_index, FileIdentifier *path, struct stat *buf );
    int FS_stat( int filegroup_index, FileIdentifier *path, CellFsStat *buf );
    void Lock() {int ret = sys_lwmutex_lock( &mFileGroupSystemMutex, 0 ); if (ret != CELL_OK) printf("Error locking filegroup system mutex %d\n", ret);}
    void Unlock() {int ret = sys_lwmutex_unlock( &mFileGroupSystemMutex); if (ret != CELL_OK) printf("Error unlocking filegroup system mutex %d\n", ret);}
    void DecrementNumFileGroups(){m_numFileGroups--;}
    CFileGroupOpenedFile* GetOpenedFile(); //Get next available CFileGroupOpenedFile from pool
    void FreeOpenedFile(CFileGroupOpenedFile*); //Return CFileGroupOpenedFile to pool
#ifdef _DEBUG
    static int refCount;
#endif
private:
    CFileGroup* m_mapGroup;
    CFileGroup* m_shaderGroup;
    CFileGroup* m_fileGroups[MAX_FILE_GROUPS];
    int m_numFileGroups; 
    int m_currentFileGroup;
	sys_lwmutex_t mFileGroupSystemMutex;
    int m_lastGroupPopulated;

    //Pool of CFileGroupOpenedFile objects
    struct FileGroupFilePoolNode
    {
        CFileGroupOpenedFile thisFile;
        FileGroupFilePoolNode* nextNode;
    };
    FileGroupFilePoolNode* m_openedFilePool;
    FileGroupFilePoolNode* m_availableFileNodes;
    FileGroupFilePoolNode* m_openedFileNodes;
    int m_totalOpenedFiles;
    bool m_initComplete;
};

class CFileGroup
{
public:
    CFileGroup(CFileGroupSystem* pFs);
    ~CFileGroup();
    void Clear();
    int Populate(const char* fileGroupName, bool usepreload = false);
    CFileGroupOpenedFile* FS_fopen( FileIdentifier *pFileName, const char *pOptions, unsigned int flags, __int64 *size );
    int FS_stat( FileIdentifier *path, CellFsStat *buf );
    void DecrementOpenedCount();
    void IncrementOpenedCount(){mOpenedCount++;}
    int GetFile() const {return mReadBuffer.mFile;}
    size_t GetPosOffset() const {return mPosOffset;}
    int  GetOpenedCount(){return mOpenedCount;}
    void* GetPreloadData(){return mPreloadSection;}
    void FlagForDeletion(){mFlaggedForDeletion = true;}
    void Lock() {mFs->Lock();}
    void Unlock() {mFs->Unlock();}
    bool IsPopulated() {return (mReadBuffer.mFile);}
    void FreeOpenedFile(CFileGroupOpenedFile* freeFile){mFs->FreeOpenedFile(freeFile);}
    void StartReadThread();
    void StopReadThread();
    void TidyUpSynchObjects();
    sys_ppu_thread_t GetReadThread(){return mReadBuffer.mReadThread;}
    bool UsingReadThread(){return (mReadBuffer.mReadThread);}
//    unsigned char* GetIgnoreUncompressBuffer(){return mFs->GetIgnoreUncompressBuffer();}
    size_t Size(){return mFileStat.st_size;}
    bool HasBeenDeleted(){return mReadBuffer.mFileGroupDeleted;}
    void CloseFile(){mReadBuffer.CloseFile();}
    void AllocateReadBuffer(size_t bufSize){mReadBuffer.AllocateBuffer(bufSize);}
    void SetReadChunkSize(size_t readChunkSize){mReadBuffer.mReadChunkSize = readChunkSize;}
    void FreeReadBuffer(){mReadBuffer.FreeBuffer();}
#ifdef _DEBUG
    static int refCount;
#endif
#ifdef TIME_FILE_OPERATIONS
    void PrintFileGroupStats();
    void ResetFileTimings();
    system_time_t GetfReadTime(){return mReadBuffer.m_fReadTime;}
    int GetfReadBytes(){return mReadBuffer.m_fReadBytes;}
    void IncrReadTime(system_time_t timeVal){mReadBuffer.m_fReadTime += timeVal;}
    void IncrReadBytes(int bytesVal){mReadBuffer.m_fReadBytes += bytesVal; /*if(mReadBuffer.m_fReadBytes%10485760 == 0) PrintFileGroupStats();*/};
    system_time_t m_memCopyTime;
    system_time_t m_uncompressTime;
    int m_fsReads;
    int m_fgets;
    int m_uncompressCalls;
    int m_rewinds;
    int m_fileJumps;
    int m_bytesfRead;
    int m_bytesReadFromFileByConsumer;
    int m_bytesReadFromBuffer;
#endif
private:

    CellFsErrno PopulateDirectory(DirectoryHeader* hdr, int* entriesPopulated);
    int FindFile(FileIdentifier *pFileName);
    char* ReformatFileName(char* inputFileName);
	unsigned int HashFileName(const char * fName);
    SmallDirectoryEntry*     mDirectoryEntries;
    unsigned int		     mNumDirectoryEntries;
    void*                    mPreloadSection;
    int                      mPreloadEntries;
#ifdef FILEGROUP_USE_HASH_DIRECTORY
    struct HashBucket
    {
        unsigned int mPosition;
        unsigned int mCount;
    };
    unsigned int*            mHashDirectory; //ordered into buckets
    HashBucket               mHashBuckets[FILEGROUP_DIRECTORY_BUCKETS]; 
#endif
	unsigned int		     mLastOpenedDirectoryIndex;
    unsigned int		     mLastFailedId;
    CellFsStat                      mFileStat;
    size_t                          mPosOffset; //Position at which the data starts
    CFileGroupSystem*               mFs;
    int                             mOpenedCount;
    bool                            mFlaggedForDeletion;
    FileGroupReadBuffer             mReadBuffer;
#ifdef MEMCMP_FILE_OPERATIONS 
    DirectoryEntryExtraInfo*    mDirectoryExtraInfo;
#endif
};

#endif //_PS3
#endif //FILEGROUP_H