/*
 * Automatic language and codepage detector
 * 
 * Bob Powell, 2/97
 * Copyright (C) 1996, 1997, Microsoft Corp.  All rights reserved.
 */

#ifdef  __cplusplus

#include <wtypes.h>
#include <limits.h>

#include "lcdetect.h"
#include "lccommon.h"

#include <qsort.h>


// Turn this on in SOURCES to enable debug output
#ifdef DEBUG_LCDETECT
#include <stdio.h>
extern int g_fDebug;
#define debug(x) { if (g_fDebug) { x; }}
#define unmapch(x) ((x) >= 2 ? (x)+'a'-2 : ' ')
#else
#define debug(x)
#endif

class LCDetect;
typedef LCDetect *PLCDetect;

class Language;
class Language7Bit;
class Language8Bit;
class LanguageUnicode;
typedef Language *PLanguage;
typedef Language7Bit *PLanguage7Bit;
typedef Language8Bit *PLanguage8Bit;
typedef LanguageUnicode *PLanguageUnicode;

class CScore;
class CScores;

/****************************************************************/

#define MAXSCORES 50            // Max possible simultaneous # of scores

#define MINRAWSCORE 100         // Score threshhold (weight * char count) 
                                // for further processing

/****************************************************************/

// Histograms

// A histogram stores an array of n-gram occurrence counts.  
// HElt stores the count, at present this is an unsigned char.

// The in-memory structure is similar to the file.
// The histogram array pointers m_panElts point into the mapped file image.

class Histogram {

public:
    Histogram (const PFileHistogramSection pHS, const PHIdx pMap);
    Histogram (const Histogram &H, const PHIdx pMap);
    virtual ~Histogram (void);

    DWORD Validate (DWORD nBytes) const;

    UCHAR Dimensionality (void) { return m_nDimensionality; }
    UCHAR EdgeSize (void) { return m_nEdgeSize; }
    USHORT CodePage (void) { return m_nCodePage; }
    USHORT GetRangeID (void) { return m_nRangeID; }
    USHORT NElts (void) { return m_nElts; }
    PHIdx GetMap (void) { return m_pMap; }

    HElt Ref (USHORT i1) const { return m_panElts[i1]; }
    HElt Ref (UCHAR i1, UCHAR i2) const {
        return m_panElts[(i1 * m_nEdgeSize) + i2]; }
    HElt Ref (UCHAR i1, UCHAR i2, UCHAR i3) const {
        return m_panElts[((i1 * m_nEdgeSize) + i2) * m_nEdgeSize + i3]; }

    HElt *Array (void) { return m_panElts; }

protected:
    UCHAR m_nDimensionality;        // 1=unigram, 2=digram etc.
    UCHAR m_nEdgeSize;              // edge size (is a function of char map)
    union {
        USHORT m_nCodePage;         // For 7 and 8-bit, is code page
        USHORT m_nRangeID;          // For Unicode, is sub-language range ID
    };
    USHORT m_nElts;                 // (edge size ^ dimensionality)
    PHIdx m_pMap;                   // char/WCHAR to histogram idx mapping

    HElt *m_panElts;                // array of elements / counts
};
typedef Histogram *PHistogram;

/****************************************************************/

// A Language object stores all the detection state for a given language,
// i.e. primary language ID.

class Language {
public:
    // nCodePages is same as nSubLangs
    Language (PLCDetect pL, int nLangID, int nCodePages, int nRangeID = 0);
    virtual ~Language (void) { }

    virtual DWORD AddHistogram (PFileHistogramSection pHS, DWORD nBytes, int nIdx) = 0;

    // Score the code pages for this language
    virtual void ScoreCodePage (LPCSTR, int nCh, CScore &S, int &idx) const;

    int LanguageID (void) const { return m_nLangID; }
    int NCodePages (void) const { return m_nCodePages; }
    int NSubLangs (void) const { return m_nSubLangs; }
    int RangeID (void) const { return m_nRangeID; }
    int GetScoreIdx (void) const { return m_nScoreIdx; }
    void SetScoreIdx (int nScoreIdx) { m_nScoreIdx = nScoreIdx; }

    virtual int GetCodePage (int n) const { return 0; }
    virtual int GetSublangRangeID (int n) const { return 0; }
    virtual int GetSublangID (int n) const { return 0; }

    virtual DetectionType Type (void) = 0;
    virtual Language7Bit const * GetLanguage7Bit (void) const { return NULL; }
    virtual Language8Bit const * GetLanguage8Bit (void) const { return NULL; }
    virtual LanguageUnicode const * GetLanguageUnicode (void) const { return NULL; }

protected:
    PLCDetect m_pLC;

    int m_nLangID;      // Win32 primary language ID
    int m_nRangeID;     // Unicode range ID, for Unicode langs
    union {
        int m_nCodePages;   // # of code pages trained for this language
        int m_nSubLangs;
    };
    int m_nScoreIdx;    // Used to create a unique index into the score arrays
                        // for each lang + cp combination, to eliminate the
                        // need to search the arrays to merge scores.  Add
                        // the code page index to this to get the array index.
};

////////////////////////////////////////////////////////////////

class Language7Bit : public Language {
public:
    Language7Bit (PLCDetect pL, int nLangID, int nCodePages);
    ~Language7Bit (void);

    DWORD AddHistogram (PFileHistogramSection pHS, DWORD nBytes, int nIdx);

    void ScoreCodePage (LPCSTR, int nCh, CScore &S, int &idx) const;

    int GetCodePage (int n) const { return m_ppCodePageHistogram[n]->CodePage();}
    virtual DetectionType Type (void) { return DETECT_7BIT; }

    PHistogram GetLangHistogram (void) const { return m_pLangHistogram; }
    PHistogram GetCodePageHistogram (int i) const { 
        return m_ppCodePageHistogram[i]; }

    virtual Language7Bit const * GetLanguage7Bit (void) const { return this; }

    const PHElt * GetPHEltArray (void) const { return m_paHElt; }

private:
    PHistogram m_pLangHistogram;
    PHistogram m_ppCodePageHistogram[MAXSUBLANG];

    PHElt m_paHElt[MAXSUBLANG];
};

////////////////////////////////////////////////////////////////

class Language8Bit : public Language {
public:
    Language8Bit (PLCDetect pL, int nLangID, int nCodePages);
    ~Language8Bit (void);

    DWORD AddHistogram (PFileHistogramSection pHS, DWORD nBytes, int nIdx);

    int GetCodePage (int n) const { return m_ppHistogram[n]->CodePage(); }

    virtual DetectionType Type (void) { return DETECT_8BIT; }

    PHistogram GetHistogram (int i) const { return m_ppHistogram[i]; }

    virtual Language8Bit const * GetLanguage8Bit (void) const { return this; }

private:
    PHistogram m_ppHistogram[MAXSUBLANG];
};

////////////////////////////////////////////////////////////////

class LanguageUnicode : public Language {
public:
    LanguageUnicode (PLCDetect pL, int nLangID, int nRecordCount, int nRangeID);
    ~LanguageUnicode (void);
    
    DWORD AddHistogram (PFileHistogramSection pHS, DWORD nBytes, int nIdx);
    
    void ScoreSublanguages (LPCWSTR wcs, int nch, CScores &S) const;

    int GetSublangRangeID (int i) const{return GetHistogram(i)->GetRangeID();}
    PLanguageUnicode GetSublanguage (int n) const;

    virtual DetectionType Type (void) { return DETECT_UNICODE; }

    PHistogram GetHistogram (int i) const { return m_ppSubLangHistogram[i]; }

    virtual LanguageUnicode const * GetLanguageUnicode (void) const { 
        return this; 
    }

    const PHElt * GetPHEltArray (void) const { return m_paHElt; }

private:
    PHistogram m_ppSubLangHistogram[MAXSUBLANG];

    PHElt m_paHElt[MAXSUBLANG];
};

/****************************************************************/

class Charmap {

public:
    Charmap (PFileMapSection pMS) : m_nID(pMS->m_dwID), m_nSize(pMS->m_dwSize),
        m_nUnique(pMS->m_dwNUnique), m_pElts( (PHIdx) (&pMS[1]) ) { }

//  int ID (void) const { return m_nID; }
    int Size (void) const { return m_nSize; }
    int NUnique (void) const { return m_nUnique; }
    PHIdx Map (void) const { return m_pElts; }
    HIdx Map (WCHAR x) const { return m_pElts[x]; }

private:
    int m_nID;          // ID by which hardwired code finds the table
    int m_nSize;        // size of table (256 or 65536)
    int m_nUnique;      // # of unique output values
    
    PHIdx m_pElts;
};
typedef Charmap *PCharmap;

/****************************************************************/

// class CScore -- score for one lang and/or code page, variously used for
// individual chunks and also for an entire document.

class CScore {
public:
    // Only these two slots need to be initialized
    CScore (void) : m_nScore(0), m_nChars(0) {}
    ~CScore (void) { }
    
    const PLanguage GetLang (void) const { return m_pLang; }
    int GetScore (void) const { return m_nScore; }
    unsigned short GetCodePage (void) const { return m_nCodePage; }
    unsigned short GetCharCount (void) const { return m_nChars; }

    void SetLang (PLanguage p) { m_pLang = p; }
    void SetScore (int x) { m_nScore = x; }
    void SetCharCount (unsigned x) { m_nChars = (unsigned short)x; }
    void SetCodePage (unsigned x) { m_nCodePage = (unsigned short)x; }

    void Add (CScore &S) { 
        SetLang(S.GetLang());
        SetCodePage(S.GetCodePage());
        SetScore(GetScore() + S.GetScore());
        SetCharCount(GetCharCount() + S.GetCharCount());
    }
    CScore & operator += (CScore &S) { Add (S); return *this; }

    int operator <= (CScore &S) {
        // Special:  always put 8-bit langs first since the code page
        // matters more for them.
        if (GetLang()->Type() != S.GetLang()->Type())
            return GetLang()->Type() == DETECT_8BIT ? -1 : 1;
        return GetScore() <= S.GetScore();
    }

#ifdef DEBUG_LCDETECT
    void Print(void) {
        printf("Lang=%d CodePage=%d Score=%d NChars=%d\n",
            GetLang() ? GetLang()->LanguageID() : -1, 
            GetCodePage(), GetScore(), GetCharCount());
    }
#endif

private:
    PLanguage m_pLang;
    int m_nScore;
    unsigned short m_nCodePage;
    unsigned short m_nChars;
};
typedef CScore *PScore;

////////////////////////////////////////////////////////////////

// class CScores
//
// For SBCS detection, the index e.g. Ref(i) is the language+codepage index,
// one of a contiguous set of values which identifies each unique supported
// language and codepage combination.
//
// For DBCS detection, the index is just the Unicode language group.

class CScores {
public:
    CScores (int nAlloc, PScore p) : m_nAlloc(nAlloc), m_nUsed(0), m_p(p) { }
    virtual ~CScores (void) { }

    void Reset (void) {
        memset ((void *)m_p, 0, sizeof(CScore) * m_nUsed);
        m_nUsed = 0;
    }

    unsigned int &NElts (void) { return m_nUsed; }
    CScore &Ref (unsigned int n) {
        if (m_nUsed <= n)
            m_nUsed = n + 1; 
        return m_p[n]; 
    }

    void SelectCodePages (void);

    void RemoveZeroScores (void) {
        for (unsigned int i = 0, j = 0; i < m_nUsed; i++)
        {
            if (m_p[i].GetScore() > MINRAWSCORE)
                m_p[j++] = m_p[i];
        }
        m_nUsed = j;
    }

    // Sort by decreasing score.
    // Instantiates template qsort using CScore::operator <=

    void SortByScore (void) {
        RemoveZeroScores ();
        if (m_nUsed)
            QSort (m_p, m_nUsed, FALSE);
    }

    CScore & FindHighScore (void) {
        int highscore = 0;
        for (unsigned int i = 0, highidx = 0; i < m_nUsed; i++) {
            if (m_p[i].GetScore() > highscore)
            {
                highscore = m_p[i].GetScore();
                highidx = i;
            }
        }
        return m_p[highidx];
    }

protected:
    unsigned int m_nAlloc;
    unsigned int m_nUsed;   // high water mark to optimize NElts(), Reset()
    PScore m_p;             // score array, typically per TScores<NNN>
};

template<ULONG Size>class TScores : public CScores {

public:
    TScores (void) : CScores (Size, m_S) { }
    virtual ~TScores (void) { }

private:
    CScore m_S[Size];
};

////////////////////////////////////////////////////////////////

class LCDetect {

public:
    LCDetect (HMODULE hM);
    ~LCDetect (void);

    unsigned int GetNCharmaps() const { return m_nCharmaps; }
    unsigned int GetN7BitLanguages() const { return m_n7BitLanguages; }
    unsigned int GetN8BitLanguages() const { return m_n8BitLanguages; }
    unsigned int GetNUnicodeLanguages() const { return m_nUnicodeLanguages; }

    PLanguage7Bit Get7BitLanguage (int i) const { return m_pp7BitLanguages[i]; }
    PLanguage8Bit Get8BitLanguage (int i) const { return m_pp8BitLanguages[i]; }
    PLanguageUnicode GetUnicodeLanguage (int i) const { return m_ppUnicodeLanguages[i]; }

    PHIdx GetMap (int i) const { return m_ppCharmaps[i]->Map(); }

    const LCDConfigure &GetConfig () const { return m_LCDConfigureDefault; }

    DWORD LoadState (void);

    DWORD DetectA (LPCSTR pStr, int nChars, PLCDScore paScores, 
                            int *pnScores, PCLCDConfigure pLCDC) const;

    DWORD DetectW (LPCWSTR wcs, int nInputChars, PLCDScore paScores, 
                            int *pnScores, PCLCDConfigure pLCDC) const;

private:
    DWORD Initialize7BitLanguage (PFileLanguageSection pLS, PLanguage *ppL);
    DWORD Initialize8BitLanguage (PFileLanguageSection pLS, Language **ppL);
    DWORD InitializeUnicodeLanguage (PFileLanguageSection pLS,Language **ppL);
    DWORD LoadLanguageSection (void *pv, int nSectionSize, PLanguage *ppL);
    DWORD LoadHistogramSection (void *pv, int nSectionSize, Language *pL);
    DWORD LoadMapSection (void *pv, int nSectionSize);
    DWORD BuildState (DWORD nFileSize);

    void Score7Bit (LPCSTR pcszText, int nChars, CScores &S) const;
    void Score8Bit (LPCSTR pcszText, int nChars, CScores &S) const;
    int ScoreCodePage (LPCSTR pStr, int nChars, CScore &S) const;
    int ChooseDetectionType (LPCSTR pcszText, int nChars) const;
    void ScoreLanguageA (LPCSTR pStr, int nChars, CScores &S) const;
    void ScoreLanguageW (LPCWSTR wcs, int nChars, CScores &S, PCLCDConfigure) const;
    void ScoreLanguageAsSBCS (LPCWSTR wcs, int nch, CScores &S) const;
    void ScoreUnicodeSublanguages (PLanguageUnicode pL, LPCWSTR wcs, 
            int nch, CScores &S) const;

private:
    // Language training info virtual-mapped in training file

    unsigned int m_nCharmaps;
    unsigned int m_n7BitLanguages;
    unsigned int m_n8BitLanguages;
    unsigned int m_nUnicodeLanguages;

    PCharmap *m_ppCharmaps;
    PLanguage7Bit *m_pp7BitLanguages;
    PLanguage8Bit *m_pp8BitLanguages;
    PLanguageUnicode *m_ppUnicodeLanguages;

    // Cached information for the optimized scoring inner-loops.

    PHElt m_paHElt7Bit[MAX7BITLANG];
    PHElt m_paHElt8Bit[MAXSCORES];
    int m_nHElt8Bit;

    // Special 7-bit lang histogram for ScoreLanguageAsSBCS()

    PHistogram m_pHU27Bit;

    // Initialization state variables

    unsigned int m_n7BitLangsRead;
    unsigned int m_n8BitLangsRead;
    unsigned int m_nUnicodeLangsRead;
    unsigned int m_nMapsRead;
    int m_nHistogramsRead;
    int m_nScoreIdx;

    // Default configuration to use when NULL parameter passed to detect

    LCDConfigure m_LCDConfigureDefault;

    // File mapping information for the training data file

    HANDLE m_hf;
    HANDLE m_hmap;
    void *m_pv;

    HMODULE m_hModule;
};

////////////////////////////////////////////////////////////////

inline PLanguageUnicode 
LanguageUnicode::GetSublanguage (int n) const 
{ 
    return m_pLC->GetUnicodeLanguage(GetSublangRangeID(n));
}

#endif  // __cplusplus