You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
889 lines
24 KiB
889 lines
24 KiB
////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Filename : Tokenizer.h
|
|
// Purpose : Tokenizer declerations
|
|
//
|
|
// Project : WordBreakers
|
|
// Component: English word breaker
|
|
//
|
|
// Author : yairh
|
|
//
|
|
// Log:
|
|
//
|
|
// Jan 06 2000 yairh creation
|
|
// Apr 05 2000 dovh - Fixed two problematic debug / tracer buffer size
|
|
// problems. (Fix Bug 15449).
|
|
// May 07 2000 dovh - USE_WS_SENTINEL algorithm in BreakText
|
|
// Nov 11 2000 dovh - Special underscore treatment
|
|
// Added inline support routines (FindLeftmostUnderscore etc.)
|
|
//
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
#ifndef _TOKENIZER_H_
|
|
#define _TOKENIZER_H_
|
|
|
|
#include "tracer.h"
|
|
#include "PropArray.h"
|
|
#include "Query.h"
|
|
#include "stdafx.h"
|
|
#include "cierror.h"
|
|
#include "LangSupport.h"
|
|
#include "Formats.h"
|
|
|
|
#define TOKENIZER_MAXBUFFERLIMIT 1024 // max size of a token is 1024 chars
|
|
|
|
DECLARE_TAG(s_tagTokenizer, "Tokenizer");
|
|
DECLARE_TAG(s_tagTokenizerOutput, "Tokenizer Output");
|
|
DECLARE_TAG(s_tagTokenizerTrace, "Tokenizer Trace");
|
|
DECLARE_TAG(s_tagTokenizerDecision, "Tokenizer Decision");
|
|
DECLARE_TAG(s_tagTokenizerSuspect, "Tokenizer Suspect");
|
|
|
|
#if defined(DEBUG)
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// Class CTraceWordSink
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
class CTraceWordSink : public IWordSink
|
|
{
|
|
public:
|
|
CTraceWordSink(IWordSink* p) : m_apWordSink(p)
|
|
{
|
|
}
|
|
|
|
ULONG __stdcall AddRef()
|
|
{
|
|
return 1;
|
|
}
|
|
|
|
ULONG __stdcall Release()
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
STDMETHOD(QueryInterface)(
|
|
IN REFIID riid,
|
|
IN void **ppvObject)
|
|
{
|
|
Assert(false);
|
|
return E_FAIL;
|
|
}
|
|
|
|
STDMETHOD(PutWord)(
|
|
ULONG cwc,
|
|
WCHAR const* pwcInBuf,
|
|
ULONG cwcSrcLen,
|
|
ULONG cwcSrcPos)
|
|
{
|
|
Assert(cwc < TOKENIZER_MAXBUFFERLIMIT + 10);
|
|
#if (defined (DEBUG) && !defined(_NO_TRACER)) || defined(USE_TRACER)
|
|
if (CheckTraceRestrictions(elVerbose, s_tagTokenizerOutput))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerOutput,
|
|
("PutWord: %*.*S, %d, %d, %d",
|
|
cwc,
|
|
cwc,
|
|
pwcInBuf,
|
|
cwc,
|
|
cwcSrcLen,
|
|
cwcSrcPos));
|
|
}
|
|
#endif
|
|
|
|
return m_apWordSink->PutWord(cwc, pwcInBuf, cwcSrcLen, cwcSrcPos);
|
|
}
|
|
|
|
STDMETHOD(PutAltWord)(
|
|
ULONG cwc,
|
|
WCHAR const* pwcInBuf,
|
|
ULONG cwcSrcLen,
|
|
ULONG cwcSrcPos)
|
|
{
|
|
Assert(cwc < TOKENIZER_MAXBUFFERLIMIT + 10);
|
|
#if (defined (DEBUG) && !defined(_NO_TRACER)) || defined(USE_TRACER)
|
|
if (CheckTraceRestrictions(elVerbose, s_tagTokenizerOutput))
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerOutput,
|
|
("PutAltWord: %*.*S, %d, %d, %d",
|
|
cwc,
|
|
cwc,
|
|
pwcInBuf,
|
|
cwc,
|
|
cwcSrcLen,
|
|
cwcSrcPos));
|
|
}
|
|
#endif
|
|
return m_apWordSink->PutAltWord(cwc, pwcInBuf, cwcSrcLen, cwcSrcPos);
|
|
}
|
|
|
|
STDMETHOD(StartAltPhrase)()
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerOutput,
|
|
("StartAltPhrase"));
|
|
|
|
return m_apWordSink->StartAltPhrase();
|
|
}
|
|
|
|
STDMETHOD(EndAltPhrase)()
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerOutput,
|
|
("EndAltPhrase"));
|
|
|
|
return m_apWordSink->EndAltPhrase();
|
|
}
|
|
|
|
STDMETHOD(PutBreak)(WORDREP_BREAK_TYPE breakType)
|
|
{
|
|
WCHAR* p;
|
|
#if (defined (DEBUG) && !defined(_NO_TRACER)) || defined(USE_TRACER)
|
|
if (CheckTraceRestrictions(elVerbose, s_tagTokenizerOutput))
|
|
{
|
|
switch (breakType)
|
|
{
|
|
case WORDREP_BREAK_EOW:
|
|
p = L"WORDREP_BREAK_EOW";
|
|
break;
|
|
case WORDREP_BREAK_EOS:
|
|
p = L"WORDREP_BREAK_EOS";
|
|
break;
|
|
case WORDREP_BREAK_EOP:
|
|
p = L"WORDREP_BREAK_EOP";
|
|
break;
|
|
case WORDREP_BREAK_EOC:
|
|
p = L"WORDREP_BREAK_EOC";
|
|
break;
|
|
default:
|
|
p = L"Unknown break type";
|
|
}
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizerOutput,
|
|
("PutBreak %S", p));
|
|
}
|
|
#endif
|
|
return m_apWordSink->PutBreak(breakType);
|
|
}
|
|
|
|
CTraceWordSink* operator ->()
|
|
{
|
|
return this;
|
|
}
|
|
private:
|
|
CComPtr<IWordSink> m_apWordSink;
|
|
};
|
|
#endif
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// Class CTokenState
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
class CTokenState
|
|
{
|
|
public:
|
|
//
|
|
// methods
|
|
//
|
|
|
|
CTokenState();
|
|
CTokenState(CTokenState& s);
|
|
|
|
CTokenState& operator = (CTokenState& S);
|
|
|
|
void Clear(ULONG ulEnd);
|
|
|
|
public:
|
|
//
|
|
// members
|
|
//
|
|
|
|
ULONG m_ulStart;
|
|
ULONG m_ulEnd;
|
|
CPropFlag m_Properties;
|
|
WCHAR* m_pwcsToken;
|
|
};
|
|
|
|
inline CTokenState::CTokenState() : m_ulStart(0), m_ulEnd(0)
|
|
{
|
|
}
|
|
|
|
inline CTokenState::CTokenState(CTokenState& s) :
|
|
m_ulStart(s.m_ulStart),
|
|
m_ulEnd(s.m_ulEnd),
|
|
m_pwcsToken(s.m_pwcsToken),
|
|
m_Properties(s.m_Properties)
|
|
{
|
|
}
|
|
|
|
inline CTokenState& CTokenState::operator = (CTokenState& S)
|
|
{
|
|
m_ulStart = S.m_ulStart;
|
|
m_ulEnd = S.m_ulEnd;
|
|
m_Properties = S.m_Properties;
|
|
m_pwcsToken = S.m_pwcsToken;
|
|
|
|
return *this;
|
|
}
|
|
|
|
inline void CTokenState::Clear(ULONG ulEnd)
|
|
{
|
|
m_ulStart = 0;
|
|
m_ulEnd = ulEnd;
|
|
m_Properties.Clear();
|
|
m_pwcsToken = NULL;
|
|
}
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// Class CToken
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
class CToken
|
|
{
|
|
public:
|
|
//
|
|
// methods
|
|
//
|
|
|
|
CToken(ULONG ulMaxTokenSize);
|
|
|
|
bool IsNotEmpty();
|
|
void Clear();
|
|
bool IsFull();
|
|
void MarkEndToken(ULONG ulCurPosInTxtSourceBuffer);
|
|
ULONG RemoveHeadPunct(CPropFlag& PunctProperties, CTokenState& State);
|
|
ULONG RemoveTailPunct(CPropFlag& PunctProperties, CTokenState& State);
|
|
void ComputeStateProperties(CTokenState& State);
|
|
ULONG CalculateStateOffsetInTxtSourceBuffer(CTokenState& State);
|
|
|
|
ULONG FindLeftmostUnderscore(CTokenState& State);
|
|
ULONG FindRightmostUnderscore(CTokenState& State);
|
|
|
|
public:
|
|
//
|
|
// members
|
|
//
|
|
ULONG m_ulBufPos;
|
|
bool m_fHasEos;
|
|
ULONG m_ulOffsetInTxtSourceBuffer;
|
|
|
|
ULONG m_ulMaxTokenSize;
|
|
|
|
CTokenState m_State;
|
|
|
|
WCHAR m_awchBuf[TOKENIZER_MAXBUFFERLIMIT + 1];
|
|
|
|
};
|
|
|
|
inline CToken::CToken(ULONG ulMaxTokenSize) :
|
|
m_ulBufPos(0),
|
|
m_fHasEos(false),
|
|
m_ulOffsetInTxtSourceBuffer(0),
|
|
m_ulMaxTokenSize(ulMaxTokenSize)
|
|
{
|
|
m_awchBuf[0] = L'\0';
|
|
}
|
|
|
|
inline bool CToken::IsNotEmpty()
|
|
{
|
|
return (m_ulBufPos > 0);
|
|
}
|
|
|
|
inline void CToken::Clear()
|
|
{
|
|
m_ulBufPos = 0;
|
|
m_awchBuf[0] = L'\0';
|
|
m_State.Clear(0);
|
|
m_fHasEos = false;
|
|
m_ulOffsetInTxtSourceBuffer = 0;
|
|
}
|
|
|
|
|
|
inline bool CToken::IsFull()
|
|
{
|
|
return (m_ulBufPos == m_ulMaxTokenSize);
|
|
}
|
|
|
|
inline void CToken::MarkEndToken(ULONG ulCurPosInTxtSourceBuffer)
|
|
{
|
|
Assert(m_ulBufPos < m_ulMaxTokenSize + 1);
|
|
m_awchBuf[m_ulBufPos] = L'\0';
|
|
m_State.m_pwcsToken = m_awchBuf;
|
|
m_State.m_ulStart = 0;
|
|
m_State.m_ulEnd = m_ulBufPos;
|
|
|
|
|
|
if (TEST_PROP(m_State.m_Properties, PROP_EOS) &&
|
|
(m_ulBufPos < m_ulMaxTokenSize))
|
|
{
|
|
ULONG ulCur = m_State.m_ulEnd - 1;
|
|
|
|
while (TEST_PROP(GET_PROP(m_awchBuf[ulCur]), EOS_SUFFIX))
|
|
{
|
|
ulCur--;
|
|
}
|
|
|
|
if (IS_EOS(m_awchBuf[ulCur]))
|
|
{
|
|
m_fHasEos = true;
|
|
}
|
|
}
|
|
|
|
Assert(ulCurPosInTxtSourceBuffer >= m_ulBufPos);
|
|
|
|
m_ulOffsetInTxtSourceBuffer = ulCurPosInTxtSourceBuffer - m_ulBufPos;
|
|
}
|
|
|
|
inline ULONG CToken::CalculateStateOffsetInTxtSourceBuffer(CTokenState& State)
|
|
{
|
|
ULONG ulOffset =
|
|
m_ulOffsetInTxtSourceBuffer +
|
|
(State.m_pwcsToken - m_awchBuf) +
|
|
State.m_ulStart;
|
|
|
|
return ulOffset;
|
|
}
|
|
|
|
inline ULONG CToken::RemoveHeadPunct(CPropFlag& PunctProperties, CTokenState& State)
|
|
{
|
|
Assert(m_State.m_ulStart <= State.m_ulStart);
|
|
Assert(State.m_ulStart <= State.m_ulEnd);
|
|
Assert(State.m_ulEnd <= m_State.m_ulEnd);
|
|
|
|
for (ULONG ul = State.m_ulStart; ul < State.m_ulEnd; ul++)
|
|
{
|
|
if (!TEST_PROP1(GET_PROP(State.m_pwcsToken[ul]), PunctProperties) )
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
State.m_ulStart = ul;
|
|
|
|
//
|
|
// return num of characters removed
|
|
//
|
|
return ul;
|
|
}
|
|
|
|
inline ULONG CToken::RemoveTailPunct(CPropFlag& PunctProperties, CTokenState& State)
|
|
{
|
|
Assert(m_State.m_ulStart <= State.m_ulStart);
|
|
Assert(State.m_ulStart <= State.m_ulEnd);
|
|
Assert(State.m_ulEnd <= m_State.m_ulEnd);
|
|
|
|
for (ULONG ul = State.m_ulEnd; ul > State.m_ulStart; ul--)
|
|
{
|
|
if (!TEST_PROP1(GET_PROP(State.m_pwcsToken[ul - 1]), PunctProperties) )
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
ULONG ulNumOfRemovedChars = State.m_ulEnd - ul;
|
|
State.m_ulEnd = ul;
|
|
|
|
return ulNumOfRemovedChars;
|
|
}
|
|
|
|
|
|
inline void CToken::ComputeStateProperties(CTokenState& State)
|
|
{
|
|
Assert(m_State.m_ulStart <= State.m_ulStart);
|
|
Assert(State.m_ulStart <= State.m_ulEnd);
|
|
Assert(State.m_ulEnd <= m_State.m_ulEnd);
|
|
|
|
State.m_Properties.Clear();
|
|
|
|
for (ULONG ul = State.m_ulStart; ul < State.m_ulEnd; ul++)
|
|
{
|
|
State.m_Properties |= GET_PROP(State.m_pwcsToken[ul]);
|
|
}
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
// Support routines for UNDERSCORE '_' treatment.
|
|
//
|
|
// Current algorithm has the following behavior for tokens containing
|
|
// ALPHANUMERIC characters and UNDERSCORES:
|
|
//
|
|
// 1. Single underscores and consecutive underscore sequence surrounded by
|
|
// alphanumeric characters (IE underscores buried within words) are
|
|
// treated as alphanumeric characters, and do not break words, or get
|
|
// omitted. Examples: Foo_Bar => Foo_Bar, and X___Y => X___Y
|
|
//
|
|
// 2. An underscore / underscore sequence tacked to the right (left) end
|
|
// end of an alphanumeric (+ embedded underscores) token, will be part of
|
|
// the token, as long as the sequence is attacked only to one side of the
|
|
// alphanumeric token. If there are BOTH header and trailer consecutive
|
|
// underscore sequences, both header & trailer sequence will be omitted.
|
|
// Examples: __Foo_Bar => __Foo_Bar , alpha_beta_ => alpha_beta_ ,
|
|
// __HEADERFILE__ => __HEADERFILE__ , __MY_FILE_H__ => MY_FILE_H
|
|
//
|
|
// 3. Caveat: Note that other than the two rules stated above underscores are
|
|
// NOT treated as ALPHANUMERIC characters. he behavior on a mixed sequence
|
|
// of underscores, and other non-alphanumeric characters is undefined!
|
|
//
|
|
////////////////////////////////////////////////////////////////////////////////
|
|
|
|
//
|
|
// Assumes: on entry State.m_ulStart is the first alphanumeric in token
|
|
// returns: num of underscores scanned
|
|
//
|
|
inline ULONG
|
|
CToken::FindLeftmostUnderscore(CTokenState& State)
|
|
{
|
|
Assert(m_State.m_ulStart < State.m_ulStart);
|
|
Assert(State.m_ulStart <= State.m_ulEnd);
|
|
Assert(State.m_ulEnd <= m_State.m_ulEnd);
|
|
Assert( TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulStart-1]), PROP_UNDERSCORE) );
|
|
|
|
ULONG ulNumUnderscores = 0;
|
|
|
|
for (ULONG ul = State.m_ulStart;
|
|
(ul > m_State.m_ulStart) &&
|
|
(TEST_PROP(GET_PROP(State.m_pwcsToken[ul-1]), PROP_UNDERSCORE) );
|
|
ul--)
|
|
;
|
|
|
|
ulNumUnderscores = State.m_ulStart - ul;
|
|
|
|
State.m_ulStart = ul;
|
|
|
|
//
|
|
// return num of underscores scanned
|
|
//
|
|
return (ulNumUnderscores);
|
|
|
|
} // CToken::FindLeftmostUnderscore
|
|
|
|
//
|
|
// Assumes: on entry State.m_ulEnd is the last alphanumeric in token
|
|
// returns: num of underscores scanned
|
|
//
|
|
inline ULONG
|
|
CToken::FindRightmostUnderscore(CTokenState& State)
|
|
{
|
|
Assert(m_State.m_ulStart <= State.m_ulStart);
|
|
Assert(State.m_ulStart <= State.m_ulEnd);
|
|
Assert(State.m_ulEnd < m_State.m_ulEnd);
|
|
Assert( TEST_PROP(GET_PROP(State.m_pwcsToken[State.m_ulEnd]), PROP_UNDERSCORE) );
|
|
|
|
ULONG ulNumUnderscores = 0;
|
|
|
|
for (ULONG ul = State.m_ulEnd;
|
|
(ul < m_State.m_ulEnd) &&
|
|
(TEST_PROP(GET_PROP(State.m_pwcsToken[ul]), PROP_UNDERSCORE) );
|
|
ul++)
|
|
;
|
|
|
|
ulNumUnderscores = ul - State.m_ulEnd;
|
|
|
|
State.m_ulEnd = ul;
|
|
|
|
//
|
|
// return num of underscores scanned
|
|
//
|
|
return (ulNumUnderscores);
|
|
|
|
} // CToken::FindRightmostUnderscore
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
// Class CTokenizer
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
class CTokenizer
|
|
{
|
|
public:
|
|
|
|
CTokenizer(
|
|
TEXT_SOURCE* pTxtSource,
|
|
IWordSink * pWordSink,
|
|
IPhraseSink * pPhraseSink,
|
|
LCID lcid,
|
|
BOOL bQueryTime,
|
|
ULONG ulMaxTokenSize);
|
|
|
|
|
|
// destructor frees the passed buffer, if it exists
|
|
virtual ~CTokenizer(void)
|
|
{
|
|
}
|
|
|
|
void BreakText();
|
|
|
|
protected:
|
|
|
|
//
|
|
// methods
|
|
//
|
|
|
|
void ProcessToken();
|
|
void ProcessTokenInternal();
|
|
void BreakCompundString(CTokenState& State, CPropFlag& prop);
|
|
|
|
HRESULT FillBuffer();
|
|
void CalculateUpdateEndOfBuffer();
|
|
|
|
bool CheckAndCreateNumber(
|
|
WCHAR* pwcsStr,
|
|
ULONG ulLen,
|
|
WCHAR* pwcsOut,
|
|
ULONG* pulOffsetToTxt,
|
|
ULONG* pulOutLen);
|
|
|
|
int CheckAndCreateNumber(
|
|
WCHAR* pwcsStr,
|
|
ULONG ulLen,
|
|
WCHAR wchSDecimal,
|
|
WCHAR wchSThousand,
|
|
WCHAR* pwcsOut,
|
|
ULONG* pulOffsetToTxt,
|
|
ULONG* pulOutLen);
|
|
|
|
short ConvertHexCharToNumber(WCHAR wch);
|
|
void GetValuesFromDateString(
|
|
CDateTerm* pFormat,
|
|
WCHAR* pwcsDate,
|
|
LONG* plD_M1, // we can't tell in this stage whether this is a Day or a month.
|
|
LONG* plD_M2,
|
|
LONG* plYear);
|
|
|
|
void GetValuesFromTimeString(
|
|
CTimeTerm* pFormat,
|
|
WCHAR* pwcsTime,
|
|
LONG* plHour,
|
|
LONG* plMin,
|
|
LONG* plSec,
|
|
TimeFormat* pAmPm);
|
|
|
|
LONG ConvertCharToDigit(WCHAR wch);
|
|
#ifdef DEBUG
|
|
void TraceToken();
|
|
#endif DEBUG
|
|
|
|
bool VerifyAlphaUrl();
|
|
bool VerifyWwwUrl();
|
|
bool VerifyAcronym();
|
|
bool VerifyAbbreviation();
|
|
bool VerifySpecialAbbreviation();
|
|
bool VerifyHyphenation();
|
|
bool VerifyParens();
|
|
const CCliticsTerm* VerifyClitics(CTokenState& State);
|
|
bool VerifyNumber(CTokenState& State);
|
|
bool VerifyNumberOrTimeOrDate();
|
|
bool VerifyTime(CTokenState& State);
|
|
bool VerifyDate(CTokenState& State);
|
|
bool VerifyCurrency();
|
|
bool VerifyMisc();
|
|
bool VerifyCommersialSign();
|
|
|
|
void ProcessDefault();
|
|
|
|
ULONG
|
|
AddBackUnderscores(
|
|
IN CTokenState& State,
|
|
IN bool hasFrontUnderscore,
|
|
IN bool hasBackUnderscore
|
|
);
|
|
bool CheckAndRemoveOneSidedUnderscores(CTokenState& State);
|
|
|
|
void OutputUrl(
|
|
CTokenState& State);
|
|
void OutputAcronym(
|
|
CTokenState& State,
|
|
const CCliticsTerm* pCliticsTerm);
|
|
void OutputAbbreviation(
|
|
CTokenState& State);
|
|
void OutputSpecialAbbreviation(
|
|
CTokenState& State,
|
|
CAbbTerm* pTerm,
|
|
const CCliticsTerm* pCliticsTerm);
|
|
virtual void OutputHyphenation(
|
|
CTokenState& State,
|
|
const CCliticsTerm* pCliticsTerm);
|
|
void OutputParens(
|
|
CTokenState& State);
|
|
void OutputNumbers(
|
|
CTokenState& State,
|
|
ULONG ulLen,
|
|
WCHAR* pwcsNumber,
|
|
const CCliticsTerm* pCliticsTerm);
|
|
void OutputTime(
|
|
WCHAR* pwcsTime,
|
|
CTokenState& State);
|
|
void OutputDate(
|
|
WCHAR* pwcsDate1,
|
|
WCHAR* pwcsDate2,
|
|
CTokenState& State);
|
|
virtual void OutputSimpleToken(
|
|
CTokenState& State,
|
|
const CCliticsTerm* pTerm);
|
|
void OutputCurrency(
|
|
ULONG ulLen,
|
|
WCHAR* pwcsCurrency,
|
|
CTokenState& State,
|
|
const CCliticsTerm* pTerm);
|
|
void OutputMisc(
|
|
CTokenState& State,
|
|
bool bPatternContainOnlyUpperCase,
|
|
ULONG ulSuffixSize,
|
|
const CCliticsTerm* pCliticsTerm);
|
|
void OutputCommersialSignToken(CTokenState& State);
|
|
|
|
//
|
|
// members
|
|
//
|
|
|
|
LCID m_Lcid;
|
|
CAutoClassPointer<CLangSupport> m_apLangSupport;
|
|
|
|
CToken* m_pCurToken;
|
|
CToken m_Token;
|
|
|
|
#if defined(DEBUG)
|
|
CTraceWordSink m_apWordSink;
|
|
#else
|
|
CComPtr<IWordSink> m_apWordSink;
|
|
#endif
|
|
CComPtr<IPhraseSink> m_apPhraseSink;
|
|
TEXT_SOURCE* m_pTxtSource;
|
|
|
|
BOOL m_bQueryTime;
|
|
|
|
ULONG m_ulUpdatedEndOfBuffer;
|
|
bool m_bNoMoreTxt;
|
|
|
|
//
|
|
// All Chunks in buffer have a white space
|
|
//
|
|
bool m_bWhiteSpaceGuarranteed;
|
|
ULONG m_ulMaxTokenSize;
|
|
|
|
};
|
|
|
|
inline HRESULT CTokenizer::FillBuffer()
|
|
{
|
|
Trace(
|
|
elVerbose,
|
|
s_tagTokenizer,
|
|
("WBreakGetNextChar: Filling the buffer"));
|
|
|
|
HRESULT hr;
|
|
|
|
if (!m_bNoMoreTxt)
|
|
{
|
|
do
|
|
{
|
|
//
|
|
// this loop usually performs only one rotations. we use it to solve the
|
|
// problem when the user return 0 characters and a success return code.
|
|
// the following code assumes that in case you get a success return code then
|
|
// the buffer is not empty.
|
|
//
|
|
|
|
hr = m_pTxtSource->pfnFillTextBuffer(m_pTxtSource);
|
|
} while ((m_pTxtSource->iEnd <= m_pTxtSource->iCur) && SUCCEEDED(hr));
|
|
|
|
if ( FAILED(hr))
|
|
{
|
|
m_bNoMoreTxt = true;
|
|
}
|
|
}
|
|
|
|
if (m_bNoMoreTxt && m_pTxtSource->iCur >= m_pTxtSource->iEnd)
|
|
{
|
|
//
|
|
// we reached the end of the buffer.
|
|
//
|
|
return WBREAK_E_END_OF_TEXT;
|
|
}
|
|
|
|
CalculateUpdateEndOfBuffer();
|
|
|
|
return S_OK;
|
|
}
|
|
|
|
inline void CTokenizer::CalculateUpdateEndOfBuffer()
|
|
{
|
|
//
|
|
// m_ulUpdatedEndOfBuffer is a marker for the last character that we can read
|
|
// from the current buffer before and additional call to fill buffer is needed.
|
|
// we use this marker to avoid terms spitted between two consecutive buffers.
|
|
// in order to achieve the above m_ulUpdatedEndOfBuffer will point to a breaker
|
|
// character. (the only exception to that is when we have a very long term that does
|
|
// not contains breaker characters).
|
|
//
|
|
|
|
//
|
|
// we split the buffer into chunks of TOKENIZER_MAXBUFFERLIMIT size. in each
|
|
// chunk we make sure that there is a breaker.
|
|
//
|
|
|
|
ULONG ulStartChunk = m_pTxtSource->iCur;
|
|
ULONG ulEndChunk ;
|
|
bool fLastRound = false;
|
|
|
|
Assert(m_pTxtSource->iEnd > m_pTxtSource->iCur);
|
|
|
|
ulEndChunk = m_pTxtSource->iCur + m_ulMaxTokenSize > (m_pTxtSource->iEnd - 1) ?
|
|
(m_pTxtSource->iEnd - 1) : m_pTxtSource->iCur + m_ulMaxTokenSize;
|
|
ULONG ulCur;
|
|
ULONG ulBreakerMarker = 0;
|
|
m_bWhiteSpaceGuarranteed = false;
|
|
|
|
while(true)
|
|
{
|
|
ulCur = ulEndChunk;
|
|
|
|
//
|
|
// per each chunk we go backward and try to find a WS.
|
|
//
|
|
while ((ulCur > ulStartChunk) &&
|
|
(!IS_WS(m_pTxtSource->awcBuffer[ulCur])))
|
|
{
|
|
ulCur--;
|
|
}
|
|
|
|
if (ulCur == ulStartChunk)
|
|
{
|
|
|
|
//
|
|
// the last chunk that we checked did not contain any WS
|
|
//
|
|
|
|
if (m_ulMaxTokenSize == (ulEndChunk - ulStartChunk))
|
|
{
|
|
//
|
|
// full buffer case. we look for a default breaker.
|
|
//
|
|
|
|
ulCur = ulEndChunk;
|
|
|
|
while ( (ulCur > ulStartChunk) &&
|
|
!IS_BREAKER( m_pTxtSource->awcBuffer[ulCur] )
|
|
)
|
|
{
|
|
ulCur--;
|
|
}
|
|
|
|
//
|
|
// if we found a breaker then ulBreakerMarker will set to it else
|
|
// the term does not contain any breakers and we set the ulBreakerMarker
|
|
// to the end of the term. this is the only case that we spilt terms.
|
|
//
|
|
ulBreakerMarker = ulCur > ulStartChunk ? ulCur : ulEndChunk;
|
|
}
|
|
else
|
|
{
|
|
if (ulStartChunk > m_pTxtSource->iCur)
|
|
{
|
|
//
|
|
// case we had a previous chunk. in this case ulStartChunk points to
|
|
// a breaker
|
|
//
|
|
|
|
//
|
|
// ulStart points to the WS from the previous chunk.
|
|
//
|
|
ulBreakerMarker = ulStartChunk;
|
|
}
|
|
else
|
|
{
|
|
ulBreakerMarker = m_pTxtSource->iEnd;
|
|
}
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
if (fLastRound)
|
|
{
|
|
//
|
|
// ulCur points to a WS
|
|
//
|
|
ulBreakerMarker = ulCur + 1;
|
|
m_bWhiteSpaceGuarranteed = true;
|
|
|
|
break;
|
|
}
|
|
|
|
//
|
|
// move to the next chunk
|
|
//
|
|
ulStartChunk = ulCur + 1; // ulStarChunk will points to a breaker
|
|
if (ulStartChunk + m_ulMaxTokenSize < (m_pTxtSource->iEnd - 1))
|
|
{
|
|
ulEndChunk = ulStartChunk + m_ulMaxTokenSize;
|
|
|
|
}
|
|
else
|
|
{
|
|
ulEndChunk = m_pTxtSource->iEnd - 1;
|
|
fLastRound = true;
|
|
}
|
|
}
|
|
|
|
Assert(ulBreakerMarker <= m_pTxtSource->iEnd);
|
|
m_ulUpdatedEndOfBuffer = ulBreakerMarker;
|
|
|
|
}
|
|
|
|
|
|
inline short CTokenizer::ConvertHexCharToNumber(WCHAR wch)
|
|
{
|
|
//
|
|
// assumes wch is a valid HEX character
|
|
//
|
|
Assert(wch >= L'0');
|
|
|
|
if (wch <= L'9')
|
|
{
|
|
return (wch - L'0');
|
|
}
|
|
else if (wch <= L'F')
|
|
{
|
|
Assert(wch >= L'A');
|
|
return (wch - L'A' + 10);
|
|
}
|
|
else if (wch <= L'f')
|
|
{
|
|
Assert(wch >= L'a');
|
|
return (wch - L'a' + 10);
|
|
}
|
|
else if (wch <= 0xFF19)
|
|
{
|
|
Assert(wch >= 0xFF10);
|
|
return (wch - 0xFF10);
|
|
}
|
|
else if (wch <= 0xFF26)
|
|
{
|
|
Assert(wch >= 0xFF21);
|
|
return (wch - 0xFF21 + 10);
|
|
}
|
|
else
|
|
{
|
|
Assert((wch >= 0xFF41) && (wch <= 0xFF46));
|
|
return (wch - 0xFF41 + 10);
|
|
}
|
|
|
|
}
|
|
|
|
inline LONG CTokenizer::ConvertCharToDigit(WCHAR wch)
|
|
{
|
|
Assert((wch >= L'0' && wch <= L'9') || ((wch >= 0xFF10) && (wch <= 0xFF19)));
|
|
if (wch <= L'9')
|
|
{
|
|
return (wch - L'0');
|
|
}
|
|
|
|
return (wch - 0xFF10); // Full width characters.
|
|
}
|
|
|
|
#endif _TOKENIZER_H_
|