windows-xp/Source/XPSP1/NT/shell/ext/mlang/fechauto.cpp

/*----------------------------------------------------------------------------
	%%File: fechauto.c
	%%Unit: fechmap
	%%Contact: jpick

	Module that attempts to auto-detect encoding for a given stream.
----------------------------------------------------------------------------*/

#include <stdio.h>
#include <stddef.h>

#include "private.h"
#include "fechmap_.h"
#include "lexint_.h"

// Code marked by these #defines will be deleted eventually ...
// (It prints out useful information and statistics about how
// auto-detect is doing and what it's finding in the input).
//
#define JPDEBUG			0
#define JPDEBUG2		0
#define JPDEBUG3		0

#define NEED_NAMES		0

#if JPDEBUG || JPDEBUG2 || JPDEBUG3
#undef NEED_NAMES
#define NEED_NAMES		1
#endif

#if NEED_NAMES
static char *rgszIcetNames[icetCount] =
{
	"icetEucCn",
	"icetEucJp",
	"icetEucKr",
	"icetEucTw",
	"icetIso2022Cn",
	"icetIso2022Jp",
	"icetIso2022Kr",
	"icetIso2022Tw",
	"icetBig5",
	"icetGbk",
	"icetHz",
	"icetShiftJis",
	"icetWansung",
	"icetUtf7",
	"icetUtf8",
};
#endif

// Characters we care about
//
#define chSo		(UCHAR) 0x0e
#define chSi		(UCHAR) 0x0f
#define chEsc		(UCHAR) 0x1b

// Minimum Sample Size
//
#define	cchMinSample		64

// High-ASCII character threshold.  If this routine is unable
// to absolutely determine the encoding of this file, it will
// need to guess.  Files that are ASCII, but contain high-ASCII
// characters (e.g., a file with some Cyrillic characters) may
// confuse us.  If the number of high-ASCII characters falls
// below this threshold, return the encoding we guessed but 
// also return a special rc that says the file "might be ASCII."
//
// 5%, for now.
//
// 40%, for now, of the high-ascii characters must be in high-
// ascii pairs.  (Pulled down because of Big5 and the other
// DBCS encodings that can have trail bytes in the low range).
//
#define nHighCharThreshold		 5		// %
#define nHighPairThreshold		40		// %

// Used by CceDetermineInputTypeReturnAll() to determine whether any icet has
// high enough count to rule out all other icets.
//
#define CchCountThreshold(icet)	(((icet) == icetHz || (icet) == icetUtf7) ? 5 : 10)



// Tokens
//
// Stop tokens (negative) imply special handling and will cause
// the processing loop to stop (eof, err, si, so and esc are
// stop tokens).
//
#define xmn			  0
#define esc			(-1)
#define  so			(-2)
#define  si			(-3)
#define eof			(-4)
#define err			(-5)

#define _FStopToken(tk)		((tk) < 0)


// Masks used in _CBitsOnFromUlong()
//
#define lMaskBitCount1	(LONG) 0x55555555
#define lMaskBitCount2	(LONG) 0x33333333
#define lMaskBitCount3	(LONG) 0x0F0F0F0F
#define lMaskBitCount4	(LONG) 0x00FF00FF
#define lMaskBitCount5	(LONG) 0x0000FFFF

/* _  C  B I T S  O N  F R O M  U L O N G */
/*----------------------------------------------------------------------------
	%%Function: _CBitsOnFromUlong
	%%Contact: jpick

	(adapted from code in convio.c)
----------------------------------------------------------------------------*/
int __inline _CBitsOnFromUlong(ULONG ulBits)
{
	ulBits = (ulBits & lMaskBitCount1) + ((ulBits & ~lMaskBitCount1) >> 1);
	ulBits = (ulBits & lMaskBitCount2) + ((ulBits & ~lMaskBitCount2) >> 2);
	ulBits = (ulBits & lMaskBitCount3) + ((ulBits & ~lMaskBitCount3) >> 4);
	ulBits = (ulBits & lMaskBitCount4) + ((ulBits & ~lMaskBitCount4) >> 8);
	ulBits = (ulBits & lMaskBitCount5) + ((ulBits & ~lMaskBitCount5) >> 16);
	
	return (int)ulBits;
}
	
// Masks for the encodings
//
#define grfEucCn        (ULONG) 0x0001
#define grfEucJp        (ULONG) 0x0002
#define grfEucKr        (ULONG) 0x0004
#define grfEucTw        (ULONG) 0x0008
#define grfIso2022Cn    (ULONG) 0x0010
#define grfIso2022Jp    (ULONG) 0x0020
#define grfIso2022Kr    (ULONG) 0x0040
#define grfIso2022Tw    (ULONG) 0x0080
#define grfBig5         (ULONG) 0x0100
#define grfGbk          (ULONG) 0x0200
#define grfHz           (ULONG) 0x0400 
#define grfShiftJis     (ULONG) 0x0800
#define grfWansung      (ULONG) 0x1000
#define grfUtf7         (ULONG) 0x2000 	
#define grfUtf8         (ULONG) 0x4000

// grfAll assumes that the tests for Euc-Kr fall within those
// for Wansung (as far as I can tell from reading, Euc-Kr is a
// strict subset of Wansung).  The same for Euc-Cn and Gbk.  No
// need to test for both the subset and the whole.
//
#define grfAll              (ULONG) 0x7FFA
#define grfAllButIso2022    (ULONG) 0x7F0A
#define cAll                13				// == number bits set in grfAll
#define cAllButIso2022      9				// == number bits set in grfAllButIso2022

// Array that maps an encoding to its mask
//
static ULONG _mpicetgrf[icetCount] =
{
	grfEucCn,
	grfEucJp,
	grfEucKr,
	grfEucTw,
	grfIso2022Cn,
	grfIso2022Jp,
	grfIso2022Kr,
	grfIso2022Tw,
	grfBig5,
	grfGbk,
	grfHz,
	grfShiftJis,
	grfWansung,
	grfUtf7,
	grfUtf8,
};

// Prototypes
//
static int  _NGetNextUch(IStream *pstmIn, unsigned char *c, BOOL *lpfIsHigh);
static ICET _IcetFromIcetMask(ULONG ulMask);
static ICET _IcetDefaultFromIcetMask(ULONG ulMask);
static CCE  _CceResolveAmbiguity(ULONG grfIcet, ICET *lpicet, int nPrefCp, EFam efPref);
static CCE  _CceReadEscSeq(IStream *pstmIn, int nPrefCp, ICET *lpicet, BOOL *lpfGuess);


/* C C E  D E T E R M I N E  I N P U T  T Y P E */
/*----------------------------------------------------------------------------
	%%Function: CceDetermineInputType
	%%Contact: jpick

	Attempt to determine the appropriate ICET type for the given 
	stream.  Caller-supplied get/unget routines used for data access.
----------------------------------------------------------------------------*/
CCE CceDetermineInputType(
    IStream   *pstmIn,           // input stream
	DWORD     dwFlags,			// configuration flags
	EFam      efPref,			// optional: preferred encoding family
	int       nPrefCp,			// optional: preferred code page
	ICET     *lpicet,			// set to detected encoding
	BOOL     *lpfGuess			// set to fTrue if function "guessed"
)
{
	unsigned char uch;
	int nToken;
	CCE cceRet;
	BOOL fGuess;
	ICET icet;
	int cIcetActive;
	ULONG grfIcetActive;	// Bitarray tracks which encodings are still active candidates.
	ICET icetSeq;
	int i, nCount, nCountCurr;
	DWORD dwValFlags;
	BOOL fIsHigh;
	int cchHigh = 0;
	int cchHighPairs = 0;
	int cchTotal = 0;
	BOOL fLastHigh = fFalse;
	
#if JPDEBUG3
	ULONG grfIcetNoCommonChars;
#endif

#if JPDEBUG
	printf("flags: %d\n", dwFlags);
#endif
	
	// Initialize parsers
	//
	dwValFlags = grfCountCommonChars;
	if (dwFlags & grfDetectUseCharMapping)
		dwValFlags |= grfValidateCharMapping;
	ValidateInitAll(dwValFlags);
	
	// Initialize locals -- be optimistic
	//
	cceRet = cceSuccess;
	fGuess = fFalse;
	grfIcetActive = grfAllButIso2022;
	cIcetActive = cAllButIso2022;
	
#if JPDEBUG3
	grfIcetNoCommonChars = grfAllButIso2022;
#endif
	
	while (fTrue)
		{
		nToken = _NGetNextUch(pstmIn, &uch, &fIsHigh);
		if (_FStopToken(nToken))
			break;
			
		// Update (admittedly dumb) statistics -- really counts high
		// ascii characters in runs (not really pairs).  But threshold
		// constants (defined, above) were determined by calculating
        // exactly these numbers for ~25 files, so it should be ok (?).
		//
		++cchTotal;
		if (fIsHigh)
			{
			++cchHigh;
			if (fLastHigh)
				++cchHighPairs;
			}
		fLastHigh = fIsHigh;
			
		for (i = 0; i < icetCount; i++)
			{
			if (!(grfIcetActive & _mpicetgrf[i]) || (NValidateUch((ICET)i, uch, fFalse) != 0))
				continue;
				
			grfIcetActive &= ~_mpicetgrf[i];
			--cIcetActive;
#if JPDEBUG
			printf("Log:  Lost %s at offset 0x%.4x (%d), char 0x%.2x\n", rgszIcetNames[i], (cchTotal-1), (cchTotal-1), uch);
#endif
			}
			
#if JPDEBUG3
		for (i = 0; i < icetCount; i++)
			{
			if (!(grfIcetActive & _mpicetgrf[i]) || !(grfIcetNoCommonChars & _mpicetgrf[i]))
				continue;
				
			if (!FValidateCharCount(i, &nCount) || (nCount == 0))
				continue;
				
			grfIcetNoCommonChars &= ~_mpicetgrf[i];
			printf("Log:  Found first common seq for %s at offset 0x%.4x (%d)\n", rgszIcetNames[i], (cchTotal-1), (cchTotal-1));
			}
#endif
			
		if ((cIcetActive == 0) || ((cIcetActive == 1) && (cchTotal > cchMinSample)))
			break;
		}
		
	// Figure out why we exited the loop.
	//
	if (nToken == err)
		{
		cceRet = cceRead;
		goto _LRet;
		}
		
	// Process escapes separately.  Interpret the escape sequence
	// to determine for real which ISO7 flavor we have found.
	//
	if ((nToken == esc) || (nToken == so) || (nToken == si))
		{
        LARGE_INTEGER   li;
        HRESULT hr;

        LISet32(li, -1 );
        hr = pstmIn->Seek(li,STREAM_SEEK_CUR, NULL);

//		if (!pfnUnget(uch, lpvPrivate))
//			{
//			cceRet = cceUnget;
//			goto _LRet;
//			}
		cceRet = _CceReadEscSeq(pstmIn, nPrefCp, &icet, &fGuess);
#if JPDEBUG
		if (cceRet == cceSuccess)
			printf("Log:  Found encoding %s at offset 0x%.4x (%d)\n", rgszIcetNames[icet], cchTotal, cchTotal);
#endif
		// ISO is a special case -- no need to check statistics.
		//
		goto _LRet;
		}
		
#if JPDEBUG2
	printf("Counts:  %d total chars, %d high chars, %d high pairs\n", cchTotal, cchHigh, cchHighPairs); 
#endif
			
	// If the token was eof, and we're not ignoring eof, transition
	// the remaining active sets on eof.
	//
	if ((nToken == eof) && !(dwFlags & grfDetectIgnoreEof))
		{
		for (i = 0; i < icetCount; i++)
			{
			if (!(grfIcetActive & _mpicetgrf[i]) || (NValidateUch((ICET)i, 0, fTrue) != 0))
				continue;
#if JPDEBUG
			printf("Log:  Lost %s at EOF\n", rgszIcetNames[i]);
#endif
			grfIcetActive &= ~_mpicetgrf[i];
			--cIcetActive;
			}
		}
		
	Assert(cIcetActive >= 0);	// better *not* be less than 0

	// See how we've narrowed our field of choices and set the 
	// return status accordingly.
	//
	if (cIcetActive <= 0)
		{
#if JPDEBUG
		printf("Log:  Bailed out entirely at offset 0x%.4x (%d)\n", cchTotal, cchTotal);
#endif
		cceRet = cceUnknownInput;
		goto _LRet;
		}
	else if (cIcetActive == 1)
		{
		icet = _IcetFromIcetMask(grfIcetActive);
#if JPDEBUG
		printf("Log:  Found encoding %s at offset 0x%.4x (%d)\n", rgszIcetNames[icet], cchTotal, cchTotal);
#endif
		// If we matched an encoding type and also found matching 
		// common character runs, skip statistics (see comment,
		// below).
		//
		if (FValidateCharCount(icet, &nCount) && (nCount > 0))
			{
#if JPDEBUG3
			printf("Log:  %d common sequences for %s\n", nCount, rgszIcetNames[icet]);
#endif
			goto _LRet;
			}
		else
			{
			goto _LStats;
			}
		}
		
	// Did we learn anything from counting characters?
	//
	icetSeq = (ICET)-1;
	nCountCurr = 0;
	for (i = 0; i < icetCount; i++)
		{
		if (!(grfIcetActive & _mpicetgrf[i]) || !FValidateCharCount((ICET)i, &nCount))
			continue;
			
		if (nCount > nCountCurr)
			{
			icetSeq = (ICET)i;
			nCountCurr = nCount;
			}
			
#if JPDEBUG3
		printf("Log:  %d common sequences for %s\n", nCount, rgszIcetNames[i]);
#endif
		}
			
	// Any luck?  If so, return.  Don't bother checking statistics.
	// We just proved that we found at least one common run of 
	// characters in this input.  The odds against this for just a
	// plain ASCII file with some high characters seem pretty high.
	// Ignore the statistics and just return the encoding type we
	// found.
	//
	if (icetSeq != -1)
		{
		icet = icetSeq;
		goto _LRet;
		}
		
#if JPDEBUG
	printf("Log:  Active Icet Mask 0x%.8x, %d left\n", grfIcetActive, cIcetActive);
	printf("Log:  Icet's left -- ");
	for (i = 0; i < icetCount; i++)
		{
		if (grfIcetActive & _mpicetgrf[i])
            printf("%s, ", rgszIcetNames[i]);
		}
    printf("\n");
#endif

	// If caller did not want us to try to guess at the encoding
	// in the absence of definitive data, bail out.
	//
	if (!(dwFlags & grfDetectResolveAmbiguity))
		{
		cceRet = cceAmbiguousInput;
		goto _LRet;
		}
		
	// We're guessing -- note it.
	//
	fGuess = fTrue;
		
	// More than one active encoding.  Attempt to resolve ambiguity.
	//
	cceRet = _CceResolveAmbiguity(grfIcetActive, &icet, nPrefCp, efPref);
	if (cceRet != cceSuccess)
		return cceRet;
		
_LStats:
		
	// Adjust the return code based on the "statistics" we gathered,
	// above.
	//
	if (cchHigh > 0)
		{
		if ((cchTotal < cchMinSample) ||
			(((cchHigh * 100) / cchTotal) < nHighCharThreshold) ||
			(((cchHighPairs * 100) / cchHigh) < nHighPairThreshold))
			{
			cceRet = cceMayBeAscii;
			}
		}
	else
		{
		cceRet = cceMayBeAscii;		// no high-ascii characters?  definitely maybe!
		}

#if JPDEBUG2
	if (cchHigh > 0)
		{
		int nPercent1 = ((cchHigh * 100) / cchTotal);
		int nPercent2 = ((cchHighPairs * 100) / cchHigh);
		printf("Ratios -- high/total: %d%%, runs/high: %d%%\n", nPercent1, nPercent2);
		}
#endif
		
_LRet:

	// Set the return variables, if successful.
	//
	if ((cceRet == cceSuccess) || (cceRet == cceMayBeAscii))
		{
		*lpicet = icet;
		*lpfGuess = fGuess;
		}
		
#if JPDEBUG
		if (cceRet == cceSuccess)
			{
			printf("Log:  Returning %s, fGuess = %s\n", rgszIcetNames[icet], (fGuess ? "fTrue" : "fFalse"));
			}
		else if (cceRet == cceMayBeAscii)
			{
			printf("Log:  Returning %s, fGuess = %s, may-be-ASCII\n", rgszIcetNames[icet], (fGuess ? "fTrue" : "fFalse"));
			}
#endif
		
	return cceRet;
}


/* _ N  G E T  N E X T  U C H */
/*----------------------------------------------------------------------------
	%%Function: _NGetNextUch
	%%Contact: jpick

	Get the next character from the input stream.  Classify the character.
----------------------------------------------------------------------------*/
static int _NGetNextUch(IStream *pstmIn, unsigned char *c, BOOL *lpfIsHigh)
{
	ULONG rc;
	unsigned char uch;
    HRESULT hr;
		  
    hr = pstmIn->Read(&uch, 1, &rc);
	
    if (rc == 0)
		return eof;
	else if (hr != S_OK )
		return err;
		
	*lpfIsHigh = (uch >= 0x80);
	*c = uch;
		
	switch (uch)
		{
		case chEsc:
			return esc;
		case chSo:
			return so;
		case chSi:
			return si;
		default:
			return xmn;
		}
}


// Masks for _CceResolveAmbiguity() -- only externally supported character
// sets are used in ambiguity resolution.  Don't include Euc-Tw here.
//
#define grfJapan			(ULONG) (grfShiftJis | grfEucJp)
#define grfChina			(ULONG) (grfEucCn | grfGbk)
#define grfKorea			(ULONG) (grfEucKr | grfWansung)
#define grfTaiwan			(ULONG) (grfBig5)
#define grfDbcs				(ULONG) (grfShiftJis | grfGbk | grfWansung | grfBig5)
#define grfEuc				(ULONG) (grfEucJp | grfEucKr | grfEucCn)


/* _ C E  F R O M  C E  M A S K */
/*----------------------------------------------------------------------------
	%%Function: _IcetFromIcetMask
	%%Contact: jpick
----------------------------------------------------------------------------*/
static ICET _IcetFromIcetMask(ULONG ulMask)
{
	switch (ulMask)
	{
	case grfEucCn:
		return icetEucCn;
	case grfEucJp:
		return icetEucJp;
	case grfEucKr:
		return icetEucKr;
	case grfEucTw:
		return icetEucTw;
	case grfIso2022Cn:
		return icetIso2022Cn;
	case grfIso2022Jp:
		return icetIso2022Jp;
	case grfIso2022Kr:
		return icetIso2022Kr;
	case grfIso2022Tw:
		return icetIso2022Tw;
	case grfBig5:
		return icetBig5;
	case grfGbk:
		return icetGbk;
	case grfHz:
		return icetHz;
	case grfShiftJis:
		return icetShiftJis;
	case grfWansung:
		return icetWansung;
	case grfUtf7:
		return icetUtf7;
	case grfUtf8:
		return icetUtf8;
	default:
		break;
	}
	
	// Should never get here ...
	//
//	NotReached();
	
	// Can't return a bogus value, here.
	//
	return icetShiftJis;
}

/* _ C E  D E F A U L T  F R O M  C E  M A S K */
/*----------------------------------------------------------------------------
	%%Function: _IcetDefaultFromIcetMask
	%%Contact: jpick
----------------------------------------------------------------------------*/
static ICET _IcetDefaultFromIcetMask(ULONG ulMask)
{
	// Priorities -- DBCS, EUC, Japan, Taiwan, China and Korea (???).
	//
	if (ulMask & grfDbcs)
		{
		if (ulMask & grfJapan)
			return icetShiftJis;
		if (ulMask & grfChina)
			return icetGbk;
		if (ulMask & grfTaiwan)
			return icetBig5;
		if (ulMask & grfKorea)
			return icetWansung;
		}
	else // EUC
		{
		if (ulMask & grfJapan)
			return icetEucJp;
		if (ulMask & grfChina)
			return icetEucCn;
		if (ulMask & grfKorea)
			return icetEucKr;			// may be able to return icetWansung, here
		}
		
	// (Assert);
	return icetShiftJis;  // ???
}

/* _ U L  C E  M A S K  F R O M  C P  E T P */
/*----------------------------------------------------------------------------
	%%Function: _UlIcetMaskFromCpEf
	%%Contact: jpick
----------------------------------------------------------------------------*/
static ULONG _UlIcetMaskFromCpEf(int nCp, EFam ef)
{
	ULONG grf = grfAll;
	
	switch (nCp)
	{
	case nCpJapan:
		grf &= grfJapan;
		break;
	case nCpChina:
		grf &= grfChina;
		break;
	case nCpKorea:
		grf &= grfKorea;
		break;
	case nCpTaiwan:
		grf &= grfTaiwan;
		break;
	default:
		break;
	}
	
	switch (ef)
	{
	case efDbcs:
		grf &= grfDbcs;
		break;
	case efEuc:
		grf &= grfEuc;
		break;
	default:
		break;
	}
	return grf;
}


/* _ C C E  R E S O L V E  A M B I G U I T Y */
/*----------------------------------------------------------------------------
	%%Function: _CceResolveAmbiguity
	%%Contact: jpick

	Attempt to resolve ambiguous input encoding based on user
	preferences, if set, and system code page.  grfIcet contains a
	bitmask representing the encodings that are still possible after
	examining the input sample.
----------------------------------------------------------------------------*/
static CCE _CceResolveAmbiguity(ULONG grfIcet, ICET *lpicet, int nPrefCp, EFam efPref)
{
    ULONG grfIcetOrig = grfIcet;
	ULONG grfPref;
	ULONG grfSys;
	ULONG grfResult;
	int cIcet;
	
	// Build "list" of encodings based on user-prefs.
	//
	grfPref = _UlIcetMaskFromCpEf(nPrefCp, efPref);
	
	// See if the user's preferences make any difference.
	//
	grfResult = grfIcet & grfPref;
	
	if (grfResult)
		{
		cIcet = _CBitsOnFromUlong(grfResult);
		if (cIcet == 1)
			{
			*lpicet = _IcetFromIcetMask(grfResult);
			return cceSuccess;
			}
		else
			grfIcet = grfResult;			// see comment, below
		}
		
	// Now look to the system code page for help.  Look at
	// the set of encodings as modified by the user
	// preferences (??? do we want to do this ???).
	//
	if (!FIsFeCp(g_uACP) || (grfIcetOrig & grfUtf8))
		goto _LDefault;
		
	// Build "list" of encodings based on system cp.
	//
	grfSys = _UlIcetMaskFromCpEf(g_uACP, (EFam) 0);
	
	// See if the system cp makes any difference.
	//
	grfResult = grfIcet & grfSys;
	
	if (grfResult)
		{
		cIcet = _CBitsOnFromUlong(grfResult);
		if (cIcet == 1)
			{
			*lpicet = _IcetFromIcetMask(grfResult);
			return cceSuccess;
			}
		}
			
_LDefault:

    // Special case -- pick UTF-8 if it's legal and the prefs
    // don't help us.
    //
	*lpicet =
        (grfIcetOrig & grfUtf8) ? icetUtf8 : _IcetDefaultFromIcetMask(grfIcet);
	return cceSuccess;
}


/* _ C C E  R E A D  E S C  S E Q */
/*----------------------------------------------------------------------------
	%%Function: _CceReadEscSeq
	%%Contact: jpick

	We've read (and put back) an escape character.  Call the ISO-2022
	escape sequence converter to have it map the escape sequence to the
	appropriate character set.  We may be looking at the escape sequence
	for ASCII, so be prepared to read ahead to the next one.
----------------------------------------------------------------------------*/
static CCE _CceReadEscSeq(
    IStream   *pstmIn,           // input stream
	int       nPrefCp,
	ICET     *lpicet,
	BOOL     *lpfGuess
)
{
	unsigned char uch;
	CCE cceRet;
	int nToken;
	BOOL fDummy;
	
	do
		{
		cceRet = CceReadEscSeq(pstmIn, lpicet); 
		
		if ((cceRet == cceSuccess) || (cceRet != cceMayBeAscii))
			break;
		
		while (fTrue)
			{
			nToken = _NGetNextUch(pstmIn, &uch, &fDummy);
			if (_FStopToken(nToken))
				break;
			}
			
		// Why did we stop?
		//
		if (nToken == err)
			{
			cceRet = cceRead;
			break;
			}
		else if (nToken == eof)
			{
			// Means this is legal ISO-2022 input, but we've seen nothing
			// but non-flavor-specific escape sequences (e.g., only ASCII
			// or shift sequences).  Choose the encoding type based on
			// preferences (only pick from those currently supported
			// externally).
			//
			switch (nPrefCp)
				{
				case nCpKorea:
					*lpicet = icetIso2022Kr;
					break;
				case nCpJapan:
				default:						// Right ??? (gotta pick something ...)
					*lpicet = icetIso2022Jp;
					break;
				}
			*lpfGuess = fTrue;					// not *really* guessing, but ... (???)
			cceRet = cceSuccess;
			break;
			}
			
		Assert((nToken == esc) || (nToken == so) || (nToken == si));
		{
        LARGE_INTEGER   li;
        HRESULT hr;

        LISet32(li, -1 );

        hr = pstmIn->Seek(li,STREAM_SEEK_CUR, NULL);
        }
		// Put it back for CceReadEscSeq() to process.
		//
//		if (!pfnUnget(uch, lpvPrivate))
//			{
//			cceRet = cceUnget;
//			break;
//			}
			
		} while (fTrue);
	
	return cceRet;
}