|
|
/*++
Copyright (c) 2000 Microsoft Corporation
Module Name:
adllexer.cpp
Abstract:
Implementation of the lexer for the ADL language
Author:
t-eugenz - August 2000
Environment:
User mode only.
Revision History:
Created - August 2000
--*/
#include "adl.h"
//
// Constant values outside WCHAR range, for special characters
//
#define CHAR_COMMA 65538
#define CHAR_QUOTE 65539
#define CHAR_SEMICOLON 65540
#define CHAR_OPENPAREN 65541
#define CHAR_CLOSEPAREN 65542
#define CHAR_NULL 65543
#define CHAR_NEWLINE 65544
#define CHAR_RETURN 65545
#define CHAR_TAB 65546
#define CHAR_SPACE 65547
#define CHAR_AT 65548
#define CHAR_SLASH 65549
#define CHAR_PERIOD 65550
//
// States of the lexer DFA
//
#define STATE_WHITESPACE 0
#define STATE_BEGIN 1
#define STATE_IDENT 2
#define STATE_QUOTE 3
#define STATE_DONE 4
//
// If the character is found in the special character map, use the special
// symbol (>65535), otherwise use the regular character value
//
#define RESOLVE_CHAR(CHAR, MAP, ITER, ITEREND) \
((((ITER) = (MAP).find((CHAR)) ) == (ITEREND) ) ? (CHAR) : (*(ITER)).second)
AdlLexer::AdlLexer(IN const WCHAR *input, IN OUT AdlStatement *adlStat, IN const PADL_LANGUAGE_SPEC pLang) /*++
Routine Description:
Constructor for the AdlLexer. Initializes the mapping for finding special characters, and other initial state information
Arguments:
input - The input string adlStat - The AdlStatement instance, for token garbage collection pLang - The ADL language description
Return Value: none --*/
{
_input = input; _pLang = pLang; _adlStat = adlStat;
_position = 0; _tokCount = 0;
//
// Special character mapping
//
_mapCharCode[_pLang->CH_NULL] = CHAR_NULL; _mapCharCode[_pLang->CH_SPACE] = CHAR_SPACE; _mapCharCode[_pLang->CH_TAB] = CHAR_TAB; _mapCharCode[_pLang->CH_NEWLINE] = CHAR_NEWLINE; _mapCharCode[_pLang->CH_RETURN] = CHAR_RETURN; _mapCharCode[_pLang->CH_QUOTE] = CHAR_QUOTE; _mapCharCode[_pLang->CH_COMMA] = CHAR_COMMA; _mapCharCode[_pLang->CH_SEMICOLON] = CHAR_SEMICOLON; _mapCharCode[_pLang->CH_OPENPAREN] = CHAR_OPENPAREN; _mapCharCode[_pLang->CH_CLOSEPAREN] = CHAR_CLOSEPAREN; _mapCharCode[_pLang->CH_AT] = CHAR_AT; _mapCharCode[_pLang->CH_SLASH] = CHAR_SLASH; _mapCharCode[_pLang->CH_PERIOD] = CHAR_PERIOD;
//
// Only find end of map once
//
_iterEnd = _mapCharCode.end();
//
// Place all special tokens into a map, for O(log n) string searches
//
_mapStringToken[_pLang->SZ_TK_AND] = TK_AND; _mapStringToken[_pLang->SZ_TK_EXCEPT] = TK_EXCEPT; _mapStringToken[_pLang->SZ_TK_ON] = TK_ON; _mapStringToken[_pLang->SZ_TK_ALLOWED] = TK_ALLOWED; _mapStringToken[_pLang->SZ_TK_AS] = TK_AS; _mapStringToken[_pLang->SZ_TK_THIS_OBJECT] = TK_THIS_OBJECT; _mapStringToken[_pLang->SZ_TK_CONTAINERS] = TK_CONTAINERS; _mapStringToken[_pLang->SZ_TK_OBJECTS] = TK_OBJECTS; _mapStringToken[_pLang->SZ_TK_CONTAINERS_OBJECTS] = TK_CONTAINERS_OBJECTS; _mapStringToken[_pLang->SZ_TK_NO_PROPAGATE] = TK_NO_PROPAGATE;
}
DWORD AdlLexer::NextToken(OUT AdlToken **value) /*++
Routine Description:
This retrieves the next token from the input string. This is basically a DFA which begins in the WHITESPACE state, and runs until it reaches the DONE state, at which point it returns a token. Arguments:
value - Pointer to a new token containing the string value is stored in *value Return Value: DWORD - The token type, as #define'd by YACC in tokens.h --*/{
//
// Initial DFA state
//
DWORD state = STATE_WHITESPACE; DWORD tokType = TK_ERROR; wstring curToken; DWORD dwInput;
DWORD dwTokStart = 0;
//
// First token should be the grammar type
//
if( _tokCount == 0 ) { _tokCount++; return _pLang->dwLanguageType; }
dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd);
while( state != STATE_DONE ) { switch( state ) {
case STATE_WHITESPACE:
switch( dwInput ) { case CHAR_NULL: tokType = 0; state = STATE_DONE; break;
case CHAR_NEWLINE: _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd);
break;
case CHAR_RETURN: _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd); break;
case CHAR_SPACE: _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd); break;
case CHAR_TAB: _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd); break;
default: state = STATE_BEGIN; break; } break; case STATE_BEGIN:
dwTokStart = _position;
tokType = TK_ERROR;
switch( dwInput ) { case CHAR_NULL: state = STATE_DONE; break;
case CHAR_COMMA: if( tokType == TK_ERROR ) { tokType = TK_COMMA; }
case CHAR_OPENPAREN: if( tokType == TK_ERROR ) { tokType = TK_OPENPAREN; }
case CHAR_CLOSEPAREN: if( tokType == TK_ERROR ) { tokType = TK_CLOSEPAREN; }
case CHAR_SEMICOLON: if( tokType == TK_ERROR ) { tokType = TK_SEMICOLON; }
case CHAR_AT: if( tokType == TK_ERROR ) { tokType = TK_AT; }
case CHAR_SLASH: if( tokType == TK_ERROR ) { tokType = TK_SLASH; }
case CHAR_PERIOD: if( tokType == TK_ERROR ) { tokType = TK_PERIOD; }
//
// Same action for all special single-char tokens
//
curToken.append( &(_input[_position]), 1 ); _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd);
state = STATE_DONE; break; case CHAR_QUOTE: _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd);
state = STATE_QUOTE; tokType = TK_IDENT; break;
default: state = STATE_IDENT; tokType = TK_IDENT; break; }
break;
case STATE_IDENT:
switch( dwInput ) { case CHAR_NULL: case CHAR_COMMA: case CHAR_OPENPAREN: case CHAR_CLOSEPAREN: case CHAR_SEMICOLON: case CHAR_NEWLINE: case CHAR_RETURN: case CHAR_TAB: case CHAR_SPACE: case CHAR_AT: case CHAR_SLASH: case CHAR_PERIOD: case CHAR_QUOTE:
state = STATE_DONE; break;
default: curToken.append( &(_input[_position]), 1 ); _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd);
break; }
break;
case STATE_QUOTE:
switch( dwInput ) { case CHAR_NULL: case CHAR_TAB: case CHAR_NEWLINE: case CHAR_RETURN: throw AdlStatement::ERROR_UNTERMINATED_STRING; break;
case CHAR_QUOTE:
_position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd); state = STATE_DONE; break;
default: curToken.append( &(_input[_position]), 1 ); _position++; dwInput = RESOLVE_CHAR(_input[_position], _mapCharCode, _iter, _iterEnd); break; }
break;
default:
//
// Should never get here, well-defined states
//
assert(FALSE); break; } } //
// Done state was reached
// Export the string and column/row info in YACC-form here
//
AdlToken *outVal; outVal = new AdlToken(curToken.c_str(), dwTokStart, _position - 1); _adlStat->AddToken(outVal);
//
// Check if the string is a special token, case-insensitive
//
if( _mapStringToken.find(outVal->GetValue()) != _mapStringToken.end() ) { tokType = _mapStringToken[outVal->GetValue()]; }
*value = outVal;
//
// Set this token to be the error token. This way, if the string is
// not accepted by the parser, we know at which token the parser failed
// If another error occurs later, this value will be overwritten
//
_adlStat->SetErrorToken(outVal);
_tokCount++;
return tokType;}
|
