|
|
/************************************************************************//* *//* RCPP - Resource Compiler Pre-Processor for NT system *//* *//* P0GETTOK.C - Tokenization routines *//* *//* 29-Nov-90 w-BrianM Update for NT from PM SDK RCPP *//* *//************************************************************************/
#include <stdio.h>
#include "rcpptype.h"
#include "rcppdecl.h"
#include "rcppext.h"
#include "grammar.h"
#include "p0defs.h"
#include "charmap.h"
/************************************************************************
** MAP_TOKEN : a token has two representations and additional information.** (ex : const, has basic token of L_CONST,** mapped token of [L_TYPE | L_MODIFIER]** and info based on what the map token is)** MAP_AND_FILL : has two representations, but none of the extra info.** (ex : '<', has basic of L_LT, and map of L_RELOP)** NOMAP_TOKEN : has 1 representation and additional info.** (ex: a string, basic and 'map' type L_STRING and ptrs to the actual str)** NOMAP_AND_FILL : has 1 representation and no additional info.** (ex : 'while', has basic and 'map' of L_WHILE)** the FILL versions fill the token with the basic token type.************************************************************************/#define MAP_TOKEN(otok)\
(Basic_token = (otok), TS_VALUE(Basic_token))#define MAP_AND_FILL(otok)\
(yylval.yy_token = Basic_token = (otok), TS_VALUE(Basic_token))#define NOMAP_TOKEN(otok)\
(Basic_token = (otok))#define NOMAP_AND_FILL(otok)\
(yylval.yy_token = Basic_token = (otok))
/************************************************************************//* yylex - main tokenization routine *//************************************************************************/
token_t yylex(void){ REG UCHAR last_mapped; UCHAR mapped_c; REG token_t lex_token;
for(;;) { last_mapped = mapped_c = CHARMAP(GETCH());first_switch: switch(mapped_c) { case LX_EACH: case LX_ASCII: Msg_Temp = GET_MSG(2018); SET_MSG (Msg_Text, Msg_Temp, PREVCH()); error(2018); continue; break; case LX_OBRACE: return(NOMAP_AND_FILL(L_LCURLY)); break; case LX_CBRACE: return(NOMAP_AND_FILL(L_RCURLY)); break; case LX_OBRACK: return(NOMAP_AND_FILL(L_LBRACK)); break; case LX_CBRACK: return(NOMAP_AND_FILL(L_RBRACK)); break; case LX_OPAREN: return(NOMAP_AND_FILL(L_LPAREN)); break; case LX_CPAREN: return(NOMAP_AND_FILL(L_RPAREN)); break; case LX_COMMA: return(NOMAP_AND_FILL(L_COMMA)); break; case LX_QUEST: return(NOMAP_AND_FILL(L_QUEST)); break; case LX_SEMI: return(NOMAP_AND_FILL(L_SEMI)); break; case LX_TILDE: return(NOMAP_AND_FILL(L_TILDE)); break; case LX_NUMBER: return(MAP_TOKEN(getnum(PREVCH()))); break;
case LX_MINUS: switch(last_mapped = CHARMAP(GETCH())) { case LX_EQ: return(MAP_AND_FILL(L_MINUSEQ)); break; case LX_GT: return(MAP_AND_FILL(L_POINTSTO)); break; case LX_MINUS: return(MAP_AND_FILL(L_DECR)); break; default: lex_token = L_MINUS; break; } break; case LX_PLUS: switch(last_mapped = CHARMAP(GETCH())) { case LX_EQ: return(MAP_AND_FILL(L_PLUSEQ)); break; case LX_PLUS: return(MAP_AND_FILL(L_INCR)); break; default: lex_token = L_PLUS; break; } break; case LX_AND: switch(last_mapped = CHARMAP(GETCH())) { case LX_EQ: return(MAP_AND_FILL(L_ANDEQ)); break; case LX_AND: return(MAP_AND_FILL(L_ANDAND)); break; default: lex_token = L_AND; break; } break; case LX_OR: switch(last_mapped = CHARMAP(GETCH())) { case LX_EQ: return(MAP_AND_FILL(L_OREQ)); break; case LX_OR: return(MAP_AND_FILL(L_OROR)); break; default: lex_token = L_OR; break; } break; case LX_COLON: return(NOMAP_AND_FILL(L_COLON)); break; case LX_HAT: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_XOREQ)); } lex_token = L_XOR; break; case LX_PERCENT: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_MODEQ)); } lex_token = L_MOD; break; case LX_EQ: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_EQUALS)); } lex_token = L_ASSIGN; break; case LX_BANG: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_NOTEQ)); } lex_token = L_EXCLAIM; break; case LX_SLASH: switch(last_mapped = CHARMAP(GETCH())) { case LX_STAR: dump_comment(); continue; break; case LX_SLASH: DumpSlashComment(); continue; break; case LX_EQ: return(MAP_AND_FILL(L_DIVEQ)); break; default: lex_token = L_DIV; break; } break; case LX_STAR: switch(last_mapped = CHARMAP(GETCH())) { case LX_SLASH: if( ! Prep ) { Msg_Temp = GET_MSG(2138); SET_MSG (Msg_Text, Msg_Temp); error(2138); /* (nested comments) */ } else { fwrite("*/", 2, 1, OUTPUTFILE); } continue; case LX_EQ: return(MAP_AND_FILL(L_MULTEQ)); break; default: lex_token = L_MULT; break; } break; case LX_LT: switch(last_mapped = CHARMAP(GETCH())) { case LX_LT: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_LSHFTEQ)); } mapped_c = LX_LSHIFT; lex_token = L_LSHIFT; break; case LX_EQ: return(MAP_AND_FILL(L_LTEQ)); break; default: lex_token = L_LT; break; } break; case LX_LSHIFT: /*
** if the next char is not an =, then we unget and return, ** since the only way in here is if we broke on the char ** following '<<'. since we'll have already worked the handle_eos() ** code prior to getting here, we'll not see another eos, ** UNLESS i/o buffering is char by char. ??? ** see also, LX_RSHIFT */ if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_LSHFTEQ)); } UNGETCH(); return(MAP_AND_FILL(L_LSHIFT)); break; case LX_GT: switch(last_mapped = CHARMAP(GETCH())) { case LX_EQ: return(MAP_AND_FILL(L_GTEQ)); case LX_GT: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_RSHFTEQ)); } mapped_c = LX_RSHIFT; lex_token = L_RSHIFT; break; default: lex_token = L_GT; break; } break; case LX_RSHIFT: if((last_mapped = CHARMAP(GETCH())) == LX_EQ) { return(MAP_AND_FILL(L_RSHFTEQ)); } UNGETCH(); return(MAP_AND_FILL(L_RSHIFT)); break; case LX_POUND: if( ! Prep ) { Msg_Temp = GET_MSG(2014); SET_MSG (Msg_Text, Msg_Temp); error(2014);/* # sign must be first non-whitespace */ UNGETCH(); /* replace it */ Linenumber--; /* do_newline counts a newline */ do_newline(); /* may be a 'real' prepro line */ } else { fwrite("#", 1, 1, OUTPUTFILE); } continue; break; case LX_EOS: if(PREVCH() == '\\') { if( ! Prep ) { if( ! checknl()) { /* ignore the new line */ Msg_Temp = GET_MSG(2017); SET_MSG (Msg_Text, Msg_Temp); error(2017);/* illegal escape sequence */ } } else { fputc('\\', OUTPUTFILE); fputc(get_non_eof(), OUTPUTFILE); } continue; } if(Macro_depth == 0) { if( ! io_eob()) { /* not the end of the buffer */ continue; } if(fpop()) { /* have more files to read */ continue; } return(MAP_AND_FILL(L_EOF)); /* all gone . . . */ } handle_eos(); /* found end of macro */ continue; break; case LX_DQUOTE: if( ! Prep ) { str_const(); return(NOMAP_TOKEN(L_STRING)); } prep_string('\"'); continue; break; case LX_SQUOTE: if( ! Prep ) { return(MAP_TOKEN(char_const())); } prep_string('\''); continue; break; case LX_CR: /* ??? check for nl next */ continue; break; case LX_NL: if(On_pound_line) { UNGETCH(); return(NOMAP_TOKEN(L_NOTOKEN)); } if(Prep) { fputc('\n', OUTPUTFILE); } do_newline(); continue; break; case LX_WHITE: /* skip all white space */ if( ! Prep ) { /* check only once */ do { ; } while(LXC_IS_WHITE(GETCH())); } else { UCHAR c;
c = PREVCH(); do { fputc(c, OUTPUTFILE); } while(LXC_IS_WHITE(c = GETCH())); } UNGETCH(); continue; break; /* Note:
* RCPP.EXE does not support DBCS code. * Therefore, we should be displaied error message. * IBM-J PTR 12JP-0092 * MSHQ PTR xxxxx */ case LX_LEADBYTE: if( ! Prep ) { /* check only once */ Msg_Temp = GET_MSG(2018); SET_MSG (Msg_Text, Msg_Temp, PREVCH()); error(2018);
Msg_Temp = GET_MSG(2018); SET_MSG (Msg_Text, Msg_Temp, GETCH()); error(2018); } else { fputc(PREVCH(), OUTPUTFILE);#ifdef DBCS // token_t yylex(void)
fputc(get_non_eof(), OUTPUTFILE);#else
fputc(GETCH(), OUTPUTFILE);#endif // DBCS
} continue; break; case LX_ILL: if( ! Prep ) { Msg_Temp = GET_MSG(2018); SET_MSG (Msg_Text, Msg_Temp, PREVCH()); error(2018);/* unknown character */ } else { fputc(PREVCH(), OUTPUTFILE); } continue; break; case LX_BACKSLASH: if( ! Prep ) { if( ! checknl()) { /* ignore the new line */ Msg_Temp = GET_MSG(2017); SET_MSG (Msg_Text, Msg_Temp); error(2017);/* illegal escape sequence */ } } else { fputc('\\', OUTPUTFILE); fputc(get_non_eof(), OUTPUTFILE); } continue; break; case LX_DOT:dot_switch: switch(last_mapped = CHARMAP(GETCH())) { case LX_BACKSLASH: if(checknl()) { goto dot_switch; } UNGETCH(); break; case LX_EOS: if(handle_eos() == BACKSLASH_EOS) { break; } goto dot_switch; break; case LX_DOT: if( ! checkop('.') ) { Msg_Temp = GET_MSG(2142); SET_MSG (Msg_Text, Msg_Temp); error(2142);/* ellipsis requires three '.'s */ } return(NOMAP_AND_FILL(L_ELLIPSIS)); break; case LX_NUMBER: /*
** don't worry about getting correct hash value. ** The text equivalent of a real number is never ** hashed */ Reuse_1[0] = '.'; Reuse_1[1] = PREVCH(); return(MAP_TOKEN(get_real(&Reuse_1[2]))); break; } UNGETCH(); return(MAP_AND_FILL(L_PERIOD)); break; case LX_NOEXPAND: SKIPCH(); /* just skip length */ continue; case LX_ID: { pdefn_t pdef;
if(Macro_depth > 0) { if( ! lex_getid(PREVCH())) { goto avoid_expand; } } else { getid(PREVCH()); }
if( ((pdef = get_defined()) != 0) && ( ! DEFN_EXPANDING(pdef)) && ( can_expand(pdef)) ) { continue; }
avoid_expand: if( ! Prep ) { /* M00BUG get near copy of identifier???? */ HLN_NAME(yylval.yy_ident) = Reuse_1; HLN_HASH(yylval.yy_ident) = Reuse_1_hash; HLN_LENGTH(yylval.yy_ident) = (UCHAR)Reuse_1_length; return(L_IDENT); } else { fwrite(Reuse_1, Reuse_1_length - 1, 1, OUTPUTFILE); return(NOMAP_TOKEN(L_NOTOKEN)); } } continue; break; } /*
** all the multichar ( -> -- -= etc ) operands ** must come through here. we've gotten the next char, ** and not matched one of the possiblities, but we have to check ** for the end of the buffer character and act accordingly ** if it is the eob, then we handle it and go back for another try. ** otherwise, we unget the char we got, and return the base token. */ if(last_mapped == LX_EOS) { if(handle_eos() != BACKSLASH_EOS) { goto first_switch; } } UNGETCH(); /* cause we got an extra one to check */ return(MAP_AND_FILL(lex_token)); }}
/************************************************************************
**** lex_getid: reads an identifier for the main lexer. The** identifier is read into Reuse_1. This function should not handle** an end of string if it is rescanning a macro expansion, because** this could switch the context with regards to whether the macro** is expandable or not. Similarly, the noexpand marker must only be** allowed if a macro is being rescanned, otherwise let this character** be caught as an illegal character in text************************************************************************/int lex_getid(UCHAR c){ REG UCHAR *p; int length = 0;
p = Reuse_1; *p++ = c; c &= HASH_MASK; for(;;) { while(LXC_IS_IDENT(*p = GETCH())) { /* collect character */ c += (*p & HASH_MASK); /* hash it */ p++; } if(CHARMAP(*p) == LX_NOEXPAND ) { length = (int)GETCH(); continue; } UNGETCH(); break; /* out of for loop - only way out */ } if(p >= LIMIT(Reuse_1)) { /* is this error # correct? */ Msg_Temp = GET_MSG(1067); SET_MSG (Msg_Text, Msg_Temp); fatal(1067); } if(((p - Reuse_1) > LIMIT_ID_LENGTH) && ( ! Prep )) { p = Reuse_1 + LIMIT_ID_LENGTH; *p = '\0'; c = local_c_hash(Reuse_1); Msg_Temp = GET_MSG(4011); SET_MSG (Msg_Text, Msg_Temp, Reuse_1); warning(4011); /* id truncated */ } else { *p = '\0'; /* terminates identifier for expandable check */ } Reuse_1_hash = c; Reuse_1_length = (UCHAR)((p - Reuse_1) + 1); return(length != (p - Reuse_1));}
|
