Source code of Windows XP (NT5)
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

315 lines
7.8 KiB

  1. /*** token.c - functions dealing with token stream
  2. *
  3. * Copyright (c) 1996,1997 Microsoft Corporation
  4. * Author: Michael Tsang (MikeTs)
  5. * Created: 08/05/96
  6. *
  7. * This module implements a general purpose scanner. The
  8. * implementation is language independent. It is a pseudo
  9. * table driven scanner which uses a table to determine
  10. * the token type by its first character and calls the
  11. * appropriate routine to scan the rest of the token
  12. * characters.
  13. *
  14. * MODIFICATION HISTORY
  15. */
  16. #include "pch.h"
  17. /***EP OpenToken - token stream initialization
  18. *
  19. * ENTRY
  20. * pfileSrc -> source file
  21. * apfnToken -> table of token parsing functions
  22. *
  23. * EXIT-SUCCESS
  24. * returns the pointer to the allocated token structure.
  25. * EXIT-FAILURE
  26. * returns NULL.
  27. */
  28. #ifdef TUNE
  29. PTOKEN EXPORT OpenToken(FILE *pfileSrc, PFNTOKEN *apfnToken,
  30. WORD *pawcTokenType)
  31. #else
  32. PTOKEN EXPORT OpenToken(FILE *pfileSrc, PFNTOKEN *apfnToken)
  33. #endif
  34. {
  35. PTOKEN ptoken = NULL;
  36. ENTER((3, "OpenToken(pfileSrc=%p,apfnToken=%p)\n", pfileSrc, apfnToken));
  37. if ((ptoken = (PTOKEN)malloc(sizeof(TOKEN))) == NULL)
  38. MSG(("OpenToken: failed to allocate token structure"))
  39. else
  40. {
  41. memset(ptoken, 0, sizeof(TOKEN));
  42. if ((ptoken->pline = OpenLine(pfileSrc)) == NULL)
  43. {
  44. free(ptoken);
  45. ptoken = NULL;
  46. }
  47. else
  48. {
  49. ptoken->papfnToken = apfnToken;
  50. #ifdef TUNE
  51. ptoken->pawcTokenType = pawcTokenType;
  52. #endif
  53. }
  54. }
  55. EXIT((3, "OpenToken=%p\n", ptoken));
  56. return ptoken;
  57. } //OpenToken
  58. /***EP CloseToken - free token structure
  59. *
  60. * ENTRY
  61. * ptoken -> token structure
  62. *
  63. * EXIT
  64. * None
  65. */
  66. VOID EXPORT CloseToken(PTOKEN ptoken)
  67. {
  68. ENTER((3, "CloseToken(ptoken=%p)\n", ptoken));
  69. CloseLine(ptoken->pline);
  70. free(ptoken);
  71. EXIT((3, "CloseToken!\n"));
  72. } //CloseToken
  73. /***EP GetToken - get a token from a line buffer
  74. *
  75. * This procedure scans the line buffer and returns a token.
  76. *
  77. * ENTRY
  78. * ptoken -> token structure
  79. *
  80. * EXIT-SUCCESS
  81. * returns TOKERR_NONE
  82. * EXIT-FAILURE
  83. * returns error code - TOKERR_*
  84. */
  85. int EXPORT GetToken(PTOKEN ptoken)
  86. {
  87. int rc = TOKERR_NO_MATCH;
  88. ENTER((3, "GetToken(ptoken=%p)\n", ptoken));
  89. if (ptoken->wfToken & TOKF_CACHED)
  90. {
  91. ptoken->wfToken &= ~TOKF_CACHED;
  92. rc = TOKERR_NONE;
  93. }
  94. else
  95. {
  96. int c, i;
  97. do
  98. {
  99. if ((c = LineGetC(ptoken->pline)) == EOF)
  100. {
  101. ptoken->wErrLine = ptoken->pline->wLineNum;
  102. ptoken->wErrPos = ptoken->pline->wLinePos;
  103. rc = TOKERR_EOF;
  104. break;
  105. }
  106. ptoken->wTokenPos = (WORD)(ptoken->pline->wLinePos - 1);
  107. ptoken->wTokenLine = ptoken->pline->wLineNum;
  108. ptoken->iTokenType = TOKTYPE_NULL;
  109. ptoken->llTokenValue = 0;
  110. ptoken->wTokenLen = 0;
  111. ptoken->szToken[ptoken->wTokenLen++] = (char)c;
  112. for (i = 0; ptoken->papfnToken[i]; i++)
  113. {
  114. if ((rc = (*ptoken->papfnToken[i])(c, ptoken)) ==
  115. TOKERR_NO_MATCH)
  116. {
  117. continue;
  118. }
  119. else
  120. {
  121. #ifdef TUNE
  122. if (rc == TOKERR_NONE)
  123. ptoken->pawcTokenType[i]++;
  124. #endif
  125. break;
  126. }
  127. }
  128. if (rc == TOKERR_NO_MATCH)
  129. {
  130. ptoken->szToken[ptoken->wTokenLen] = '\0';
  131. ptoken->wErrLine = ptoken->pline->wLineNum;
  132. if ((ptoken->wErrPos = ptoken->pline->wLinePos) != 0)
  133. ptoken->wErrPos--;
  134. PrintTokenErr(ptoken, "unrecognized token", TRUE);
  135. }
  136. else if (rc != TOKERR_NONE)
  137. {
  138. PrintScanErr(ptoken, rc);
  139. }
  140. } while ((rc == TOKERR_NONE) && (ptoken->iTokenType == TOKTYPE_NULL));
  141. }
  142. EXIT((3, "GetToken=%d (Type=%d,Value=%I64d,Token=%s,TokenLine=%d,TokenPos=%d)\n",
  143. rc, ptoken->iTokenType, ptoken->llTokenValue,
  144. ptoken->szToken, ptoken->wTokenLine, ptoken->wTokenPos));
  145. return rc;
  146. } //GetToken
  147. /***EP UnGetToken - push a token back to the token stream
  148. *
  149. * This procedure unget the last token.
  150. *
  151. * ENTRY
  152. * ptoken -> token structure
  153. *
  154. * EXIT-SUCCESS
  155. * returns TOKERR_NONE
  156. * EXIT-FAILURE
  157. * returns error code - TOKERR_*
  158. */
  159. int EXPORT UnGetToken(PTOKEN ptoken)
  160. {
  161. int rc;
  162. ENTER((3, "UnGetToken(ptoken=%p)\n", ptoken));
  163. if (!(ptoken->wfToken & TOKF_CACHED))
  164. {
  165. ptoken->wfToken |= TOKF_CACHED;
  166. rc = TOKERR_NONE;
  167. }
  168. else
  169. {
  170. ASSERT(ptoken->wfToken & TOKF_CACHED);
  171. rc = TOKERR_ASSERT_FAILED;
  172. }
  173. EXIT((3, "UnGetToken=%d\n", rc));
  174. return rc;
  175. } //UnGetToken
  176. /***EP MatchToken - Match the next token type
  177. *
  178. * ENTRY
  179. * ptoken -> token structure
  180. * iTokenType - token type to match
  181. * lTokenValue - token value to match
  182. * dwfMatch - match flags
  183. * pszErrMsg -> error message to print if not matched
  184. *
  185. * EXIT-SUCCESS
  186. * returns TOKERR_NONE
  187. * EXIT-FAILURE
  188. * returns negative error code
  189. */
  190. int EXPORT MatchToken(PTOKEN ptoken, int iTokenType, LONG lTokenValue,
  191. DWORD dwfMatch, PSZ pszErrMsg)
  192. {
  193. int rc;
  194. ENTER((3, "MatchToken(ptoken=%p,TokType=%d,TokValue=%ld,dwfMatch=%lx,ErrMsg=%s)\n",
  195. ptoken, iTokenType, lTokenValue, dwfMatch,
  196. pszErrMsg? pszErrMsg: "<none>"));
  197. if (((rc = GetToken(ptoken)) == TOKERR_NONE) &&
  198. ((ptoken->iTokenType != iTokenType) ||
  199. !(dwfMatch & MTF_ANY_VALUE) &&
  200. ((LONG)ptoken->llTokenValue != lTokenValue)))
  201. {
  202. if (dwfMatch & MTF_NOT_ERR)
  203. {
  204. UnGetToken(ptoken);
  205. rc = TOKERR_NO_MATCH;
  206. }
  207. else
  208. {
  209. rc = TOKERR_SYNTAX;
  210. }
  211. }
  212. if ((rc != TOKERR_NONE) && !(dwfMatch & MTF_NOT_ERR))
  213. {
  214. char szMsg[MAX_MSG_LEN + 1];
  215. if (pszErrMsg == NULL)
  216. {
  217. sprintf(szMsg, "expecting %s",
  218. gapszTokenType[iTokenType - TOKTYPE_LANG - 1]);
  219. if (!(dwfMatch & MTF_ANY_VALUE) && (iTokenType == TOKTYPE_SYMBOL))
  220. {
  221. sprintf(&szMsg[strlen(szMsg)], " '%c'",
  222. SymCharTable[lTokenValue - 1]);
  223. }
  224. pszErrMsg = szMsg;
  225. }
  226. PrintTokenErr(ptoken, pszErrMsg, TRUE);
  227. if (rc == TOKERR_EOF)
  228. {
  229. rc = TOKERR_SYNTAX;
  230. }
  231. }
  232. EXIT((3, "MatchToken=%d (Type=%d,Value=%I64d,Token=%s)\n",
  233. rc, ptoken->iTokenType, ptoken->llTokenValue, ptoken->szToken));
  234. return rc;
  235. } //MatchToken
  236. /***EP PrintTokenErr - print token error line
  237. *
  238. * ENTRY
  239. * ptoken -> token structure
  240. * pszErrMsg -> error message string
  241. * fErr - TRUE if it is an error, FALSE if warning
  242. *
  243. * EXIT
  244. * None
  245. */
  246. VOID EXPORT PrintTokenErr(PTOKEN ptoken, PSZ pszErrMsg, BOOL fErr)
  247. {
  248. WORD i;
  249. ENTER((3, "PrintTokenErr(ptoken=%p,Line=%d,Pos=%d,Msg=%s)\n",
  250. ptoken, ptoken->wTokenLine, ptoken->wTokenPos, pszErrMsg));
  251. ErrPrintf("\n%5u: %s",
  252. ptoken->wTokenLine, ptoken->pline->szLineBuff);
  253. ErrPrintf(" ");
  254. for (i = 0; i < ptoken->wTokenPos; ++i)
  255. {
  256. if (ptoken->pline->szLineBuff[i] == '\t')
  257. {
  258. ErrPrintf("\t");
  259. }
  260. else
  261. {
  262. ErrPrintf(" ");
  263. }
  264. }
  265. ErrPrintf("^***\n");
  266. if (pszErrMsg != NULL)
  267. {
  268. ErrPrintf("%s(%d): %s: %s\n",
  269. gpszASLFile, ptoken->wTokenLine, fErr? "error": "warning",
  270. pszErrMsg);
  271. }
  272. EXIT((3, "PrintTokenErr!\n"));
  273. } //PrintTokenErr