|
|
/*----------------------------------------------------------------------------
%%File: jislex.c %%Unit: fechmap %%Contact: jpick
Simple converter for decoding a subset of possible ISO-2022-7 encoded files (ISO-2022). Data is translated to and from Unicode. Converter operates according to user options. Module currently handles ISO-2022-JP (and JIS) and ISO-2022-KR. Converter is set up to handle ISO-2022-TW and ISO-2022-CN, but there are as yet no conversion tables for these.----------------------------------------------------------------------------*/
#include <stdio.h>
#include <stddef.h>
#include "private.h"
#include "fechmap_.h"
#include "lexint_.h"
// State table for reading ISO-2022-7 encoded text
//
// Lexer recognizes the following designator sequences, used
// to select a one or two byte character set:
//
// <esc> $ @ -- JIS C 6626-1978 (synonym of <esc> $ ( @)
// <esc> $ A -- GB 2312-80 (synonym of <esc> $ ( A)
// <esc> $ B -- JIS X 0208-1983 (synonym of <esc> $ ( B)
//
// <esc> $ ( @ -- JIS C 6626-1978
// <esc> $ ( A -- GB 2312-80
// <esc> $ ( B -- JIS X 0208-1983
// <esc> $ ( C -- KS C 5601-1992
// <esc> $ ( D -- JIS X 0212-1990
// <esc> $ ( E -- ??? (ISO-IR-165:1992) ???
// <esc> $ ( G -- CNS 11643-1992 Plane 1
// <esc> $ ( H -- CNS 11643-1992 Plane 2
// <esc> $ ( I -- CNS 11643-1992 Plane 3
// <esc> $ ( J -- CNS 11643-1992 Plane 4
// <esc> $ ( K -- CNS 11643-1992 Plane 5
// <esc> $ ( L -- CNS 11643-1992 Plane 6
// <esc> $ ( M -- CNS 11643-1992 Plane 7
//
// <esc> $ ) C -- KSC 5601-1987 (Implies ISO-2022-KR ??)
//
// <esc> & @ <esc> $ B -- JIS X 0208-1990
//
// <esc> ( B -- Ascii
// <esc> ( H -- Deprecated variant of JIS-Roman
// <esc> ( I -- Half-Width Katakana
// <esc> ( J -- JIS-Roman
// <esc> ( T -- GB 1988-89 Roman
//
// Lexer recognizes the following shift sequences, used to allow
// interpretation of a given byte or bytes:
//
// <si> -- locking shift, interpret bytes as G0
// <so> -- locking shift, interpret bytes as G1
// <esc> n -- locking shift, interpret bytes as G2
// <esc> o -- locking shift, interpret bytes as G3
// <esc> N -- single shift, interpret bytes as G2
// <esc> O -- single shift, interpret bytes as G3
//
// REVIEW (jpick): don't currently need the final four shift
// sequences. If we support ISO-2022-CN, we'll need to use
// G2 and G3 and potentially, then, the last four shifts.
//
/*----------------------------------------------------------------------------
Character Classification Table----------------------------------------------------------------------------*/
// Tokens
//
#define txt (JTK) 0
#define ext (JTK) 1 // extended characters that are legal under certain circumstances
#define esc (JTK) 2
#define si (JTK) 3
#define so (JTK) 4
#define dlr (JTK) 5
#define at (JTK) 6
#define amp (JTK) 7
#define opr (JTK) 8
#define cpr (JTK) 9
#define tkA (JTK) 10
#define tkB (JTK) 11
#define tkC (JTK) 12
#define tkD (JTK) 13
#define tkE (JTK) 14
#define tkG (JTK) 15
#define tkH (JTK) 16
#define tkI (JTK) 17
#define tkJ (JTK) 18
#define tkK (JTK) 19
#define tkL (JTK) 20
#define tkM (JTK) 21
#define tkT (JTK) 22
#define unk (JTK) 23 // Unexpected character
#define eof (JTK) 24 // end-of-file
#define err (JTK) 25 // read error
#define nTokens 26
// Lookup table for ISO-2022-7 encoded files
//
static JTK _rgjtkCharClass[256] =// 0 1 2 3 4 5 6 7 8 9 a b c d e f
{// nul soh stx etx eot enq ack bel bs tab lf vt np cr so si 0
txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, so, si,
// dle dc1 dc2 dc3 dc4 nak syn etb can em eof esc fs gs rs us 1
txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, esc, txt, txt, txt, txt,
// sp ! " # $ % & ' ( ) * + , - . / 2
txt, txt, txt, txt, dlr, txt, amp, txt, opr, cpr, txt, txt, txt, txt, txt, txt,
// 0 1 2 3 4 5 6 7 8 9 : ; < = > ? 3
txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt,
// @ A B C D E F G H I J K L M N O 4
at, tkA, tkB, tkC, tkD, tkE, txt, tkG, tkH, tkI, tkJ, tkK, tkL, tkM, txt, txt,
// P Q R S T U V W X Y Z [ \ ] ^ _ 5
txt, txt, txt, txt, tkT, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt,
// ` a b c d e f g h i j k l m n o 6
txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt,
// p q r s t u v w x y z { | } ~ del 7
txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt, txt,
// 8
unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk,
// 9
unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk,
// a
unk, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext,
// b
ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext,
// c
ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext,
// d
ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext, ext,
// e
unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk,
// f
unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk, unk,
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
};
/*----------------------------------------------------------------------------
State Table----------------------------------------------------------------------------*/
// Final states have the high-bit set. States that represent the reading
// of a valid character escape sequence also encode the character set
// "name" (moniker??) -- the state with the high bit masked off.
//
// Table State
//
typedef unsigned char TST;
// Final State Mask, Related
//
#define grfFinal (TST) 0x80
#define _NEscTypeFromState(nState) (int) ((nState) & 0x7f)
// ASCII Escape Sequence (Final State)
#define ASC (TST) (grfFinal | 0x00) // Ascii
// Japanese Escape Sequences (Final States)
#define JS0 (TST) (grfFinal | 0x01) // JIS-Roman
#define JS1 (TST) (grfFinal | 0x02) // Half-Width Katakana
#define JS2 (TST) (grfFinal | 0x03) // JIS C 6226-1978
#define JS3 (TST) (grfFinal | 0x04) // JIS X 0208-1983
#define JS4 (TST) (grfFinal | 0x05) // JIS X 0208-1990
#define JS5 (TST) (grfFinal | 0x06) // JIS X 0212-1990
// Chinese (PRC) Escape Sequences (Final States)
#define CS0 (TST) (grfFinal | 0x07) // GB 1988-89 Roman
#define CS1 (TST) (grfFinal | 0x08) // GB 2312-80
// Chinese (Taiwan) Escape Sequences (Final States)
#define TS0 (TST) (grfFinal | 0x09) // CNS 11643-1992 Plane 1
#define TS1 (TST) (grfFinal | 0x0a) // CNS 11643-1992 Plane 2
#define TS2 (TST) (grfFinal | 0x0b) // CNS 11643-1992 Plane 3
#define TS3 (TST) (grfFinal | 0x0c) // CNS 11643-1992 Plane 4
#define TS4 (TST) (grfFinal | 0x0d) // CNS 11643-1992 Plane 5
#define TS5 (TST) (grfFinal | 0x0e) // CNS 11643-1992 Plane 6
#define TS6 (TST) (grfFinal | 0x0f) // CNS 11643-1992 Plane 7
// Korean Escape Sequences (Final State)
#define KS0 (TST) (grfFinal | 0x10) // KS C 5601-1992
// Document "Signal" for ISO-2022-KR (Doc needs special processing)
#define KSD (TST) (grfFinal | 0x11) // ISO-2022-KR Document Signal
// Number of unique *character set* escape sequences
//
#define cCsEsc 18
// Special States (not escape sequence) (Final States)
//
#define TXT (TST) (grfFinal | (cCsEsc + 1)) // Process Text
#define EXT (TST) (grfFinal | (cCsEsc + 2)) // Process (Possibly Illegal) Extended Chars
#define FIN (TST) (grfFinal | (cCsEsc + 3)) // Finish
#define EOI (TST) (grfFinal | (cCsEsc + 4)) // Unexpected End-Of-Input
#define UNK (TST) (grfFinal | (cCsEsc + 5)) // Unknown State (Unexpected Character)
#define ERR (TST) (grfFinal | (cCsEsc + 6)) // Read Error
// Shift Sequences (do not specify character set) (Final States)
//
#define LSO (TST) (grfFinal | (cCsEsc + 7)) // Locking shift out (g1 into GL)
#define LSI (TST) (grfFinal | (cCsEsc + 8)) // Locking shift in (g0 into GL)
// For convenience, also define constants for the sets
// that the states represent.
//
#define csNIL (-1) // Invalid Designator
#define csASC (_NEscTypeFromState(ASC)) // Ascii
#define csJS0 (_NEscTypeFromState(JS0)) // JIS-Roman
#define csJS1 (_NEscTypeFromState(JS1)) // Half-Width Katakana
#define csJS2 (_NEscTypeFromState(JS2)) // JIS C 6226-1978
#define csJS3 (_NEscTypeFromState(JS3)) // JIS X 0208-1983
#define csJS4 (_NEscTypeFromState(JS4)) // JIS X 0208-1990
#define csJS5 (_NEscTypeFromState(JS5)) // JIS X 0212-1990
#define csCS0 (_NEscTypeFromState(CS0)) // GB 1988-89 Roman
#define csCS1 (_NEscTypeFromState(CS1)) // GB 2312-80
#define csTS0 (_NEscTypeFromState(TS0)) // CNS 11643-1992 Plane 1
#define csTS1 (_NEscTypeFromState(TS1)) // CNS 11643-1992 Plane 2
#define csTS2 (_NEscTypeFromState(TS2)) // CNS 11643-1992 Plane 3
#define csTS3 (_NEscTypeFromState(TS3)) // CNS 11643-1992 Plane 4
#define csTS4 (_NEscTypeFromState(TS4)) // CNS 11643-1992 Plane 5
#define csTS5 (_NEscTypeFromState(TS5)) // CNS 11643-1992 Plane 6
#define csTS6 (_NEscTypeFromState(TS6)) // CNS 11643-1992 Plane 7
#define csKS0 (_NEscTypeFromState(KS0)) // KS C 5601-1992 (into G0)
#define csKSD (_NEscTypeFromState(KSD)) // KS C 5601-1992 (into G1)
// Table States (Intermediate States)
#define ST0 (TST) 0
#define ST1 (TST) 1
#define ST2 (TST) 2
#define ST3 (TST) 3
#define ST4 (TST) 4
#define ST5 (TST) 5
#define ST6 (TST) 6
#define ST7 (TST) 7
#define ST8 (TST) 8
#define ST9 (TST) 9
// Number of "real" (table) states
//
#define nStates 10
#define IsFinal(state) ((state) & grfFinal)
// State Have Seen Looking For
// ----------------------------------------------------------
// ST0 -- Start State -- <ESC> Text
// ST1 <ESC> $ & (
// ST2 <ESC> $ ( ) @ A B (**)
// ST3 <ESC> $ ( @ A B C D E G H I J K L M
// ST4 <ESC> $ ) C
// ST5 <ESC> & @
// ST6 <ESC> & @ <ESC>
// ST7 <ESC> & @ <ESC> $
// ST8 <ESC> & @ <ESC> $ B
// ST9 <ESC> ( B H I J T
//
// (**) "<ESC> $ ID" is a synonym of "<ESC> $ ( ID" for ID=(@, A, B)
//
// Because of the large number of tokens, this table is
// inverted (tokens x states).
//
static signed char _rgchNextState[nTokens][nStates] ={//
// S S S S S S S S S S
// T T T T T T T T T T
// 0 1 2 3 4 5 6 7 8 9
//--------------------------------------------------------------------
//
/* txt */ TXT, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* ext */ EXT, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* esc */ ST1, UNK, UNK, UNK, UNK, UNK, ST7, UNK, UNK, UNK,/* si */ LSI, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* so */ LSO, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* $ */ TXT, ST2, UNK, UNK, UNK, UNK, UNK, ST8, UNK, UNK,/* @ */ TXT, UNK, JS2, JS2, UNK, ST6, UNK, UNK, UNK, UNK,/* & */ TXT, ST5, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* ( */ TXT, ST9, ST3, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* ) */ TXT, UNK, ST4, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* A */ TXT, UNK, CS1, CS1, UNK, UNK, UNK, UNK, UNK, UNK,/* B */ TXT, UNK, JS3, JS3, UNK, UNK, UNK, UNK, JS4, ASC,/* C */ TXT, UNK, UNK, KS0, KSD, UNK, UNK, UNK, UNK, UNK,/* D */ TXT, UNK, UNK, JS5, UNK, UNK, UNK, UNK, UNK, UNK,/* E */ TXT, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* G */ TXT, UNK, UNK, TS0, UNK, UNK, UNK, UNK, UNK, UNK,/* H */ TXT, UNK, UNK, TS1, UNK, UNK, UNK, UNK, UNK, JS0,/* I */ TXT, UNK, UNK, TS2, UNK, UNK, UNK, UNK, UNK, JS1,/* J */ TXT, UNK, UNK, TS3, UNK, UNK, UNK, UNK, UNK, JS0,/* K */ TXT, UNK, UNK, TS4, UNK, UNK, UNK, UNK, UNK, UNK,/* L */ TXT, UNK, UNK, TS5, UNK, UNK, UNK, UNK, UNK, UNK,/* M */ TXT, UNK, UNK, TS6, UNK, UNK, UNK, UNK, UNK, UNK,/* T */ TXT, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, CS0,/* unk */ UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK, UNK,/* eof */ FIN, EOI, EOI, EOI, EOI, EOI, EOI, EOI, EOI, EOI,/* err */ ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR,};
// Also for ISO-2022 out. Build arrays of possible character
// sets for each type of input character set. Character sets
// should appear in order of hit probability (e.g., in 2022-Jp
// JS3 is the most common set). Mark the end of array with -1.
// (Only store these for non-ascii sets).
//
//
// China (icetIso2022Cn)
static int _rgceCn[] = { -1, };
// Japan (icetIso2022Jp)
static int _rgceJp[] = { csJS3, csJS1, csJS5, -1, };
// Korea (icetIso2022Kr)
static int _rgceKr[] = { -1, };
// Taiwan (icetIso2022Tw)
static int _rgceTw[] = { -1, };
static int *_mpicetrgce[icetCount] = { 0, // icetEucCn
0, // icetEucJp
0, // icetEucKr
0, // icetEucTw
_rgceCn, // icetIso2022Cn
_rgceJp, // icetIso2022Jp
_rgceKr, // icetIso2022Kr
_rgceTw, // icetIso2022Tw
0, // icetBig5
0, // icetGbk
0, // icetShiftJis
0, // icetWansung
0, // icetUtf8
};
/* _ J T K G E T N E X T *//*----------------------------------------------------------------------------
%%Function: _JtkGetNext %%Contact: jpick
Get the next character and classify it. Return the token.----------------------------------------------------------------------------*/static JTK __inline _JtkGetNext(IStream *pstmIn, PUCHAR puch){ ULONG rc; HRESULT hr; hr = pstmIn->Read(puch, 1, &rc); if (hr != S_OK ) return err; else if (rc == 0) return eof; else return _rgjtkCharClass[*puch];}
/* C C E R E A D E S C S E Q *//*----------------------------------------------------------------------------
%%Function: CceReadEscSeq %%Contact: jpick
Read pointer is positioned at an escape sequence, figure out which escape sequence it is.----------------------------------------------------------------------------*/CCE CceReadEscSeq(IStream *pstmIn, ICET *lpicet){ UCHAR uch; TST tstCurr; JTK jtk; CCE cceRet;#ifdef DEBUG
TST tstPrev;#endif
// Sanity checks ...
//
#ifdef DEBUG
if (!pstmIn || !lpicet) return cceInvalidParameter;#endif
tstCurr = ST0;
while (1) { // Find the next stopping state.
//
do { // Get the next character and clasify it.
//
jtk = _JtkGetNext(pstmIn, &uch); #ifdef DEBUG
// Save the previous state for debugging purposes, only.
//
tstPrev = tstCurr;#endif
// Transition -- note that order is different than
// "normal" transition tables.
//
tstCurr = _rgchNextState[jtk][tstCurr]; } while (!IsFinal(tstCurr)); switch (tstCurr) { case JS0: // JIS-Roman
case JS1: // Half-Width Katakana
case JS2: // JIS C 6226-1978
case JS3: // JIS X 0208-1983
case JS4: // JIS X 0208-1990
case JS5: // JIS X 0212-1990
*lpicet = icetIso2022Jp; cceRet = cceSuccess; goto _LRet; case CS0: // GB 1988-89 Roman
case CS1: // GB 2312-80
*lpicet = icetIso2022Cn; cceRet = cceSuccess; goto _LRet; case TS0: // CNS 11643-1992 Plane 1
case TS1: // CNS 11643-1992 Plane 2
case TS2: // CNS 11643-1992 Plane 3
case TS3: // CNS 11643-1992 Plane 4
case TS4: // CNS 11643-1992 Plane 5
case TS5: // CNS 11643-1992 Plane 6
case TS6: // CNS 11643-1992 Plane 7
*lpicet = icetIso2022Tw; cceRet = cceSuccess; goto _LRet; case KS0: // KS C 5601-1992
case KSD: // ISO-2022-KR Document Signal
*lpicet = icetIso2022Kr; cceRet = cceSuccess; goto _LRet; case ASC: // Ascii
case LSO: case LSI: case TXT: case EXT: case FIN: // Insufficient information to choose a flavor ...
cceRet = cceMayBeAscii; goto _LRet; case ERR: cceRet = cceRead; goto _LRet; default: // UNK, EOI
cceRet = cceUnknownInput; goto _LRet; } } _LRet:
return cceRet;}
|
