windows-server-2003/enduser/speech/common/spcommon/ltscart.cpp

/*******************************************************************************
* LtsCart.cpp *
*----------*
*
*   ** WARNING **
*   CART code for LTS. This code was created in MS Research and LiJ owns
*   the algorithm. YunusM eliminated the private heap used by this code
*   and used the new and delete operators instead.
*
*   Created By: LIJ (MS Research)                           Date: 06/18/99
*   Current Owner: Fil
*
*   Copyright (C) 1999 Microsoft Corporation. All Rights Reserved
*******************************************************************************/

//--- Includes --------------------------------------------------------------

#include "StdAfx.h"
#include "LtsCart.h"

#pragma warning(disable : 4100)

/* the following are for exceptions: single letter and NULL output */
static const char *bogus_pron_1033 = "B OW G AH S P R AH N AH N S IY EY SH AH N";
static const char *bogus_pron_1041 = "N A N I"; // what?

static const char *single_letter_pron_1033[52] =
{
    "EY",
    "B IY",
    "S IY",
    "D IY",
    "IY",
    "EH F",
    "JH IY",
    "EY CH",
    "AY",
    "JH EY",
    "K EY",
    "EH L",
    "EH M",
    "EH N",
    "OW",
    "P IY",
    "K Y UW",
    "AA R",
    "EH S",
    "T IY",
    "Y UW",
    "V IY",
    "D AH B AX L Y UW",
    "EH K S",
    "W AY",
    "Z IY",
    //
    // PLURAL SPELLINGS
    //
    "EY Z",
    "B IY Z",
    "S IY Z",
    "D IY Z",
    "IY Z",
    "EH F S",
    "JH IY Z",
    "EY CH AX Z",
    "AY Z",
    "JH EY Z",
    "K EY Z",
    "EH L Z",
    "EH M Z",
    "EH N Z",
    "OW Z",
    "P IY Z",
    "K Y UW Z",
    "AA R Z",
    "EH S AX Z",
    "T IY Z",
    "Y UW Z",
    "V IY Z",
    "D AH B AX L Y UW Z",
    "EH K S AX Z",
    "W AY Z",
    "Z IY Z"
};
static const char *single_letter_pron_1041[52] =
{
    "EE",
    "B II",
    "SH II",
    "D II",
    "II",
    "E H U",
    "J II",
    "EE CH I",
    "A I",
    "J EE",
    "K EE",
    "E R U",
    "E M U",
    "E N U",
    "OO",
    "P II",
    "K Y UU",
    "AA R U",
    "E S U",
    "T II",
    "Y UU",
    "B U I",
    "D A B U R Y UU",
    "E STOP K U S U",
    "W A I",
    "Z E STOP T O",
    //
    // PLURAL SPELLINGS
    //
    "EE Z U",
    "B II Z U",
    "SH II Z U",
    "D II Z U",
    "II Z U",
    "E H U Z U",
    "J II Z U",
    "EE CH I Z U",
    "A I Z U",
    "J EE Z U",
    "K EE Z U",
    "E R U Z U",
    "E M U Z U",
    "E N U Z U",
    "OO Z U",
    "P II Z U",
    "K Y UU Z U",
    "AA R U Z U",
    "E S U Z U",
    "T II Z U",
    "Y UU Z U",
    "B U I Z U",
    "D A B U R Y UU Z U",
    "E STOP K U S U Z U",
    "W A I Z U",
    "Z E STOP T O Z U"
};


/*
* not worthwhile to use binary search with only about 30 entries
*/
static int symbol_to_id(LTS_SYMTAB *tab, char *sym)
{
    USES_CONVERSION;
    SPDBG_FUNC("symbol_to_id");

    int i;
    for (i = 0; i < tab->n_symbols; i++)
    {
        if (CSTR_EQUAL == CompareString(MAKELCID(MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), SORT_DEFAULT), NORM_IGNORECASE,
            A2T(tab->storage + tab->sym_idx[i]), -1, A2T(sym), -1))
        {
            return i;
        }
    }
    return NO_SYMBOL;
} // static int symbol_to_id(LTS_SYMTAB *tab, char *sym)


static char *id_to_symbol(LTS_SYMTAB *tab, int id)
{
    SPDBG_FUNC("id_to_symbol");

    if (id < 0 || id > tab->n_symbols)
    {
        return NULL;
    }
    else
    {
        return tab->storage + tab->sym_idx[id];
    }
} // static char *id_to_symbol(LTS_SYMTAB *tab, int id)


__inline void ODS (const char *format, ...)
{
#ifdef _DEBUG
    SPDBG_FUNC("ODS");

    va_list arglist;
    va_start (arglist, format);

    char buf[2048];
    _vsnprintf(buf, 2048, format, arglist);
    OutputDebugStringA(buf);

    va_end (arglist);
#endif
}

__inline int ans_simp_question (LTS_FEATURE *feat, SIMPLE_QUESTION question,
                                LTS_SAMPLE *sample)
{
    SPDBG_FUNC("ans_simp_question");

    SYMBOL id;
    int *phones = feat[question.questype].feature[question.feature];

    SAMPLE_GET_CONTEXT(sample, question.questype, question.context,
        question.offset, id);

    return (TST_BIT(phones, id) ? TRUE : FALSE);
} // __inline int ans_simp_question (LTS_FEATURE *feat, SIMPLE_QUESTION question,


static int product_eval (LTS_FEATURE *feat, char *term, LTS_SAMPLE *sample)
{
    SPDBG_FUNC("product_eval");

    int negate, result;
    SIMPLE_QUESTION ques;
    char *cptr;

    cptr = term;
    while (TRUE)
    {
        /* negation sign */
        if (*cptr == '~')
        {
            negate = TRUE;
            cptr++;
        }
        else
        {
            negate = FALSE;
        }

        if (!isdigit(*cptr))
        {
            //quit (-1, "Invalid product in product_eval\n");
            //      OutputDebugString("Invalid product in product_eval\n");
            return FALSE;
        }

        for (result = *cptr++ - '0'; isdigit (*cptr); cptr++)
        {
            result = result * 10 + (*cptr - '0');
        }

        QUES_DECODE(result, ques.questype, ques.context, ques.offset,
            ques.feature);
        if ((negate ^ ans_simp_question (feat, ques, sample)) == FALSE)
        {
            return FALSE;
        }

        if (*cptr == '\0')
        {
            break;
        }
        if (*cptr++ != '&')
        {
            //quit (-1, "product_eval:  syntax error in product term %s\n", term);
            /*
            char szTemp[512];

              sprintf(szTemp, "product_eval:  syntax error in product term %s\n", term);
              OutputDebugString(szTemp);
            */
            return FALSE;
        }
    }

    return TRUE;
} // static int product_eval (LTS_FEATURE *feat, char *term, LTS_SAMPLE *sample)


static int ans_comp_question(LTS_FEATURE *feat, char *prod,
                             LTS_SAMPLE *sample)
{
    SPDBG_FUNC("ans_comp_question");

    int i, num_products, limit;
    char *cptr, string[LONGEST_STR], *products[MAX_PRODUCTS];

    strcpy(string, prod);
    for (cptr = string, num_products = 1; *cptr != '\0'; cptr++)
    {
        if (*cptr == '|') num_products++;
    }

    if (num_products > MAX_PRODUCTS)
    {
        //quit(1, "please increase MAX_PRODUCTS up to %d at least\n", num_products);

        /*
        char szTemp[256];
        sprintf(szTemp, "please increase MAX_PRODUCTS up to %d at least\n", num_products);
        OutputDebugString(szTemp);
        */

        return FALSE;
    }

    for (i = 0, limit = num_products -1, cptr = string; ; i++)
    {
        products[i] = cptr++;
        if (i == limit)
        {
            break;
        }

        for (; *cptr != '|'; cptr++) {};
        *cptr++ = '\0';
    }

    for (i = 0; i < num_products; i++)
    {
        if (product_eval (feat, products[i], sample) == TRUE)
        {
            return TRUE;
        }
    }

    return FALSE;
} // static int ans_comp_question(LTS_FEATURE *feat, char *prod,


static T_NODE *find_leaf(LTS_FEATURE *feat, T_NODE *root, LTS_SAMPLE *sample)
{
    SPDBG_FUNC("find_leaf");

    if (!root->yes_child)
    {
        return root;
    }
    else if (ans_comp_question(feat, root->prod, sample))
    {
        return find_leaf(feat, root->yes_child, sample);
    }
    else
    {
        return find_leaf(feat, root->no_child, sample);
    }
} // static T_NODE *find_leaf(LTS_FEATURE *feat, T_NODE *root, LTS_SAMPLE *sample)


static int lts_product_eval (LTS_FEATURE *feat, LTS_PROD *term,
                             LTS_SAMPLE *sample, LTS_PROD **next)
{
    SPDBG_FUNC("lts_product_eval");

    int negate, result;
    SIMPLE_QUESTION ques;
    LTS_PROD *cptr = term;

    while (TRUE)
    {
        if ((*cptr) & PROD_NEG)
        {
            negate = TRUE;
            result = (*cptr) ^ PROD_NEG;
        }
        else
        {
            negate = FALSE;
            result = (*cptr);
        }

        QUES_DECODE(result, ques.questype, ques.context, ques.offset,
            ques.feature);
        if ((negate ^ ans_simp_question (feat, ques, sample)) == FALSE)
        {
            while (*cptr != PROD_TERM && *cptr != QUES_TERM)
            {
                cptr++;
            }
            if (*cptr == QUES_TERM)
            {
                *next = NULL;
            }
            else
            {
                *next = cptr + 1;
            }
            return FALSE;
        }

        cptr++;
        if (*cptr == QUES_TERM)
        {
            *next = NULL;
            break;
        }
        else if (*cptr == PROD_TERM)
        {
            *next = cptr + 1;
            break;
        }
    }

    return TRUE;
} // static int lts_product_eval (LTS_FEATURE *feat, LTS_PROD *term,


static int lts_ans_comp_question(LTS_TREE UNALIGNED *tree, LTS_FEATURE *feat,
                                 int idx, LTS_SAMPLE *sample)
{
    SPDBG_FUNC("lts_ans_comp_question");

    LTS_PROD *next, *term = (LTS_PROD *) ((char *) tree->p_prod + idx);

    while (TRUE)
    {
        if (lts_product_eval (feat, term, sample, &next) == TRUE)
        {
            return TRUE;
        }
        if (next == NULL)
        {
            break;
        }
        term = next;
    }

    return FALSE;
} // static int lts_ans_comp_question(LTS_TREE *tree, LTS_FEATURE *feat,


static LTS_NODE *lts_find_leaf(LTS_TREE UNALIGNED *tree, LTS_FEATURE *feat,
                               LTS_NODE *root, LTS_SAMPLE *sample)
{
    SPDBG_FUNC("lts_find_leaf");

    if (IS_LEAF_NODE(root))
    {
        return root;
    }
    else if (lts_ans_comp_question(tree, feat, ((LTS_NODE UNALIGNED *)root)->idx, sample))
    {
        return lts_find_leaf(tree, feat, root + ((LTS_NODE UNALIGNED *)root)->yes, sample);
    }
    else
    {
        return lts_find_leaf(tree, feat, root + ((LTS_NODE UNALIGNED *)root)->yes + 1, sample);
    }
} // static LTS_NODE *lts_find_leaf(LTS_TREE *tree, LTS_FEATURE *feat,


static LTS_DIST *lts_find_leaf_count(LTS_FOREST *l_forest, SYMBOL *pIn,
                                     SYMBOL *pOut)
{
    SPDBG_FUNC("lts_find_leaf_count");

    LTS_TREE UNALIGNED *tree = l_forest->tree[*pIn];
    LTS_NODE UNALIGNED *leaf;
    LTS_SAMPLE sample;

    /*
    * construct a sample in order to share all the code with training
    */
    sample.pIn = pIn;
    sample.pOut = pOut;

    /* *pOut cannot be NULL_SYMBOL_ID */
    *pOut = NULL_SYMBOL_ID + 1;

    leaf = lts_find_leaf(tree, l_forest->features, &(tree->nodes[0]), &sample);
    return (LTS_DIST *) ((char *)tree->p_dist + leaf->idx);
} // static LTS_DIST *lts_find_leaf_count(LTS_FOREST *l_forest, SYMBOL *pIn,

static LTS_OUT_RESULT *allocate_out_result(LTS_FOREST *l_forest)
{
    SPDBG_FUNC("allocate_out_result");

    LTS_OUT_RESULT *res = new LTS_OUT_RESULT;
    if (res)
    {
        res->out_strings = new LTS_OUT_STRING *[MAX_ALT_STRINGS];
        if (res->out_strings)
        {
            res->num_allocated_strings = MAX_ALT_STRINGS;
            res->num_strings = 0;
        }
        else
        {
            delete res;
            res = NULL;
        }
    }

    return res;
} // static LTS_OUT_RESULT *allocate_out_result(LTS_FOREST *l_forest)

static void free_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res)
{
    SPDBG_FUNC("free_out_result");

    int i;

    for (i = 0; i < res->num_strings; i++)
    {
        delete res->out_strings[i];
    }
    if (res->num_allocated_strings == MAX_ALT_STRINGS)
    {
        delete res->out_strings;
    }
    else
    {
        free(res->out_strings);  /* dirty */
    }

    delete res;
} // static void free_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res)


static bool reallocate_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
                                  int min)
{
    SPDBG_FUNC("reallocate_out_result");

    int s = res->num_allocated_strings, old_size = s;
    LTS_OUT_STRING **p;

    while (s < min)
        s += INC_ALT_STRINGS;
    p = res->out_strings;

    res->out_strings = (LTS_OUT_STRING **)
        calloc(s, sizeof(LTS_OUT_STRING *));
    if (!res->out_strings)
    {
        return false;
    }

    memcpy(res->out_strings, p, old_size * sizeof(LTS_OUT_STRING *));

    if (old_size == MAX_ALT_STRINGS)
    {
        delete p;
    }
    else
    {
        free(p);
    }

    res->num_allocated_strings = s;
    ODS("increased out_strings to %d in order to meet %d\n", s, min);

    return true;
} // static void reallocate_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,


static bool grow_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,
                            SYMBOL i, int count, float inv_sum,
                            LTS_OUT_RESULT *tmpRes)
{
    SPDBG_FUNC("grow_out_result");

    int j;

    if (res->num_strings + tmpRes->num_strings >= res->num_allocated_strings)
    {
        if (!reallocate_out_result(l_forest, res,
            res->num_strings + tmpRes->num_strings))
        {
            return false;
        }
    }
    for (j = 0; j < tmpRes->num_strings; j++)
    {
        SYMBOL *psrc = tmpRes->out_strings[j]->psym;
        SYMBOL *ptgt;
        res->out_strings[res->num_strings + j] = new LTS_OUT_STRING;
        if (!res->out_strings)
        {
            return false;
        }

        ptgt = res->out_strings[res->num_strings + j]->psym;
        *ptgt++ = i;
        while (*psrc != NULL_SYMBOL_ID)
        {
            *ptgt++ = *psrc++;
        }
        *ptgt++ = NULL_SYMBOL_ID;
        res->out_strings[res->num_strings + j]->prob = count * inv_sum *
            tmpRes->out_strings[j]->prob;
    }
    res->num_strings += tmpRes->num_strings;
    free_out_result(l_forest, tmpRes);

    return true;
} // static void grow_out_result(LTS_FOREST *l_forest, LTS_OUT_RESULT *res,


static LTS_OUT_RESULT *gen_one_output(LTS_FOREST *l_forest, int len,
                                      SYMBOL *input_id, int in_index,
                                      SYMBOL *output_id, float cutoff)
{
    SPDBG_FUNC("gen_one_output");

    SYMBOL out[SP_MAX_WORD_LENGTH], *pOut;
    LTS_OUT_RESULT *res = allocate_out_result(l_forest);
    if (!res)
    {
        return NULL;
    }

    int sum, i, dim;
    LTS_DIST UNALIGNED *pdf;
    LTS_PAIR UNALIGNED *l_pair, *lp;
    float cut, inv_sum;

    /*
    * copy output_id to local
    */
    SYMBOL *psrc = output_id - 1, *ptgt = out;
    while (*psrc != NULL_SYMBOL_ID) psrc--;
    while (psrc != output_id)
        *ptgt++ = *psrc++;
    pOut = ptgt;
    /* sanity check */
    if (pOut - out != in_index + 1)
    {
        // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        int *z=0;
        z[0]=z[1];
    }

    if (in_index == len - 1)
    {
        pdf = lts_find_leaf_count(l_forest, input_id + in_index, pOut);
        l_pair = &(pdf->p_pair);
        dim = pdf->c_dists;
        for (lp = l_pair, sum = 0, i = 0; i < dim; i++, lp++)
        {
            sum += lp->cnt;
        }
        SPDBG_ASSERT(sum > 0);
        inv_sum = 1.0f / sum;
        cut = cutoff * sum;
        for (lp = l_pair, i = 0; i < dim; i++, lp++)
        {
            if ((float)(lp->cnt) > cut)
            {
                res->out_strings[res->num_strings] = new LTS_OUT_STRING;
                if (NULL == res->out_strings[res->num_strings])
                {
                    return NULL;
                }
                res->out_strings[res->num_strings]->psym[0] = (SYMBOL) lp->id;
                res->out_strings[res->num_strings]->psym[1] = NULL_SYMBOL_ID;
                res->out_strings[res->num_strings]->prob = lp->cnt * inv_sum;
                res->num_strings++;
            } /* cut */
        }
    }
    else
    {
        LTS_OUT_RESULT *tmpRes;

        pdf = lts_find_leaf_count(l_forest, input_id + in_index, pOut);
        dim = pdf->c_dists;
        l_pair = &(pdf->p_pair);
        for (lp = l_pair, sum = 0, i = 0; i < dim; i++, lp++)
        {
            sum += lp->cnt;
        }
        SPDBG_ASSERT(sum > 0);

        inv_sum = 1.0f / sum;
        cut = cutoff * sum;
        for (lp = l_pair, i = 0; i < dim; i++, lp++)
        {
            if ((float)(lp->cnt) > cut)
            {
                SYMBOL *pTmpOut = pOut + 1;
                *pOut = (SYMBOL) lp->id;
                tmpRes = gen_one_output(l_forest, len, input_id, in_index + 1, pTmpOut, cutoff);
                if (!tmpRes)
                {
                    return NULL;
                }

                if (!grow_out_result(l_forest, res, (SYMBOL)(lp->id), lp->cnt,
                    inv_sum, tmpRes))
                {
                    return NULL;
                }
            }
        } /* i */
    } /* else */

    return res;
} // static LTS_OUT_RESULT *gen_one_output(LTS_FOREST *l_forest, int len,


static int comp_out_result_prob(const void *vp1, const void *vp2)
{
    SPDBG_FUNC("comp_out_result_prob");

    LTS_OUT_STRING **p1 = (LTS_OUT_STRING **) vp1,
        **p2 = (LTS_OUT_STRING **) vp2;

    if ((*p1)->prob > (*p2)->prob)
    {
        return -1;
    }
    else if ((*p1)->prob < (*p2)->prob)
    {
        return 1;
    }
    else
    {
        return 0;
    }
} // static int comp_out_result_prob(const void *vp1, const void *vp2)


static void lts_fill_out_buffer(LTS_FOREST *l_forest, LTS_OUT_RESULT *out,
                                char *word)
{
    SPDBG_FUNC("lts_fill_out_buffer");

    int i, j, n;
    float inv_sum, sum = 0.0f;
    char phnstr[LONGEST_STR];
    char *tmp;
    LTS_SYMTAB *tab = l_forest->symbols;

    if (out == NULL)
    {
        return;
    }

    if (word)
    {
        strcpy(l_forest->out.word, word);
    }
    else
    {
        l_forest->out.word[0] = 0;
    }

    /* normalize probabilities */
    for (i = 0; i < out->num_strings; i++)
    {
        sum += out->out_strings[i]->prob;
    }
    inv_sum = 1.0f / sum;
    for (i = 0; i < out->num_strings; i++)
    {
        out->out_strings[i]->prob *= inv_sum;
    }

    /*
    * sort them according to the prob field
    */
    qsort(out->out_strings, out->num_strings, sizeof(LTS_OUT_STRING *),
        &comp_out_result_prob);

    if (out->num_strings > MAX_OUTPUT_STRINGS - l_forest->out.num_prons)
    {
        n = MAX_OUTPUT_STRINGS - l_forest->out.num_prons;
        for (sum = 0.0f, i = 0; i < n; i++)
        {
            sum += out->out_strings[i]->prob;
        }
        inv_sum = 1.0f / sum;
        for (i = 0; i < n; i++)
        {
            out->out_strings[i]->prob *= inv_sum;
        }
    }
    else
    {
        n = out->num_strings;
    }

    for (j = l_forest->out.num_prons, i = 0; i < n; i++)
    {
        SYMBOL *p = out->out_strings[i]->psym;
        char *psrc, *ptgt;

        if (out->out_strings[i]->prob < MIN_OUT_PROB)
        {
            continue;
        }

        phnstr[0] = 0;
        l_forest->out.pron[j].prob = out->out_strings[i]->prob;

        while (*p != NULL_SYMBOL_ID)
        {
            tmp = id_to_symbol(&(tab[OUTPUT]), *p++);
            SPDBG_ASSERT(tmp);
            if (tmp)
            {
                strcat(phnstr, tmp);
                strcat(phnstr, " ");
            }
        }

        psrc = phnstr;
        ptgt = l_forest->out.pron[j].pstr;
        while (*psrc)
        {
            if (*psrc != '#' && *psrc != '_')
            {
                *ptgt++ = *psrc++;
            }
            else if (*psrc == '_')
            {
                *ptgt++ = ' ';
                psrc++;
            }
            else
            {
                psrc += 2; /* skip an extra space */
            }
            /* extreme case, truncate it */
            if (ptgt - l_forest->out.pron[j].pstr >= SP_MAX_PRON_LENGTH)
            {
                for (ptgt--; !isspace(*ptgt); ptgt--) {}; /* never output partial phone */
                ptgt++;
                break;
            }
        }
        // output could contain only '# '
        if (ptgt > l_forest->out.pron[j].pstr && *(ptgt - 1) == ' ')
        {
            *(ptgt - 1) = 0; /* remove the last space */
        }
        else
        {
            *ptgt = 0; /* shouldn't happen unless ptgt didn't move */
        }
        if (ptgt > l_forest->out.pron[j].pstr)
        {
            j++;
        }
    } /* i */

    if (j <= MAX_OUTPUT_STRINGS)
    {
        l_forest->out.num_prons = j;
    }
    else
    {
        l_forest->out.num_prons = MAX_OUTPUT_STRINGS; // should never happen
    }

    free_out_result(l_forest, out);
} // static void lts_fill_out_buffer(LTS_FOREST *l_forest, LTS_OUT_RESULT *out,


void assign_a_fixed_pron(LTS_OUTPUT *out, const char *pron, char *word)
{
    SPDBG_FUNC("assign_a_fixed_pron");

    out->num_prons = 1;
    strcpy(out->word, word);
    out->pron[0].prob = 1.0f;
    if (strlen(pron) < SP_MAX_PRON_LENGTH)
    {
        strcpy(out->pron[0].pstr, pron);
    }
    else
    {
        char *p;
        strncpy(out->pron[0].pstr, pron, SP_MAX_PRON_LENGTH);
        p = &(out->pron[0].pstr[SP_MAX_PRON_LENGTH - 1]);
        while (!isspace(*p))
        {
            p--; /* truncate the last partial phoneme */
        }
        *p = 0;
    }
} // void assign_a_fixed_pron(LTS_OUTPUT *out, char *pron, char *word)

inline BOOL IsCharInRangeA(int ch, int chMin, int chMax)
{
    return (unsigned)(ch - chMin) <= (unsigned)(chMax - chMin);
}

void assign_a_spelling_pron(LTS_OUTPUT *out, const char * single_letter_pron[52], char *word)
{
    SPDBG_FUNC("assign_a_spelling_pron");

    char *p;
    int cchPron = 0;

    strcpy(out->word, word);
    if (ispunct(*word))
    {
        p = word + 1;
    }
    else
    {
        p = word;
    }

    out->num_prons = 1;
    out->pron[0].prob = 1.0f;
    out->pron[0].pstr[0] = 0;

    char * pchPron = out->pron[0].pstr;

    while (*p)
    {
        int cPOffset = 0;   // 0 for single letter, 26 for plurals
        int c = *p++;

        // Lowercaseify, and skip over non-letters
        if (IsCharInRangeA(c, 'A', 'Z'))
        {
            c += 'a' - 'A';
        }
        else if (!IsCharInRangeA(c, 'a', 'z'))
        {
            continue;
        }

        // Check if the next two characters are 'S (apostrophe S).  Include the following cases: words ending in 's 'S s' S'
        // If they are we use a the plural pronunciation for the letter and skip over the letter and 'S
        if ((p[0] == '\'') && ((0 == p[1] && 's' == c) || 's' == p[1] || 'S' == p[1]))
        {
            cPOffset = 26;
            p += p[1] ? 1 : 0;         // skip 'S
        }

        //  Make sure the string isn't too long accounting for the new phone and seperator
        const char * const pchPronT = single_letter_pron[cPOffset + c - 'a'];
        const int cchPronT = strlen(pchPronT);

        if ((cchPron + 1 + cchPronT) < (SP_MAX_PRON_LENGTH - 1)) // +1 for separating space, -1 for terminating NUL
        {
            strcpy(pchPron + cchPron, pchPronT);

            cchPron += cchPronT;

            pchPron[cchPron++] = ' ';
        }
        else
        {
            break;
        }
    }

    if (cchPron)
    {
        pchPron[cchPron - 1] = 0; // trim trailing space char
    }
}


HRESULT LtscartGetPron(LTS_FOREST *l_forest, char *word, LTS_OUTPUT **ppLtsOutput)
{
    SPDBG_FUNC("LtscartGetPron");

    HRESULT hr = S_OK;
    LTS_OUT_RESULT *pres = NULL;
    char *p, *base;
    SYMBOL buffer[LONGEST_STR], *pbuf = buffer + 1;
    int len, id, hasvowel = 0, allcapital = 1;

    l_forest->out.num_prons = 0;
    buffer[0] = NULL_SYMBOL_ID;
    len = 0;


    if (word == NULL || (base = strtok(word, " \t\n")) == NULL)
    {
        assign_a_fixed_pron(&(l_forest->out), l_forest->bogus_pron, "NUL");
        *ppLtsOutput = &(l_forest->out);
        return S_FALSE;
    }
    else
    {
        base = strtok(word, " \t\n");
        if (ispunct(*base))
        {
            for (p = base; *p && ispunct(*p); p++) {};
        }
        else
        {
            p = base;
        }
    }

    char ach[2];
    ach[1] = 0;

    while (*p)
    {
        const int d = *p++;
        const int c = tolower(d);

        if (!hasvowel && (c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u' || c == 'y'))
        {
            hasvowel = 1;
        }
        if (allcapital && d == c)
        {
            allcapital = 0;
        }

        ach[0] = (char)c;

        if ((id = symbol_to_id (&(l_forest->symbols[INPUT]), ach)) == NO_SYMBOL || id == NULL_SYMBOL_ID)
        {
            ODS("cannot find the symbol %c, skip!\n", c);
            continue;
        }

        pbuf[len++] = (SYMBOL) id;
    }

    pbuf[len] = NULL_SYMBOL_ID;
    if (len >= SP_MAX_WORD_LENGTH || len <= 0)
    {
        // fill in bogus pron below
    }
    else if (len == 1)
    {
        LTS_SYMTAB *tab = l_forest->symbols;
        char *p = id_to_symbol(&(tab[INPUT]), pbuf[0]);
        int c = tolower(p[0]);
        if (c >= 'a' && c <= 'z')
        {
            assign_a_fixed_pron(&(l_forest->out), l_forest->single_letter_pron[c - 'a'], word);
        }
    }
    else if (!hasvowel)
    {
        assign_a_spelling_pron(&(l_forest->out), l_forest->single_letter_pron, word);
    }
    else
    {
        if (allcapital)
        {
            assign_a_spelling_pron(&(l_forest->out), l_forest->single_letter_pron, word);
        }
        pres = gen_one_output(l_forest, len, pbuf, 0, pbuf, DEFAULT_PRUNE);
        if (!pres)
        {
            return E_OUTOFMEMORY;
        }

        lts_fill_out_buffer(l_forest, pres, word);
    }

    if (l_forest->out.num_prons == 0)
    {
        hr = S_FALSE;

        assign_a_fixed_pron(&(l_forest->out), l_forest->bogus_pron, word);
    }

    *ppLtsOutput = &(l_forest->out);

    SPDBG_RETURN(hr);
} /* LtscartGetPron */


LTS_FOREST *LtscartReadData (LCID lcid, PBYTE map_addr)
{
    SPDBG_FUNC("LtscartReadData");

    int i;
    LTS_FOREST *l_forest;
    LTS_SYMTAB *tab;
    LTS_FEATURE *feat;
    int output = 0;

    l_forest = (LTS_FOREST *) calloc(1, sizeof(LTS_FOREST));
    if (!l_forest)
    {
        return NULL;
    }

    if (lcid == 1033)
    {
        l_forest->bogus_pron = bogus_pron_1033;
        l_forest->single_letter_pron = single_letter_pron_1033;
    }
    else if (lcid == 1041)
    {
        l_forest->bogus_pron = bogus_pron_1041;
        l_forest->single_letter_pron = single_letter_pron_1041;
    }
    else
    {
        return NULL;
    }

    //read in the symbol table
    l_forest->symbols = (LTS_SYMTAB *) calloc(2, sizeof(LTS_SYMTAB));
    if (!l_forest->symbols)
    {
        return NULL;
    }

    tab = &(l_forest->symbols[INPUT]);
    CopyMemory(&(tab->n_symbols), map_addr + output, sizeof(int));
    output += sizeof(int);

    tab->sym_idx = (int *)(map_addr + output);
    output += tab->n_symbols * sizeof(int);

    CopyMemory(&(tab->n_bytes), map_addr + output, sizeof(int));
    output += sizeof(int);

    tab->storage = (char*)(map_addr + output);
    output += tab->n_bytes * sizeof(char);

    tab = &(l_forest->symbols[OUTPUT]);
    CopyMemory(&(tab->n_symbols), map_addr + output, sizeof(int));
    output += sizeof(int);

    tab->sym_idx = (int*)(map_addr + output);
    output += tab->n_symbols * sizeof(int);
    CopyMemory(&(tab->n_bytes), map_addr + output, sizeof(int));
    output += sizeof(int);

    tab->storage = (char*)(map_addr + output);
    output += tab->n_bytes * sizeof(char);

    // read in the feature vector
    l_forest->features = (LTS_FEATURE *) calloc(2, sizeof(LTS_FEATURE));
    if (!l_forest->features)
    {
        return NULL;
    }

    feat = &(l_forest->features[INPUT]);

    CopyMemory(&(feat->n_feat), map_addr + output, sizeof(int));
    output += sizeof(int);

    CopyMemory(&(feat->dim), map_addr + output, sizeof(int));
    output += sizeof(int);

    feat->feature = (int **) calloc(feat->n_feat, sizeof(int *));
    if (!feat->feature)
    {
        return NULL;
    }

    for (i = 0; i < feat->n_feat; i++)
    {
        feat->feature[i] = (int*)(map_addr + output);
        output += feat->dim * sizeof(int);
    }

    feat = &(l_forest->features[OUTPUT]);
    CopyMemory(&(feat->n_feat), map_addr + output, sizeof(int));
    output += sizeof(int);

    CopyMemory(&(feat->dim), map_addr + output, sizeof(int));
    output += sizeof(int);

    feat->feature = (int **) calloc(feat->n_feat, sizeof(int *));
    if (!feat->feature)
    {
        return NULL;
    }

    for (i = 0; i < feat->n_feat; i++)
    {
        feat->feature[i] = (int*)(map_addr + output);
        output += feat->dim * sizeof(int);
    }

    /*
    * read in the tree
    */
    l_forest->tree = (LTS_TREE **) calloc(l_forest->symbols[INPUT].n_symbols,
					   sizeof(LTS_TREE *));
    if (!l_forest->tree)
    {
        return NULL;
    }

    for (i = 1; i < l_forest->symbols[INPUT].n_symbols; i++)
    {
        LTS_TREE *l_root;
        l_forest->tree[i] = l_root = (LTS_TREE *) calloc(1, sizeof(LTS_TREE));
        if (!l_root)
        {
            return NULL;
        }

        CopyMemory(&(l_root->n_nodes), map_addr + output, sizeof(int));
        output += sizeof(int);

        l_root->nodes = (LTS_NODE*)(map_addr + output);
        output += l_root->n_nodes * sizeof(LTS_NODE);

        CopyMemory(&(l_root->size_dist), map_addr + output, sizeof(int));
        output += sizeof(int);

        l_root->p_dist = (LTS_DIST*)(map_addr + output);
        output += l_root->size_dist * sizeof(char);

        CopyMemory(&(l_root->size_prod), map_addr + output, sizeof(int));
        output += sizeof(int);

        if (l_root->size_prod > 0)
        {
            l_root->p_prod = (LTS_PROD*)(map_addr + output);
            output += l_root->size_prod * sizeof(char);
        }
    }

    return l_forest;
} // LTS_FOREST *LtscartReadData(char *forest_image, HANDLE *hFile1,


void LtscartFreeData(LTS_FOREST *l_forest)
{
    SPDBG_FUNC("LtscartFreeData");

    for (int i = 1; i < l_forest->symbols[INPUT].n_symbols; i++)
    {
        free(l_forest->tree[i]);
    }
    free(l_forest->tree);

    free(l_forest->features[INPUT].feature);
    free(l_forest->features[OUTPUT].feature);
    free(l_forest->features);

    free(l_forest->symbols);

    free(l_forest);
} // void LtscartFreeData(LTS_FOREST *l_forest, HANDLE m_hFile,