archive
/
windows-nt-4.0
mirror of https://github.com/lianthony/NT4.0
2
0
Fork 0
Code Issues Projects Releases Wiki Activity
Windows NT 4.0 source code leak
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2 Commits
1 Branch
0 Tags
184 MiB
 
 
 
 
 
 
Tree: b4a8d373d8
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'b4a8d373d8'
${ noResults }
windows-nt-4.0/private/windows/win4help/ftsrch/query.cpp

977 lines
35 KiB
Raw Blame History

#include "stdafx.h"
#include <math.h>
#include "vmbuffer.h"
#include "memex.h"
#include "saveload.h"
#include "TXDBase.h"
#include "bytemaps.h"
#include "textset.h"
#include "dict.h"
#include "vector.h"
#include "query.h"
// This file contains the definition of class CQuery
// Constructors
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
CQuery::CQuery()
{
m_cConWts = 0;
#if 0
m_cOverFlows = 0;
#endif
m_pszQueryText = NULL;
}
// Destructor
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
CQuery::~CQuery()
{
if (m_vbVectorConcept.Base)
FreeVirtualBuffer(&m_vbVectorConcept);
if (m_vbVectorTermFreq.Base)
FreeVirtualBuffer(&m_vbVectorTermFreq);
if (m_vbVectorWt.Base)
FreeVirtualBuffer(&m_vbVectorWt);
#if 0
if (m_vbTFOverFlow.Base)
FreeVirtualBuffer(&m_vbTFOverFlow);
#endif
}
CQuery *CQuery::NewQuery(CTextSet *pts)
{
CQuery *pQuery= NULL;
__try
{
pQuery= New CQuery;
pQuery->Initialize(pts, 100, 100000);
}
__finally
{
if (_abnormal_termination() && pQuery)
{
delete pQuery; pQuery= NULL;
}
}
return pQuery;
}
// Access Functions:
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
void CQuery::Initialize(CTextSet *textsetIn, DWORD cInEstConWtPairs, DWORD cInMaxConWtPairs)
{
ASSERT(textsetIn != NULL);
m_ptdb = textsetIn;
m_cDocuments = m_ptdb->PColl()->m_cDocuments;
ASSERT(m_cDocuments != 0);
m_vbConcepts = m_ptdb->PColl()->m_vbConcepts;
ASSERT(m_vbConcepts.Base);
m_aWtInvIndex = m_ptdb->PColl()->m_aWtInvIndex;
ASSERT(m_aWtInvIndex);
CreateVirtualBuffer(&m_vbVectorConcept , cInEstConWtPairs * sizeof(DWORD), cInMaxConWtPairs * sizeof(DWORD));
CreateVirtualBuffer(&m_vbVectorTermFreq, cInEstConWtPairs * sizeof(DWORD), cInMaxConWtPairs * sizeof(DWORD));
CreateVirtualBuffer(&m_vbVectorWt , 0 , cInMaxConWtPairs * sizeof(float));
#if 0
CreateVirtualBuffer(&m_vbTFOverFlow , 0 , 0x4000 * sizeof(TFOverFlowStruct));
#endif
// Initialize allocated memory
// VritualAlloc zeroes all memory it commits, so we don't have to worry about zeroing the virtual buffers
}
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
void CQuery::RecordConcept(DWORD ConceptId)
{
// Search for this concept id in the current document. If you find it,
// simply increment its frequency and that will take care of everything.
// If you don't find it, then enter the concept for the document.
DWORD i; // index of the con,wt pair being considered for match
for (i = 0; i < m_cConWts && Concept(i) != ConceptId; i++);
if (i == m_cConWts)
{
// This concept doesn't exist in the query. Record it.
__try
{
Concept(m_cConWts) = ConceptId;
}
__except (VirtualBufferExceptionFilter(GetExceptionCode(), GetExceptionInformation(), &m_vbVectorConcept))
{
RaiseException(STATUS_NO_MEMORY, EXCEPTION_NONCONTINUABLE, 0, NULL);
}
__try
{
TermFreq(m_cConWts) = 1; // this is the first time this concept occured for this document
}
__except (VirtualBufferExceptionFilter(GetExceptionCode(), GetExceptionInformation(), &m_vbVectorTermFreq))
{
RaiseException(STATUS_NO_MEMORY, EXCEPTION_NONCONTINUABLE, 0, NULL);
}
m_cConWts++;
}
else
{
// Term already exists in this document. Increase the occurence frequency.
// Since the term already exists in the document, it has a frequency of at least 1
#if 0
// The only time when the value can be 0 is when the frequency has exceeded 0xFFFF. In
// that case, the overflowing value is stored in the over flow area
if (TermFreq(i) == 0)
{
// go to the over flow area and update the value that tracks this term frequency
}
else
#endif
if (TermFreq(i) == 0xFFFF)
{
// we reached the upperbound on this value.
}
else // normal case. No overflow is involved. This is what happens MOST of the time.
(TermFreq(i))++;
}
}
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
// ASSUMPTION : We are only weighting one query vector. This will hold true all the time.
BOOL CQuery::WeightVector(BYTE TFModType, BYTE WeightType, BYTE NormType)
{
DWORD i;
// Copy the Term Frequencies into an array of floating points. All operations will be computed
// on these floating point weights. The final results can then be converted to a fixed point.
// IMPORTANT : ALL WEIGHTS SHOULD BE NORMALIZED TO ENSURE THAT EACH WEIGHT IS LESS THAN ONE.
// THE FIXED POINT VALUE ONLY REPRESENTS VALUES BETWEEN 0.0 AND 1.0
for (i = 0; i < m_cConWts; i++)
{
__try
{
TermWt(i) = (float)GetRealTermFreq(i);
}
__except (VirtualBufferExceptionFilter(GetExceptionCode(), GetExceptionInformation(), &m_vbVectorWt))
{
RaiseException(STATUS_NO_MEMORY, EXCEPTION_NONCONTINUABLE, 0, NULL);
}
}
ApplyWeightingScheme(TFModType, WeightType, NormType, 0, m_cConWts);
// Plug back the weighted values into the term frequency array
// ASSUMPTION : Each weight in TermWt is between 0.0 and 1.0
// Multiplying this with WT_ONE forces each TermWt weight to be a
// fixed point number ranging between 0 and WT_ONE.
for (i = 0; i < m_cConWts; i++)
TermFreq(i) = (WORD)((double)TermWt(i) * (double)WT_ONE);
return TRUE;
}
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
void CQuery::ApplyWeightingScheme(BYTE TFModType, BYTE WeightType, BYTE NormType, DWORD iFirstConWt, DWORD cConWts)
{
register DWORD i;
double Wt; // used to hold different types of cumulative values at various points in the computations
// First modify weight based on the term frequency component
switch (TFModType)
{
case NEWTF_NONE: // do nothing
break;
case NEWTF_BINARY: // Since all the terms are in, turn them on
for (i = 0; i < cConWts; i++)
TermWt(i) = (float)1.0;
break;
case NEWTF_MAXNORM:
Wt = 0.0;
for (i = 0; i < cConWts; i++)
if (TermWt(i) > Wt)
Wt = TermWt(i);
// increase Max by 0.00001 to place all normalized TFs between 0.0 and 1.0
Wt += 0.00001;
for (i = 0; i < cConWts; i++)
TermWt(i) = (float) ((double)TermWt(i) / Wt);
break;
case NEWTF_AUGNORM:
Wt = 0.0;
for (i = 0; i < cConWts; i++)
if (TermWt(i) > Wt)
Wt = TermWt(i);
// increase Max by 0.00001 to place all normalized TFs between 0.0 and 1.0
Wt += 0.00001;
for (i = 0; i < cConWts; i++)
TermWt(i) = (float) (0.5 + 0.5 * (double)TermWt(i) / Wt);
break;
default:
// Assertion failure.
break;
}
// Then modify the weight based on the collection frequency component
switch (WeightType)
{
case WT_NONE: // do nothing
break;
// if a concept occurs in all docs, let's assign it a small value instead of assigning it a 0.0
case WT_TFIDF:
for (i = 0; i < cConWts; i++)
if (m_cDocuments == DocFromCumFreq(Concept(i + iFirstConWt)))
TermWt(i) = (float) 0.005;
else
TermWt(i) = (float) ((double)TermWt(i) * log((double)m_cDocuments / (double)DocFromCumFreq(Concept(i + iFirstConWt))));
break;
case WT_PROB:
for (i = 0; i < cConWts; i++)
if (m_cDocuments == DocFromCumFreq(Concept(i + iFirstConWt)))
TermWt(i) = (float) 0.005;
else
TermWt(i) = (float) ((double)TermWt(i) * log((double)(m_cDocuments - DocFromCumFreq(Concept(i + iFirstConWt))) / (double)DocFromCumFreq(Concept(i + iFirstConWt))));
break;
default:
// ASSERTion failure.
break;
}
switch (NormType)
{
case NORM_NONE:
break;
case NORM_SUM:
Wt = 0.0;
for (i = 0; i < cConWts; i++)
Wt += (double) TermWt(i);
for (i = 0; i < cConWts; i++)
TermWt(i) = (float) ((double)TermWt(i) / Wt);
break;
case NORM_COSINE:
Wt = 0.0;
// compute sum of squares of weights in the vector
for (i = 0; i < cConWts; i++)
Wt += TermWt(i) * TermWt(i);
Wt = sqrt(Wt);
// normalize each weight by the sum of squares computed above
for (i = 0; i < cConWts; i++)
TermWt(i) = (float) ((double)TermWt(i) / Wt);
break;
case NORM_MAX:
break;
}
}
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
BOOL CQuery::RankDocuments(SimStruct *aInSimilarity, DWORD cInHits)
{
register DWORD i, j;
DWORD ConceptId, DocId;
DWORD cDocs;
DWORD DocPos; // tracks the position of a document in the similiarity structure
DWORD startDocPos;
if (cInHits == 0)
{
SetLastError(QUERYERROR_NOHITS);
return FALSE;
}
if (aInSimilarity == NULL)
{
SetLastError(QUERYERROR_EMPTYSIMARRAY);
return FALSE;
}
// ASSUME THAT THE SIMILARITY STRUCTURE ARRAY HAS ENOUGH ENTRIES TO SUPPORT cInHits
// Zero out any existing similarity values
for (i = 0; i < cInHits; i++)
aInSimilarity[i].Similarity = 0;
// Compute similarity. Walk the doc,wt list for each concept
for (i = 0; i < m_cConWts; i++)
{
// Ignore concepts that have a zero weight. Later, we may want to extend this idea
// to suppress weights below a small value.
if (TermFreq(i) == 0)
continue;
ConceptId = Concept(i);
cDocs = DocFromCumFreq(ConceptId);
// Consider each doc in the (Doc, Wt) list for this concept and score docs that
// are in the predetermined hit list.
startDocPos = DocList(ConceptId); // get the starting point of the inverted list.
for (j = 0; j < cDocs; j++)
{
if (j == 0)
DocId = m_ptdb->PColl()->GetDocumentGap(&startDocPos) - 1;
else
DocId += m_ptdb->PColl()->GetDocumentGap(&startDocPos);
DocPos = GetDocPosInList(aInSimilarity, cInHits, DocId);
if (DocPos != DOESNOTEXIST)
aInSimilarity[DocPos].Similarity += TermFreq(i) * WtFromInvList(ConceptId, j);
/* IF WE LIMIT SIMILARITY TO 24 BITS, USE THE FOLLOWING LINE. IF WE LIMIT TO ANY OTHER NUMBER
OF BITS n, n < 32, RIGHT SHIFT THE RHS BY 32 - n.
aInSimilarity[DocPos].Similarity += (TermFreq(i) * WtFromInvList(ConceptId, j)) >> 8;
*/
}
}
/* MOVE SORTING TO THE CALLER!
// sort the scored documents.
qsort(aInSimilarity, cInHits, sizeof(SimStruct), CompareSimStruct);
*/
// return the number of hits
return cInHits;
}
/*************************************************************************
* FUNCTION : *
* *
* RETURNS : *
* *
* PURPOSE : *
* *
* PARAMETERS : *
* *
* SIDE EFFECTS : *
* *
* DESCRIPTION : *
* *
* HISTORY : *
* *
* Author Date Action *
* ------ ---- ------ *
* *
* KrishnaN 4/23/94 Creation. *
* *
*************************************************************************/
// cInHits is the number of previous hits. cInMaxHits is the maximum number of documents to retrieve
DWORD CQuery::RetrieveWithFeedback( SimStruct *aInSimilarity, DWORD cInMaxHits,
PWCHAR pwRelDocText, int cwRelDocText,
PWCHAR pwNonRelDocText, int cwNonRelDocText
)
{
DWORD i, j, k;
DWORD DocId;
DWORD cHits = 0;
DWORD cDocs, DocPos;
DWORD LastDocInPos = 0; // Document position which has the least partial similarity match
DWORD UBCurrentDoc; // Upper bound of the current document
DWORD CCurrentDoc; // C (CurrentDoc)
DWORD CFirstDocOut = 0; // C (FirstDocOut)
DWORD UBFirstDocOut; // Upper bound of the first document outside the RSet
LPDWORD aQTermSummation= NULL; // summation of query terms
DWORD startDocPos;
ASSERT(aInSimilarity);
ASSERT(pwRelDocText && cwRelDocText);
__try
{
// Add terms from the query. We will either have to reindex the initial query
// or store the term frequencies the first time they were computed.
// Assume that each term only occurs once in the query.
// This assumption usually holds good for queries typed in by the user.
// It doesn't matter much even if it doesn't hold good because the document
// text overwhelms the original query.
for (i = 0; i < m_cConWts; i++) // Enforce the above assumption.
TermFreq(i) = 1;
// Add terms from the relevant documents to the query
IndexDocumentText(pwRelDocText, cwRelDocText, TRUE);
// For the non-relevant document text, decrease termfreqs of concepts it has in common with
// the newly formed query.
// NOTE : The caller should pass in the document text of only the highest ranked
// non-relevant document to get the best results (Dec-Hi relevance feedback method).
if (pwNonRelDocText && cwNonRelDocText)
{
IndexDocumentText(pwNonRelDocText, cwNonRelDocText, FALSE);
// At this point, we may have some zero weighted concepts in the
// query. Remove any such concept, weight pairs.
for (i = j = 0; i < m_cConWts;)
{
// search for the next zero weighted concept
for (; j < m_cConWts && TermFreq(j) > 0; j++);
i = j; // update i so that outer loop terminates appropriately
if (j < m_cConWts) // we found a zero weighted concept
{
// search for the next non-zero weighted concept
for (k = j + 1; k < m_cConWts && TermFreq(k) == 0; k++);
if (k < m_cConWts) // we found a non-zero weighted concept
{
// copy the con,wt pair
Concept(j) = Concept(k);
TermFreq(j) = TermFreq(k);
// erase the copied pair
TermFreq(k) = 0;
j++; // update j so that the for loop advances
}
else // no more non-zero weighted concepts. we are done.
i = k;
}
}
ASSERT(i <= m_cConWts);
// Count the new number of ConWt pairs
for (m_cConWts = 0; TermFreq(m_cConWts) > 0; m_cConWts++);
}
if (!m_cConWts) __leave;
// Now weight the query vector
WeightVector(NEWTF_NONE, WT_TFIDF, NORM_COSINE);
SortQuery();
aQTermSummation = (LPDWORD)VAlloc(FALSE, m_cConWts * sizeof(DWORD));
// Compute summation of query terms. This summation will be used to compute the upperbound.
// aQTermSummation[j] gives is the sum of weights of query terms j+1 to the last term in the query
// The << 16 left shift takes care of multiplication by 1 i.e. it makes this a true 32-bit value
for (aQTermSummation[m_cConWts - 1] = TermFreq(i) << 16, i = m_cConWts - 1; i > 0; i--)
aQTermSummation[i - 1] = aQTermSummation[i] + TermFreq(i-1) << 16;
/*
// scale the values to 24 bit
for (i = 0; i < m_cConWts; i++)
aQTermSummation[i] = aQTermSummation[i] >> 8;
*/
/* IMPORTANT ASSUMPTION : The aInSimilarity array is properly initialized.
Proper initialization includes resetting all docid and sim values to 0
and the CollId field to the appropriate collection id.
If aInSimilarity is not properly initialized, the docid, sim values will
still be correct, but the caller will have no way of finding the collection
id of the docid, sim values set here.
*/
// Compute similarity. Walk the doc,wt list for each concept
// Compute until all terms are exhausted or the stopping conditions are met
// Skip terms that occur too frequently (how frequent is too frequent ?)
i = 0;
do
{
// CODE TO SKIP TERMS THAT ARE TOO FREQUENT CAN APPEAR HERE
// if (term is too frequent)
// {
// i++;
// continue;
// }
cDocs = DocFromCumFreq(Concept(i));
DocId = 0;
startDocPos = DocList(Concept(i));
// Consider each doc in the (Doc, Wt) list for this concept and score docs that
// are in the predetermined hit list.
for (j = 0; j < cDocs; j++)
{
// The first doc in an inverted list for a concept is encoded as docid + 1. The subsequent
// gaps are encoded as they are.
if (j == 0)
DocId = m_ptdb->PColl()->GetDocumentGap(&startDocPos) - 1;
else
DocId += m_ptdb->PColl()->GetDocumentGap(&startDocPos);
DocPos = GetDocPosInList2(aInSimilarity, cHits, DocId);
// ALG : If RsetNotFull then
if (cHits < cInMaxHits)
{
// ALG : Compute C(Document);
// ALG : Enter Document into the RSet
if (DocPos == DOESNOTEXIST)
{
// Add this new document
DocPos = cHits;
aInSimilarity[DocPos].DocId = DocId;
cHits++;
aInSimilarity[DocPos].Similarity += TermFreq(i) * WtFromInvList(Concept(i), j);
/* If we scale similarity to 24 bits, use this line instead of the above
aInSimilarity[DocPos].Similarity += (TermFreq(i) * WtFromInvList(Concept(i), j)) >> 8;
*/
if (aInSimilarity[DocPos].Similarity < aInSimilarity[LastDocInPos].Similarity)
LastDocInPos = DocPos;
}
else
{
// recompute the LastDocIn document if this document was LastDocIn before this cumulation
aInSimilarity[DocPos].Similarity += TermFreq(i) * WtFromInvList(Concept(i), j);
/* If we scale similarity to 24 bits, use this line instead of the above
aInSimilarity[DocPos].Similarity += (TermFreq(i) * WtFromInvList(Concept(i), j)) >> 8;
*/
if (DocPos == LastDocInPos)
for (k = 0; k < cHits; k++)
if (aInSimilarity[k].Similarity < aInSimilarity[LastDocInPos].Similarity)
LastDocInPos = k;
}
}
// ALG : else
else
{
// ALG : Compute Upperbound (Document)
// At this point we will also compute the partial similarity for this document
if (DocPos == DOESNOTEXIST)
{
CCurrentDoc = TermFreq(i) * WtFromInvList(Concept(i), j);
/* If we scale similarity to 24 bits, use this line instead of the above
CCurrentDoc = (TermFreq(i) * WtFromInvList(Concept(i), j)) >> 8;
*/
UBCurrentDoc = aQTermSummation[i];
}
else
{
CCurrentDoc = aInSimilarity[DocPos].Similarity + (TermFreq(i) * WtFromInvList(Concept(i), j));
/* If we scale similarity to 24 bits, use this line instead of the above
CCurrentDoc = aInSimilarity[DocPos].Similarity + ((TermFreq(i) * WtFromInvList(Concept(i), j)) >> 8);
*/
// The upper bound could exceed the maximum possible similarity value. We should protect
// against that by bounding the upper bound.
if ((MAXSIM - aInSimilarity[DocPos].Similarity) < aQTermSummation[i])
UBCurrentDoc = MAXSIM;
else
UBCurrentDoc = aInSimilarity[DocPos].Similarity + aQTermSummation[i];
}
// ALG : If U(Document) <= C(LastDoc) then
// ALG : DoNotAllocate / Remove Document
// If U < C condition is met and the doc is already in, remove it
if (UBCurrentDoc <= aInSimilarity[LastDocInPos].Similarity)
{
// This document is a loser. Check to see if it is at least better than
// the first document outside the RSet.
if (CCurrentDoc > CFirstDocOut)
CFirstDocOut = CCurrentDoc;
// Remove this loser if it was already entered
if (DocPos != DOESNOTEXIST)
{
// remove current document from the list
// remove by copying the document at the end into this document's position
aInSimilarity[DocPos].Similarity = aInSimilarity[cHits - 1].Similarity;
aInSimilarity[DocPos].DocId = aInSimilarity[cHits - 1].DocId;
cHits--;
ASSERT (cHits);
// Now that we changed the document set, recompute the last doc position
for (k = 0; k < cHits; k++)
if (aInSimilarity[k].Similarity < aInSimilarity[LastDocInPos].Similarity)
LastDocInPos = k;
}
}
// ALG : else
// ALG : Compute C (Document)
// ALG : if (C (Document) > C (LastDoc) then
// ALG : Enter Document into the RSet
else
{
if (CCurrentDoc > aInSimilarity[LastDocInPos].Similarity)
{
if (DocPos == DOESNOTEXIST)
{
// Since the RSet is already full, the only way to enter the current document
// is by replacing the document at the LastDocInPos - i.e replacing the doc with
// the least partial match
// Before replacing the LastDocIn, let us save it as the FirstDocOut
CFirstDocOut = aInSimilarity[LastDocInPos].Similarity;
// Replace
aInSimilarity[LastDocInPos].DocId = DocId;
aInSimilarity[LastDocInPos].Similarity = CCurrentDoc;
// Now that we changed the document set, recompute the last doc position
for (k = 0; k < cHits; k++)
if (aInSimilarity[k].Similarity < aInSimilarity[LastDocInPos].Similarity)
LastDocInPos = k;
}
else
{
aInSimilarity[DocPos].Similarity = CCurrentDoc;
// recompute the LastDocIn document if this document was LastDocIn before this cumulation
if (DocPos == LastDocInPos)
for (k = 0; k < cHits; k++)
if (aInSimilarity[k].Similarity < aInSimilarity[LastDocInPos].Similarity)
LastDocInPos = k;
}
}
}
}
}
/* BEGIN : Fix FOR BUG 18016 */
if (cHits < cInMaxHits)
UBFirstDocOut = 0xFFFFFFFF; // No doc outside the RSet, so the first doc out potentially has infinite upperboud
else
/* END : fix for BUG 18016 */
// Compute upper bound of FirstDocOut
UBFirstDocOut = CFirstDocOut + aQTermSummation[i];
i++;
// ALG : until LastQueryTerm or U(FirstDocOut) <= C(LastDocIn)
// NOTE : We converted a repeat - until into a do - while, so the loop termination conditions are different
// between algorithm and the implementation.
} while (i < m_cConWts && UBFirstDocOut > aInSimilarity[LastDocInPos].Similarity && TermFreq(i) > 0 ); // INTRODUCE MORE STOPPING CONDITIONS HERE
#if 0 // statistics
if ( i < m_cConWts )
{
char szBuffer[200];
DWORD cDocsExamined = 0, cDocsNotExamined = 0;
for (k = 0; k < i; k++)
cDocsExamined += DocFromCumFreq(Concept(k));
for (k = i; k < m_cConWts; k++)
cDocsNotExamined += DocFromCumFreq(Concept(k));
wsprintf(szBuffer, "Examined only %u lists out of %u lists and only %u docs out of %u docs", i, m_cConWts, cDocsExamined, cDocsExamined+cDocsNotExamined);
MessageBox(GetFocus(), szBuffer, "Query Optimization", MB_OK);
}
#endif // 0, statistics
/* MOVE SORTING TO THE CALLER. THIS IS DONE TO ENABLE MULTIPLE FILE SEARCHES. THE CALLER WILL
GET ALL THE RESULTS INTO A HUGE SIMSTRUCT ARRAY AND SORT IT
// sort the scored documents.
qsort(aInSimilarity, cHits, sizeof(SimStruct), CompareSimStruct);
*/
}
__finally
{
if (aQTermSummation) VFree(aQTermSummation);
}
return cHits;
}
void CQuery::SortQuery()
{
TempConWtStruct * aConWts = NULL;
register DWORD i;
__try
{
// Sort the query concept, wt pairs based on the weight of the concepts. This will be used
// when we employ stop conditions to reduce the number of documents considered.
// Since the concepts and weights are not in the same structure, we need to
// copy them to a temporary buffer and then copy the sorted values back
aConWts = (TempConWtStruct *) VAlloc(FALSE, sizeof(TempConWtStruct) * m_cConWts);
for (i = 0; i < m_cConWts; i++)
{
aConWts[i].ConceptId = Concept(i);
aConWts[i].Weight = TermFreq(i);
}
qsort(aConWts, m_cConWts, sizeof(TempConWtStruct), CompareTempConWtStruct);
for (i = 0; i < m_cConWts; i++)
{
Concept(i) = aConWts[i].ConceptId;
TermFreq(i) = (WORD)aConWts[i].Weight;
}
}
__finally
{
if (aConWts) VFree(aConWts);
}
}
// Compare two dwords and return 0, < 0, or > 0 to help qsort sort in decreasing order
int _cdecl CompareTempConWtStruct(const void *arg1, const void *arg2)
{
if (((TempConWtStruct *)arg2)->Weight > ((TempConWtStruct *)arg1)->Weight)
return 1;
else if (((TempConWtStruct *)arg2)->Weight < ((TempConWtStruct *)arg1)->Weight)
return -1;
else
return 0;
}
// Compare two dwords and return 0, < 0, or > 0 to help qsort sort in decreasing order
int _cdecl CompareSimStruct(const void *arg1, const void *arg2)
{
if (((SimStruct *)arg2)->Similarity > ((SimStruct *)arg1)->Similarity)
return 1;
else if (((SimStruct *)arg2)->Similarity < ((SimStruct *)arg1)->Similarity)
return -1;
else
return 0;
}
// ASSUMPTION : There are at least two elements in the list
__inline DWORD CQuery::GetDocPosInList(SimStruct *aInSimilarity, DWORD cInHits, DWORD DocId)
{
register DWORD high = cInHits, low = 0, mid;
while (low < high)
{
mid = low + (high - low)/2;
if (DocId < aInSimilarity[mid].DocId)
high = mid;
else if (DocId > aInSimilarity[mid].DocId)
low = mid + 1;
else
return mid;
}
return DOESNOTEXIST;
}
__inline DWORD CQuery::GetDocPosInList2(SimStruct *aInSimilarity, DWORD cInHits, DWORD DocId)
{
register DWORD i;
for (i = 0; i < cInHits; i++)
if (aInSimilarity[i].DocId == DocId)
return i;
// the doc has not been found
return DOESNOTEXIST;
}
void CQuery::IndexDocumentText(PWCHAR pwDocText, int cwText, BOOL fRelevant)
{
int n, nTokens, nMore;
PUINT pwHash = NULL;
PBYTE pbType = NULL;
PWCHAR *paStart = NULL,
*paEnd = NULL;
PWCHAR pwText = pwDocText; // we will leave the pwDocText untouched so that
// the caller can delete that memory buffer.
DWORD ConId;
nMore = cwText;
ASSERT(pwText && cwText);
__try
{
// cwText is probably a lot more than we need, but it guarantees us that we won't run out of memory
// for tokens
pwHash = New UINT[cwText];
pbType = New BYTE[cwText];
paStart = New PWCHAR[cwText];
paEnd = New PWCHAR[cwText];
if (pwText && pwHash && paStart && pbType && paEnd)
{
nTokens = WordBreakW(&pwText, &nMore, paStart, paEnd, pbType, pwHash, cwText, REMOVE_SPACE_CHARS);
for (n = 0; n < nTokens; n++)
{
// EnterWord with last param set to TRUE is only looking up, not entering, a word
ConId = m_ptdb->PDict()->EnterWord(paStart[n], paEnd[n] - paStart[n], TRUE, TRUE);
if (ConId != EOL && ConId != STOPWORD)
if (fRelevant)
RecordConcept(ConId);
else // not relevant
{
DWORD i;
// For each concept in the document, check to see if it exists
// in the query. If it does, subtract it from the query's term frequency
for (i = 0; i < m_cConWts && Concept(i) != ConId; i++);
if (i < m_cConWts)
// This concept exists in the query. Subtract this term from the query.
if (TermFreq(i) > 0)
TermFreq(i) -= 1;
}
}
}
}
__finally
{
if (paEnd) { delete paEnd; paEnd = NULL; }
if (paStart) { delete paStart; paStart = NULL; }
if (pbType) { delete pbType; pbType = NULL; }
if (pwHash) { delete pwHash; pwHash = NULL; }
}
}