|
|
//+---------------------------------------------------------------------------
//
// Microsoft Windows
// Copyright (C) Microsoft Corporation 1997-1998
//
// File: cidump.cxx
//
// Contents: CI catalog dump utility
//
// History: 09-Apr-97 SitaramR Created
//
//----------------------------------------------------------------------------
#define _OLE32_
#define __QUERY__
extern "C"{ #include <nt.h>
#include <ntioapi.h>
#include <ntrtl.h>
#include <nturtl.h>
}
#include <ctype.h>
#include <float.h>
#include <limits.h>
#include <malloc.h>
#include <math.h>
#include <memory.h>
#include <stddef.h>
#include <string.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <windows.h>
#include <imagehlp.h>
#include <lmcons.h>
#define _DCOM_
#define _CAIROSTG_
#include <cidebnot.h>
#include <cierror.h>
#define OLEDBVER 0x0250 // enable ICommandTree interface
#include <oleext.h>
#include <oledberr.h>
#include <oledb.h>
#include <query.h>
#include <stgprop.h>
#include <filter.h>
#include <filterr.h>
#include <vquery.hxx>
#include <restrict.hxx>
//
// Base services
//
#include <ciexcpt.hxx>
#include <smart.hxx>
#include <tsmem.hxx>
#include <xolemem.hxx>
#include <dynarray.hxx>
#include <dynstack.hxx>
#include <dblink.hxx>
#include <cisem.hxx>
#include <thrd32.hxx>
#include <readwrit.hxx>
#include <ci.h>
//
// Debug files from
//
#include <cidebug.hxx>
#include <vqdebug.hxx>
//
// CI-specific
//
#include <align.hxx>
#include <memser.hxx>
#include <memdeser.hxx>
#include <tgrow.hxx>
#include <funypath.hxx>
#include <params.hxx>
#include <key.hxx>
#include <keyarray.hxx>
#include <irest.hxx>
#include <cursor.hxx>
#include <idxids.hxx>
#include <dberror.hxx>
// property-related macros and includes
#include <propapi.h>
#include <propstm.hxx>
extern UNICODECALLOUTS UnicodeCallouts;#define DebugTrace( x, y, z )
#ifdef PROPASSERTMSG
#undef PROPASSERTMSG
#endif
#define PROPASSERTMSG( x, y )
#include <rcstrmhd.hxx>
#include <xact.hxx>
#include <pidxtbl.hxx>
#include "cistore.hxx"
#include "physidx.hxx"
#include "pcomp.hxx"
#include "cidir.hxx"
DECLARE_INFOLEVEL(ci)//DECLARE_INFOLEVEL(vq)
//extern BOOL ExceptDllMain( HANDLE hDll, DWORD dwReason, LPVOID lpReserved );
unsigned fVerbose = 0; // Verbose mode dumps all keys, wids and occurrences
unsigned fStats = 0; // Just dump wid/occ counts.
unsigned fFullStats = 0; // Just dump wid/occ counts.
unsigned fKeys = 0; // Key info
unsigned fDistribution = 0; // Key distribution info
unsigned fOccurrences = 0; // Occurrence distribution info
extern "C" GUID CLSID_CTextIFilter = CLSID_TextIFilter;
DWORD Bytes( BitOffset & boff ){ return ( boff.Page() * CI_PAGE_SIZE ) + ( ( boff.Offset() + 7 ) / 8 );}
ULONGLONG BitDistance( BitOffset & b1, BitOffset & b2 ){ ULONGLONG bA = ((ULONGLONG) b1.Page() * (ULONGLONG) ( CI_PAGE_SIZE * 8) ) + b1.Offset(); ULONGLONG bB = ((ULONGLONG) b2.Page() * (ULONGLONG) ( CI_PAGE_SIZE * 8) ) + b2.Offset();
return ( bB - bA );}
//+---------------------------------------------------------------------------
//
// Function: LocaleToCodepage
//
// Purpose: Returns a codepage from a locale
//
// Arguments: [lcid] -- Locale
//
// History: 09-Apr-97 SitaramR Created
//
//----------------------------------------------------------------------------
ULONG LocaleToCodepage( LCID lcid ){ const BUFFER_LENGTH = 10; WCHAR wcsCodePage[BUFFER_LENGTH];
int cwc = GetLocaleInfoW( lcid, LOCALE_IDEFAULTANSICODEPAGE, wcsCodePage, BUFFER_LENGTH );
//
// If error, return Ansi code page
//
if ( cwc == 0 ) { ciDebugOut(( DEB_ERROR, "GetLocaleInfoW for lcid %d returned %d\n", lcid, GetLastError() ));
return CP_ACP; }
return wcstoul( wcsCodePage, 0 , 10 );}
void Usage(){ printf( "Usage: cidump <catalog path> [-e indexid] [-d] [-s] [-f] [-v] [-w workid]\n" " -d : Distribution of keys in documents (key, wid, occ)\n" " -e : Exclude this index\n" " -k : Print keys with #files, #occ, pid, size\n" " -s : Stats only\n" " -f : Full Stats only (for all keys)\n" " -v : Verbose (full occurrence information)\n" " -w : Data for this workid only\n\n" "Example: cidump e:\\testdump\\catalog.wci\n" );}
void DumpDirectoryKey( unsigned i, CDirectoryKey & Key ){ BitOffset bo; Key.Offset( bo ); printf( " key %d cb 0x%x, PropId 0x%x, opage 0x%x, obits 0x%x\n", i, Key.Count(), Key.PropId(), bo.Page(), bo.Offset() );} //DumpDirectoryKey
//+---------------------------------------------------------------------------
//
// Function: main
//
// Purpose: Main dump routine
//
// History: 09-Apr-97 SitaramR Created
// 02-Nov-98 KLam Passed disk space to leave to CiStorage
//
//----------------------------------------------------------------------------
int __cdecl main( int argc, char * argv[] ){ char * pszCatDir = argv[1]; WORKID widTarget = 0xFFFFFFFF;
INDEXID aExclude[100]; unsigned cExclude = 0;
if ( argc < 2 ) { Usage(); return 0; }
for ( int i = 1; i < argc; i++ ) { if ( argv[i][0] == '-' ) { switch ( argv[i][1] ) { case 'e': case 'E': i++; aExclude[cExclude] = strtoul( argv[i], 0, 16 ); cExclude++; break;
case 's': case 'S': fStats = 1; break;
case 'd': case 'D': fDistribution = 1; break;
case 'f': case 'F': fFullStats = 1; break;
case 'k': case 'K': fKeys = 1; break;
case 'o': case 'O': fOccurrences = 1; break;
case 'v': case 'V': fVerbose = 1; break;
case 'w': case 'W': i++; widTarget = strtoul( argv[i], 0, 10 ); break;
default: Usage(); return 0; } } else pszCatDir = argv[i]; }
if ( fKeys ) { fFullStats = 0; fStats = 0; fVerbose = 0; }
if ( fDistribution ) { fFullStats = 0; fStats = 0; fVerbose = 0; fKeys = 0; fOccurrences = 0; }
if ( fOccurrences ) { fFullStats = 0; fStats = 0; fVerbose = 0; fKeys = 0; fDistribution = 0; }
if ( (pszCatDir[0] != '\\' || pszCatDir[1] != '\\') && (pszCatDir[0] == '\0' || pszCatDir[1] != ':' || pszCatDir[2] != '\\') ) { printf("Use full path name for catalog path!\n\n"); Usage(); return 1; }
WCHAR wszCatDir[MAX_PATH];
LocaleToCodepage( GetSystemDefaultLCID() );
ULONG cwcActual = MultiByteToWideChar( LocaleToCodepage( GetSystemDefaultLCID() ), 0, pszCatDir, strlen( pszCatDir ) + 1, wszCatDir, MAX_PATH );
//ExceptDllMain( 0, DLL_PROCESS_ATTACH, 0 );
//
// Keep stats on where we are when we die...
//
CIndexRecord recCrash; CKeyBuf keyCrash; BitOffset boffCrash;
ULONG maxOccCounts = 1024;
TRY { ULONGLONG * aOccCounts = new ULONGLONG[ maxOccCounts ]; ULONGLONG * acbOccCounts = new ULONGLONG[ maxOccCounts ];
CTransaction xact; XPtr<CiStorage> xStorage( new CiStorage( wszCatDir, *((ICiCAdviseStatus *)0), CI_MIN_DISK_SPACE_TO_LEAVE_DEFAULT ) );
XPtr<PIndexTable> xIndexTable( xStorage->QueryIndexTable( xact ) );
SIndexTabIter xIndexIter( xIndexTable->QueryIterator() );
if ( xIndexIter->Begin() ) { CIndexRecord record;
while ( xIndexIter->NextRecord( record ) ) { if ( record.Type() == itMaster || record.Type() == itShadow ) { //
// Do we want to skip this one?
//
for ( unsigned i = 0; i < cExclude; i++ ) { if ( aExclude[i] == record.Iid() ) break; }
if ( i < cExclude ) continue;
recCrash = record; boffCrash.Init( 0, 0 );
if ( !fKeys && !fDistribution && !fOccurrences ) { if ( record.Type() == itMaster ) printf( "Index master: " ); else printf( "Index shadow: " );
printf( "Object id = 0x%x, Index id = 0x%x, MaxWorkid = 0x%x\n", record.ObjectId(), record.Iid(), record.MaxWorkId() ); }
ULONG maxWid = record.MaxWorkId();
SStorageObject xStorageObj( xStorage->QueryObject( record.ObjectId() ) );
PMmStream * pStream = xStorage->QueryExistingIndexStream( xStorageObj.GetObj(), PStorage::eOpenForRead ); XPtr<PMmStream> xStream( pStream ); CPhysIndex *pPhysIndex = new CPhysIndex ( *xStorage.GetPointer(), xStorageObj.GetObj(), record.ObjectId(), PStorage::eOpenForRead, xStream ); Win4Assert( 0 == xStream.GetPointer() );
XPtr<CPhysIndex> xPhysIndex( pPhysIndex );
CiDirectory *pDir = (CiDirectory *) xStorage->QueryExistingDirectory( xStorageObj.GetObj(), PStorage::eOpenForRead ); XPtr<CiDirectory> xDir( pDir);
unsigned cLevel1 = xDir->Level1Count(); unsigned cLevel2 = xDir->Level2Count();
if ( !fKeys && !fDistribution && !fOccurrences ) { printf( "directory has %d level 1 and %d level 2 keys\n", cLevel1, cLevel2 );
if ( fVerbose ) { for ( unsigned i = 0; i < cLevel1; i++ ) DumpDirectoryKey( i, xDir->GetLevel1Key( i ) );
for ( i = 0; i < cLevel2; i++ ) DumpDirectoryKey( i, xDir->GetLevel2Key( i ) ); } }
ULONGLONG cOccInIndex = 0; ULONGLONG cWidsWithOneOcc = 0; ULONGLONG cWidsWith5OrLessOcc = 0; ULONGLONG cWidsWithOcc = 0; ULONG cKeysWithOneWid = 0; ULONG cKeysInIndex = 0; ULONG cbMinimumKey = 100000; ULONG cbMaximumKey = 0; ULONGLONG cBitsForOcc = 0; ULONGLONG cBitsForWid = 0; ULONGLONG cBitsForKey = 0; ULONG cKeysInMoreThanHalfWids = 0; ULONG cOccMinOverall = 0xffffffff; ULONG cOccMaxOverall = 0;
ZeroMemory( aOccCounts, sizeof( ULONGLONG ) * maxOccCounts ); ZeroMemory( acbOccCounts, sizeof( ULONGLONG ) * maxOccCounts );
CPersDeComp *pDecomp = new CPersDeComp( *xDir.GetPointer(), record.Iid(), *xPhysIndex.GetPointer(), record.MaxWorkId(), TRUE, TRUE ); XPtr<CPersDeComp> xDecomp( pDecomp );
const CKeyBuf *pKey; BitOffset off; xDecomp->GetOffset( off );
BitOffset boffBefore; boffBefore.Init( 0, 0 );
for ( pKey = xDecomp->GetKey(); pKey != NULL; pKey = xDecomp->GetNextKey() ) { BitOffset boAfterKey; xDecomp->GetOffset( boAfterKey ); cBitsForKey += BitDistance( boffBefore, boAfterKey );
cKeysInIndex++;
keyCrash = *pKey; xDecomp->GetOffset( boffCrash ); BOOL fFirst = TRUE;
unsigned cWid = 0; unsigned cOccTotal = 0; unsigned cOccMin = 0xFFFFFFFF; unsigned cOccMax = 0;
BitOffset boBeforeWid; xDecomp->GetOffset( boBeforeWid );
if ( fDistribution ) printf( "%ws", pKey->GetStr() );
for ( WORKID wid = xDecomp->WorkId(); wid != widInvalid; wid = xDecomp->NextWorkId() ) { BitOffset boAfterWid; xDecomp->GetOffset( boAfterWid ); cBitsForWid += BitDistance( boBeforeWid, boAfterWid );
cWidsWithOcc++;
xDecomp->GetOffset( boffCrash ); cWid++; if ( fFirst ) { if ( fVerbose || fFullStats || wid == widTarget ) { printf( " Key size = 0x%x, pid = 0x%x, buffer = %ws, " "near page %u (0x%x), bit offset %u (0x%x) --> File offset 0x%x\n", pKey->Count(), pKey->Pid(), pKey->GetStr(), boffCrash.Page(), boffCrash.Page(), boffCrash.Offset(), boffCrash.Offset(), boffCrash.Page() * CI_PAGE_SIZE + (boffCrash.Offset() + 7)/8 );
if ( !fKeys ) fFirst = FALSE; } }
if ( (fVerbose && 0 == widTarget) || wid == widTarget ) printf( "\t wid = 0x%x, MaxOcc = 0x%x, OccCount = 0x%x, " "near page %u (0x%x), bit offset %u (0x%x) --> File offset 0x%x, " "\n\t Occurrences = ", wid, xDecomp->MaxOccurrence(), xDecomp->OccurrenceCount(), boffCrash.Page(), boffCrash.Page(), boffCrash.Offset(), boffCrash.Offset(), boffCrash.Page() * CI_PAGE_SIZE + (boffCrash.Offset() + 7)/8 );
unsigned cOcc = 0;
BitOffset bo1; xDecomp->GetOffset( bo1 );
for ( OCCURRENCE occ = xDecomp->Occurrence(); occ != OCC_INVALID; occ = xDecomp->NextOccurrence() ) { xDecomp->GetOffset( boffCrash ); cOcc++;
if ( (fVerbose && 0 == widTarget) || wid == widTarget ) printf( "0x%x ", occ ); }
if ( cOcc >= maxOccCounts ) { ULONG newMax = cOcc * 2;
ULONGLONG *a1 = new ULONGLONG[ newMax ]; ULONGLONG *a2 = new ULONGLONG[ newMax ];
ZeroMemory( a1, sizeof( ULONGLONG ) * newMax ); ZeroMemory( a2, sizeof( ULONGLONG ) * newMax );
RtlCopyMemory( a1, aOccCounts, sizeof( ULONGLONG ) * maxOccCounts ); RtlCopyMemory( a2, acbOccCounts, sizeof( ULONGLONG ) * maxOccCounts );
delete [] aOccCounts; delete [] acbOccCounts; aOccCounts = a1; acbOccCounts = a2;
maxOccCounts = newMax; }
#if 0
if ( cOcc >= 500 ) { printf( "cocc %d in wid %d, pid %#x, len %d, %ws\n", cOcc, wid, pKey->Pid(), pKey->StrLen(), pKey->GetStr() );
printf( " cb %d: ", pKey->Count() ); BYTE const * pb = pKey->GetBuf(); for ( ULONG i = 0; i < pKey->Count(); i++ ) printf( " %#x", pb[i] ); printf( "\n" ); }#endif
BitOffset bo2; xDecomp->GetOffset( bo2 ); ULONG cDelta = (ULONG) BitDistance( bo1, bo2 ); cBitsForOcc += cDelta;
aOccCounts[ cOcc ]++; acbOccCounts[ cOcc ] += cDelta;
// if ( fOccurrences )
// printf( "%d, %d\n", cOcc, cDelta );
cOccInIndex += cOcc;
if ( 1 == cOcc ) cWidsWithOneOcc++;
if ( cOcc <= 5 ) cWidsWith5OrLessOcc++;
if ( (fVerbose && 0 == widTarget) || wid == widTarget ) printf( "\n" );
cOccTotal += cOcc;
if ( cOcc > cOccMax ) cOccMax = cOcc;
if ( cOcc < cOccMin ) cOccMin = cOcc;
xDecomp->GetOffset( boBeforeWid ); }
if ( 1 == cWid ) cKeysWithOneWid++;
if ( cWid >= ( maxWid/2 ) ) { printf( "key with >= half wids (%u):\n", cWid );
printf( " pid %#x, len %d, %ws\n", pKey->Pid(), pKey->StrLen(), pKey->GetStr() );
printf( " cb %d: ", pKey->Count() ); BYTE const * pb = pKey->GetBuf(); for ( ULONG i = 0; i < pKey->Count(); i++ ) printf( " %#x", pb[i] ); printf( "\n" );
cKeysInMoreThanHalfWids++; }
BitOffset boffAfter; xDecomp->GetOffset( boffAfter );
DWORD b = Bytes( boffAfter ) - Bytes( boffBefore );
if ( b < cbMinimumKey ) cbMinimumKey = b;
if ( b > cbMaximumKey ) cbMaximumKey = b;
// Print this data:
// # of files with the key
// # of occurrences of key in all files
// property id
// # of bytes taken by this key in the index
// string form of the key
//
if ( fKeys ) printf( "%7u %10u %4u %7u %ws\n", cWid, cOccTotal, pKey->Pid(), b, pKey->GetStr() );
boffBefore = boffAfter;
if ( cOccMin < cOccMinOverall ) cOccMinOverall = cOccMin;
if ( cOccMax > cOccMaxOverall ) cOccMaxOverall = cOccMax;
if ( fFullStats ) printf( "%u Workid(s), Total Occ = %u, Min Occ = %u, Max Occ = %u\n", cWid, cOccTotal, cOccMin, cOccMax );
if ( fDistribution ) printf( "! %d! %d\n", cWid, cOccTotal ); }
BitOffset boEnd; xDecomp->GetOffset( boEnd ); BitOffset boStart;
if ( fOccurrences ) for ( ULONG i = 0; i <= cOccMaxOverall; i++ ) { printf( "%I64u, ", aOccCounts[ i ] ); printf( "%I64u\n", acbOccCounts[ i ] / 8 ); }
if ( !fDistribution && !fKeys ) { printf( "cWidsWithOcc %I64u\n", cWidsWithOcc ); printf( "bytes in index: %I64u\n", BitDistance( boStart, boEnd ) / 8 );
printf( "max workid %d\n", maxWid ); printf( "cKeysInIndex: %d\n", cKeysInIndex ); printf( "cOccInIndex %I64u\n", cOccInIndex ); printf( "cWidsWithOneOcc %I64u\n", cWidsWithOneOcc ); printf( "cWidsWith5OrLessOcc %I64u\n", cWidsWith5OrLessOcc ); printf( "cKeysWithOneWid %d\n", cKeysWithOneWid ); printf( "cKeysInMoreThanHalfWids %d\n", cKeysInMoreThanHalfWids );
printf( "cOccMaxOverall %d\n", cOccMaxOverall ); printf( "cOccMinOverall %d\n", cOccMinOverall );
printf( "maximum key bytes %d\n", cbMaximumKey ); printf( "minimum key bytes %d\n", cbMinimumKey ); printf( "bytes for occ data: %I64u\n", ( cBitsForOcc / 8 ) ); printf( "bytes for key data: %I64u\n", ( cBitsForKey / 8 ) ); printf( "bytes for wid data: %I64u\n", ( cBitsForWid / 8 ) );
printf( "total for o+k+w data: %I64u\n", ( cBitsForOcc/8 + cBitsForKey/8 + cBitsForWid/8 ) ); } } else if ( !fKeys && !fDistribution && !fOccurrences ) { switch ( record.Type() ) { case itZombie: printf( "Zombie: " ); break;
case itDeleted: printf( "Deleted: " ); break;
case itPartition: printf( "Partition: " ); break;
case itChangeLog: printf( "ChangeLog: " ); break;
case itFreshLog: printf( "FreshLog: " ); break;
case itPhraseLat: printf( "Phrase Lat: " ); break;
case itKeyList: printf( "Key List: " ); break;
case itMMLog: printf( "MM Log: " ); break;
case itNewMaster: printf( "New Master: " ); break;
case itMMKeyList: printf( "MM Key List: " ); break; } // switch
printf( "Object id = 0x%x, Index id = 0x%x, MaxWorkid = 0x%x\n", record.ObjectId(), record.Iid(), record.MaxWorkId() ); } // if
} // while
} // if
else printf( "0 records found in index table\n" );
delete [] aOccCounts; delete [] acbOccCounts; } CATCH( CException, e ) { if ( HRESULT_FROM_WIN32(ERROR_SHARING_VIOLATION) == e.GetErrorCode() ) { printf("Sharing violation -- Is cisvc running?\n"); } else { printf( "Caught exception 0x%x in %s 0x%x\n", e.GetErrorCode(), ( recCrash.Type() == itMaster ) ? "master index" : "shadow index", recCrash.Iid() );
printf( "\tProcessing key: %ws (pid = 0x%x)\n", keyCrash.GetStr(), keyCrash.Pid() );
printf( "\tNear page %u (0x%x), bit offset %u (0x%x) --> File offset 0x%x\n", boffCrash.Page(), boffCrash.Page(), boffCrash.Offset(), boffCrash.Offset(), boffCrash.Page() * CI_PAGE_SIZE + (boffCrash.Offset() + 7)/8 ); } } END_CATCH
return 0;}
|
