/*++ Copyright (c) 1998 Microsoft Corporation Module Name: partctrl.cpp Abstract: SIS Groveler partition controller Authors: John Douceur, 1998 Environment: User Mode Revision History: --*/ #include "all.hxx" PartitionController::PartitionController( Groveler *groveler, GrovelStatus groveler_status, int target_entries_per_extraction, int max_extraction_interval, int base_grovel_interval, int max_grovel_interval, int low_confidence_grovel_interval, int low_disk_space_grovel_interval, int partition_info_update_interval, int base_restart_extraction_interval, int max_restart_extraction_interval, int base_restart_groveling_interval, int max_restart_groveling_interval, int base_regrovel_interval, int max_regrovel_interval, int volscan_regrovel_threshold, int partition_balance_time_constant, int read_time_increase_history_size, int read_time_decrease_history_size, int file_size_history_size, bool error_retry_log_extraction, bool error_retry_groveling, __int64 base_usn_log_size, __int64 max_usn_log_size, int sample_group_size, double acceptance_p_value, double rejection_p_value, double base_use_multiplier, double max_use_multiplier, double peak_finder_accuracy, double peak_finder_range, double base_cpu_load_threshold, double max_cpu_load_threshold, double *hash_match_ratio, double *compare_match_ratio, double *dequeue_hash_ratio, double *hash_read_time_estimate, double *compare_read_time_estimate, double *mean_file_size, double *read_time_confidence, int *volume_serial_number, int partition_index, double read_report_discard_threshold, int min_file_size, int min_file_age, bool allow_compressed_files, bool allow_encrypted_files, bool allow_hidden_files, bool allow_offline_files, bool allow_temporary_files, int num_excluded_paths, const _TCHAR **excluded_paths) : read_mean_comparator(2, sample_group_size, acceptance_p_value, rejection_p_value, peak_finder_accuracy), file_size_filter(file_size_history_size, *mean_file_size), read_time_confidence_estimator(2, *read_time_confidence), partition_grovel_accumulator(partition_balance_time_constant) { ASSERT(this != 0); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 1, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPC -\tconstructing PartitionController for drive %s\n"), sis_drives.partition_mount_name(partition_index))); ASSERT(groveler != 0); this->groveler = groveler; ASSERT(target_entries_per_extraction > 0); this->target_entries_per_extraction = target_entries_per_extraction; ASSERT(max_extraction_interval > 0); this->max_extraction_interval = max_extraction_interval; ASSERT(base_grovel_interval > 0); this->base_grovel_interval = base_grovel_interval; ASSERT(max_grovel_interval > 0); ASSERT(max_grovel_interval >= base_grovel_interval); this->max_grovel_interval = max_grovel_interval; ASSERT(partition_info_update_interval > 0); this->partition_info_update_interval = partition_info_update_interval; ASSERT(base_restart_extraction_interval > 0); this->base_restart_extraction_interval = base_restart_extraction_interval; ASSERT(max_restart_extraction_interval > 0); ASSERT(max_restart_extraction_interval >= base_restart_extraction_interval); this->max_restart_extraction_interval = max_restart_extraction_interval; ASSERT(base_restart_groveling_interval > 0); this->base_restart_groveling_interval = base_restart_groveling_interval; ASSERT(max_restart_groveling_interval > 0); ASSERT(max_restart_groveling_interval >= base_restart_groveling_interval); this->max_restart_groveling_interval = max_restart_groveling_interval; this->error_retry_log_extraction = error_retry_log_extraction; this->error_retry_groveling = error_retry_groveling; ASSERT(base_usn_log_size > 0); this->base_usn_log_size = base_usn_log_size; ASSERT(max_usn_log_size > 0); ASSERT(max_usn_log_size >= base_usn_log_size); this->max_usn_log_size = max_usn_log_size; ASSERT(hash_match_ratio != 0); ASSERT(*hash_match_ratio >= 0.0); ASSERT(*hash_match_ratio <= 1.0); this->hash_match_ratio = hash_match_ratio; ASSERT(compare_match_ratio != 0); ASSERT(*compare_match_ratio >= 0.0); ASSERT(*compare_match_ratio <= 1.0); this->compare_match_ratio = compare_match_ratio; ASSERT(dequeue_hash_ratio != 0); ASSERT(*dequeue_hash_ratio >= 0.0); ASSERT(*dequeue_hash_ratio <= 1.0); this->dequeue_hash_ratio = dequeue_hash_ratio; ASSERT(mean_file_size != 0); ASSERT(*mean_file_size >= 0.0); this->mean_file_size = mean_file_size; ASSERT(read_time_confidence != 0); ASSERT(*read_time_confidence >= 0.0); ASSERT(*read_time_confidence <= 1.0); this->read_time_confidence = read_time_confidence; ASSERT(base_use_multiplier > 0.0); this->base_use_multiplier = base_use_multiplier; ASSERT(partition_index >= 0); this->partition_index = partition_index; ASSERT(base_regrovel_interval > 0); this->base_regrovel_interval = base_regrovel_interval; ASSERT(max_regrovel_interval >= base_regrovel_interval); this->max_regrovel_interval = max_regrovel_interval; ASSERT(volscan_regrovel_threshold >= base_regrovel_interval); ASSERT(volscan_regrovel_threshold <= max_regrovel_interval); this->volscan_regrovel_threshold = volscan_regrovel_threshold; ASSERT(read_report_discard_threshold >= 0.0); ASSERT(read_report_discard_threshold <= 1.0); this->read_report_discard_threshold = read_report_discard_threshold; ASSERT(min_file_size >= 0); this->min_file_size = min_file_size; ASSERT(min_file_age >= 0); this->min_file_age = min_file_age; this->allow_compressed_files = allow_compressed_files; this->allow_encrypted_files = allow_encrypted_files; this->allow_hidden_files = allow_hidden_files; this->allow_offline_files = allow_offline_files; this->allow_temporary_files = allow_temporary_files; ASSERT(num_excluded_paths >= 0); this->num_excluded_paths = num_excluded_paths; ASSERT(excluded_paths != 0); this->excluded_paths = excluded_paths; ASSERT(peak_finder_accuracy > 0.0); ASSERT(peak_finder_accuracy <= 1.0); this->peak_finder_accuracy = peak_finder_accuracy; ASSERT(peak_finder_range >= 1.0); read_peak_finder[RT_hash] = new PeakFinder(peak_finder_accuracy, peak_finder_range); read_peak_finder[RT_compare] = new PeakFinder(peak_finder_accuracy, peak_finder_range); ASSERT(base_cpu_load_threshold >= 0.0); this->base_cpu_load_threshold = base_cpu_load_threshold; ASSERT(read_time_increase_history_size > 0); ASSERT(read_time_decrease_history_size > 0); ASSERT(hash_read_time_estimate != 0); ASSERT(*hash_read_time_estimate >= 0.0); read_time_filter[RT_hash] = new DirectedIncidentFilter(read_time_increase_history_size, read_time_decrease_history_size, *hash_read_time_estimate); ASSERT(compare_read_time_estimate != 0); ASSERT(*compare_read_time_estimate >= 0.0); read_time_filter[RT_compare] = new DirectedIncidentFilter(read_time_increase_history_size, read_time_decrease_history_size, *compare_read_time_estimate); read_time_estimate[RT_hash] = hash_read_time_estimate; read_time_estimate[RT_compare] = compare_read_time_estimate; log_max_grovel_interval = log(double(max_grovel_interval)); ASSERT(low_confidence_grovel_interval > 0); log_low_confidence_slope = log_max_grovel_interval - log(double(low_confidence_grovel_interval)); if (log_low_confidence_slope < 0.0) { log_low_confidence_slope = 0.0; } ASSERT(low_disk_space_grovel_interval > 0); low_disk_space_slope = max_grovel_interval - low_disk_space_grovel_interval; if (low_disk_space_slope < 0.0) { low_disk_space_slope = 0.0; } ASSERT(max_use_multiplier >= base_use_multiplier); use_multiplier_slope = max_use_multiplier - base_use_multiplier; ASSERT(max_cpu_load_threshold <= 1.0); ASSERT(max_cpu_load_threshold >= base_cpu_load_threshold); cpu_load_threshold_slope = max_cpu_load_threshold - base_cpu_load_threshold; ASSERT(volume_serial_number != 0); this->volume_serial_number = volume_serial_number; update_partition_info((void *)this); ASSERT(volume_total_bytes > 0.0); ASSERT(volume_free_bytes >= 0.0); current_usn_log_size = base_usn_log_size; restart_groveling_interval = base_restart_groveling_interval; remaining_grovel_interval = 0; restart_volume_scan = false; extended_restart_in_progress = false; initialize_groveling(groveler_status); log_extractor_dead = true; restart_extraction_interval = base_restart_extraction_interval; remaining_restart_extraction_interval = 0; restart_extraction((void *)this); } PartitionController::~PartitionController() { ASSERT(read_peak_finder[RT_hash] != 0); delete read_peak_finder[RT_hash]; read_peak_finder[RT_hash] = 0; ASSERT(read_peak_finder[RT_compare] != 0); delete read_peak_finder[RT_compare]; read_peak_finder[RT_hash] = 0; ASSERT(read_time_filter[RT_hash] != 0); delete read_time_filter[RT_hash]; read_time_filter[RT_hash] = 0; ASSERT(read_time_filter[RT_compare] != 0); delete read_time_filter[RT_compare]; read_time_filter[RT_compare] = 0; } bool PartitionController::control_operation( DWORD grovel_duration, DWORD *count_of_files_hashed, DWORDLONG *bytes_of_files_hashed, DWORD *count_of_files_matching, DWORDLONG *bytes_of_files_matching, DWORD *count_of_files_compared, DWORDLONG *bytes_of_files_compared, DWORD *count_of_files_merged, DWORDLONG *bytes_of_files_merged, DWORD *count_of_files_enqueued, DWORD *count_of_files_dequeued, double cpu_load) { ASSERT(this != 0); ASSERT(grovel_duration > 0); ASSERT(count_of_files_hashed != 0); ASSERT(bytes_of_files_hashed != 0); ASSERT(count_of_files_matching != 0); ASSERT(bytes_of_files_matching != 0); ASSERT(count_of_files_compared != 0); ASSERT(bytes_of_files_compared != 0); ASSERT(count_of_files_merged != 0); ASSERT(bytes_of_files_merged != 0); ASSERT(count_of_files_enqueued != 0); ASSERT(count_of_files_dequeued != 0); ASSERT(cpu_load >= 0.0); ASSERT(cpu_load <= 1.0); ASSERT(!groveler_dead); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCco -\toperating on drive %s\n"), sis_drives.partition_mount_name(partition_index))); int files_to_compare = groveler->count_of_files_to_compare(); ASSERT(files_to_compare >= 0); int files_in_queue = groveler->count_of_files_in_queue(); ASSERT(files_in_queue >= 0); int ready_time = groveler->time_to_first_file_ready(); ASSERT(files_in_queue == 0 || ready_time >= 0); bool more_work_to_do = files_to_compare > 0 || files_in_queue > 0 && ready_time < volscan_regrovel_threshold; if (log_extractor_dead && !performing_full_volume_scan && !more_work_to_do) { initiate_full_volume_scan = true; } partition_grovel_accumulator.increment(); ASSERT(groveler != 0); bool ok; if (initiate_full_volume_scan || performing_full_volume_scan && !more_work_to_do) { if (initiate_full_volume_scan) { TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCco -\tinitiating full volume scan\n"))); initiate_full_volume_scan = false; performing_full_volume_scan = true; restart_volume_scan = true; } TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCco -\tperforming full volume scan\n"))); ok = control_volume_scan(grovel_duration, count_of_files_enqueued); *count_of_files_hashed = 0; *bytes_of_files_hashed = 0; *count_of_files_matching = 0; *bytes_of_files_matching = 0; *count_of_files_compared = 0; *bytes_of_files_compared = 0; *count_of_files_merged = 0; *bytes_of_files_merged = 0; *count_of_files_dequeued = 0; } else { TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCco -\tgroveling\n"))); ok = control_groveling(grovel_duration, count_of_files_hashed, bytes_of_files_hashed, count_of_files_matching, bytes_of_files_matching, count_of_files_compared, bytes_of_files_compared, count_of_files_merged, bytes_of_files_merged, count_of_files_enqueued, count_of_files_dequeued, cpu_load); } if (groveler_dead) { TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCco -\tconcluding foreground batch for drive %s\n"), sis_drives.partition_mount_name(partition_index))); SERVICE_SET_FOREGROUND_BATCH_IN_PROGRESS(partition_index, false); } else { files_to_compare = groveler->count_of_files_to_compare(); ASSERT(files_to_compare >= 0); files_in_queue = groveler->count_of_files_in_queue(); ASSERT(files_in_queue >= 0); ready_time = groveler->time_to_first_file_ready(); ASSERT(files_in_queue == 0 || ready_time >= 0); more_work_to_do = files_to_compare > 0 || files_in_queue > 0 && ready_time < volscan_regrovel_threshold; if (!performing_full_volume_scan && !more_work_to_do) { TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCco -\tconcluding foreground batch for drive %s\n"), sis_drives.partition_mount_name(partition_index))); SERVICE_SET_FOREGROUND_BATCH_IN_PROGRESS(partition_index, false); } } return ok; } void PartitionController::advance( int time_delta) { ASSERT(this != 0); ASSERT(time_delta >= 0); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 4, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCa -\tadvancing time for drive %s by %d\n"), sis_drives.partition_mount_name(partition_index), time_delta)); if (groveler_dead) { return; } ASSERT(remaining_grovel_interval >= 0); remaining_grovel_interval -= time_delta; if (remaining_grovel_interval < 0) { remaining_grovel_interval = 0; } TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCa -\tremaining grovel interval = %d\n"), remaining_grovel_interval)); } double PartitionController::priority() const { ASSERT(this != 0); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 4, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCp -\tcalculating priority on drive %s\n"), sis_drives.partition_mount_name(partition_index))); if (groveler_dead) { return DBL_MAX; } double accumulated_groveling = partition_grovel_accumulator.retrieve_value(); ASSERT(accumulated_groveling >= 0.0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCp -\taccumulated groveling = %f\n"), accumulated_groveling)); double calculated_priority = (1.0 + accumulated_groveling) * (1.0 + volume_free_bytes); ASSERT(calculated_priority > 1.0); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCp -\tcalculated priority = %f\n"), calculated_priority)); return calculated_priority; } int PartitionController::wait() const { ASSERT(this != 0); ASSERT(groveler != 0); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 4, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCw -\tcalculating wait time for drive %s\n"), sis_drives.partition_mount_name(partition_index))); int time_until_groveler_ready = max_grovel_interval; if (!groveler_dead) { int files_to_compare = groveler->count_of_files_to_compare(); ASSERT(files_to_compare >= 0); int files_in_queue = groveler->count_of_files_in_queue(); ASSERT(files_in_queue >= 0); int ready_time = groveler->time_to_first_file_ready(); ASSERT(files_in_queue == 0 || ready_time >= 0); bool more_work_to_do = files_to_compare > 0 || files_in_queue > 0 && ready_time < volscan_regrovel_threshold; if (files_to_compare > 0 || initiate_full_volume_scan && !more_work_to_do || performing_full_volume_scan && !more_work_to_do || log_extractor_dead && !more_work_to_do) { time_until_groveler_ready = 0; TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCw -\tgroveler ready now\n"))); } else if (files_in_queue > 0) { time_until_groveler_ready = ready_time; ASSERT(time_until_groveler_ready >= 0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCw -\ttime until groveler ready = %d\n"), time_until_groveler_ready)); } } TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCw -\tremaining grovel interval = %d\n"), remaining_grovel_interval)); int wait_time = __max(remaining_grovel_interval, time_until_groveler_ready); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCw -\twait time = %d\n"), wait_time)); return wait_time; } void PartitionController::demarcate_foreground_batch() { ASSERT(this != 0); unsigned int current_time = GET_TICK_COUNT(); if (!groveler_dead) { TRACE_PRINTF(TC_partctrl, 1, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCdfb -\tdemarcating foreground batch for drive %s\n"), sis_drives.partition_mount_name(partition_index))); SERVICE_SET_FOREGROUND_BATCH_IN_PROGRESS(partition_index, true); } } void PartitionController::command_full_volume_scan() { ASSERT(this != 0); unsigned int current_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 1, (_T("time: %d\n"), current_time)); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCcfvs -\tcommanding full volume scan for drive %s\n"), sis_drives.partition_mount_name(partition_index))); initiate_full_volume_scan = true; } void PartitionController::control_extraction( void *context) { ASSERT(context != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), invokation_time)); PartitionController *me = (PartitionController *)context; TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCce -\textracting log on drive %s\n"), sis_drives.partition_mount_name(me->partition_index))); ASSERT(!me->log_extractor_dead); if (me->groveler_dead || me->restart_extraction_required) { TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCce -\trestarting extraction\n"))); me->log_extractor_dead = true; me->restart_extraction_interval = me->base_restart_extraction_interval; me->remaining_restart_extraction_interval = 0; restart_extraction(context); return; } DWORD num_entries_extracted; DWORDLONG num_bytes_extracted; DWORDLONG num_bytes_skipped; DWORD num_files_enqueued; DWORD num_files_dequeued; GrovelStatus status = me->groveler->extract_log(&num_entries_extracted, &num_bytes_extracted, &num_bytes_skipped, &num_files_enqueued, &num_files_dequeued); unsigned int completion_time = GET_TICK_COUNT(); if (status == Grovel_overrun) { ASSERT(signed(num_entries_extracted) >= 0); ASSERT(signed(num_bytes_extracted) >= 0); ASSERT(signed(num_bytes_skipped) >= 0); ASSERT(signed(num_files_enqueued) >= 0); ASSERT(signed(num_files_dequeued) >= 0); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCce -\textract_log() returned Grovel_overrun\n"))); me->initiate_full_volume_scan = true; eventlog.report_event(GROVMSG_USN_LOG_OVERRUN, ERROR_SUCCESS, 1, sis_drives.partition_mount_name(me->partition_index)); if (!me->first_extraction) { __int64 usn_log_size = num_bytes_extracted + num_bytes_skipped; ASSERT(me->base_usn_log_size > 0); ASSERT(me->max_usn_log_size > 0); ASSERT(me->current_usn_log_size >= me->base_usn_log_size); ASSERT(me->current_usn_log_size <= me->max_usn_log_size); if (usn_log_size > me->current_usn_log_size && me->current_usn_log_size < me->max_usn_log_size) { if (usn_log_size > me->max_usn_log_size) { usn_log_size = me->max_usn_log_size; } _TCHAR size_string[32]; (void)StringCbPrintf(size_string, sizeof(size_string), _T("%d"), usn_log_size); eventlog.report_event(GROVMSG_SET_USN_LOG_SIZE, ERROR_SUCCESS, 2, sis_drives.partition_mount_name(me->partition_index), size_string); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCce -\tincreasing USN log size from %d to %d\n"), me->current_usn_log_size, usn_log_size)); me->current_usn_log_size = usn_log_size; DWORD lstatus = me->groveler->set_usn_log_size(usn_log_size); if (lstatus != ERROR_SUCCESS) { TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCce -\tset_usn_log_size() returned error, status=%d\n"),lstatus)); TRACE_PRINTF(TC_partctrl, 1, (_T("\t\tsuspending control_extraction()\n"))); eventlog.report_event(GROVMSG_LOG_EXTRACTOR_DEAD, lstatus, 1,sis_drives.partition_mount_name(me->partition_index)); me->log_extractor_dead = true; me->restart_extraction_interval = me->base_restart_extraction_interval; me->remaining_restart_extraction_interval = me->restart_extraction_interval; event_timer.schedule( completion_time + me->max_extraction_interval, context, restart_extraction); return; } } } } else if (status != Grovel_ok) { ASSERT(status == Grovel_error); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCce -\textract_log() returned error\n"))); TRACE_PRINTF(TC_partctrl, 1, (_T("\t\tsuspending control_extraction()\n"))); eventlog.report_event(GROVMSG_LOG_EXTRACTOR_DEAD, GetLastError(), 1,sis_drives.partition_mount_name(me->partition_index)); me->log_extractor_dead = true; me->restart_extraction_interval = me->base_restart_extraction_interval; me->remaining_restart_extraction_interval = me->restart_extraction_interval; event_timer.schedule(completion_time + me->max_extraction_interval, context, restart_extraction); return; } unsigned int extraction_time = completion_time - invokation_time; ASSERT(signed(extraction_time) >= 0); shared_data->increment_value(me->partition_index, SDF_extract_time, extraction_time); shared_data->increment_value(me->partition_index, SDF_working_time, extraction_time); int queue_length = me->groveler->count_of_files_in_queue(); ASSERT(queue_length >= 0); shared_data->set_value(me->partition_index, SDF_queue_length, queue_length); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCce -\tnum entries extracted = %d\n"), num_entries_extracted)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCce -\tnum files enqueued = %d\n"), num_files_enqueued)); ASSERT(signed(num_entries_extracted) >= 0); ASSERT(signed(num_bytes_extracted) >= 0); ASSERT(signed(num_bytes_skipped) >= 0); ASSERT(signed(num_files_enqueued) >= 0); ASSERT(signed(num_files_dequeued) >= 0); ASSERT(num_bytes_extracted >= num_entries_extracted); ASSERT(num_files_enqueued <= num_entries_extracted); ASSERT(status == Grovel_overrun || num_bytes_skipped == 0); me->first_extraction = false; if (num_entries_extracted < 1) { num_entries_extracted = 1; } ASSERT(me->extraction_interval > 0); ASSERT(me->target_entries_per_extraction > 0); int ideal_extraction_interval = me->extraction_interval * me->target_entries_per_extraction / num_entries_extracted + 1; ASSERT(ideal_extraction_interval > 0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCce -\tideal extraction interval = %d\n"), ideal_extraction_interval)); if (ideal_extraction_interval < me->extraction_interval) { me->extraction_interval = ideal_extraction_interval; } else { me->extraction_interval = int(sqrt(double(me->extraction_interval) * double(ideal_extraction_interval))); ASSERT(me->max_extraction_interval > 0); if (me->extraction_interval > me->max_extraction_interval) { me->extraction_interval = me->max_extraction_interval; } } TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCce -\textraction interval = %d\n"), me->extraction_interval)); ASSERT(me->extraction_interval > 0); ASSERT(me->extraction_interval <= me->max_extraction_interval); event_timer.schedule(invokation_time + me->extraction_interval, context, control_extraction); } void PartitionController::restart_extraction( void *context) { ASSERT(context != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), invokation_time)); PartitionController *me = (PartitionController *)context; TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCre -\tconsidering restart extraction on drive %s\n"), sis_drives.partition_mount_name(me->partition_index))); ASSERT(me->log_extractor_dead); me->remaining_restart_extraction_interval -= __min(me->remaining_restart_extraction_interval, me->max_extraction_interval); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCre -\tremaining restart extraction interval = %d\n"), me->remaining_restart_extraction_interval)); if (!me->groveler_dead && me->remaining_restart_extraction_interval == 0) { TRACE_PRINTF(TC_partctrl, 2, (_T("time: %d\n"), invokation_time)); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCre -\tattempting restart extraction on drive %s\n"), sis_drives.partition_mount_name(me->partition_index))); // Disabled this message since it serves no purpose, nealch 4/17/02 // eventlog.report_event(GROVMSG_USN_LOG_RETRY, ERROR_SUCCESS, // 1,sis_drives.partition_mount_name(me->partition_index)); DWORD lstatus = me->groveler->set_usn_log_size(me->current_usn_log_size); if (lstatus == ERROR_SUCCESS) { TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCre -\tset_usn_log_size() returned success\n"))); _TCHAR size_string[32]; (void)StringCbPrintf(size_string, sizeof(size_string), _T("%d"), me->current_usn_log_size); eventlog.report_event(GROVMSG_INIT_USN_LOG, ERROR_SUCCESS, 2, sis_drives.partition_mount_name(me->partition_index), size_string); me->restart_extraction_required = false; me->log_extractor_dead = false; me->first_extraction = true; control_extraction(context); return; } else { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCre -\tset_usn_log_size() returned error, status=%d\n"),lstatus)); ASSERT(lstatus != ERROR_SUCCESS); me->remaining_restart_extraction_interval = me->restart_extraction_interval; TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCre -\tremaining restart extraction interval = %d\n"), me->remaining_restart_extraction_interval)); me->restart_extraction_interval *= 2; if (me->restart_extraction_interval > me->max_restart_extraction_interval) { me->restart_extraction_interval = me->max_restart_extraction_interval; } TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCre -\tnext restart extraction interval = %d\n"), me->restart_extraction_interval)); } } event_timer.schedule(invokation_time + me->max_extraction_interval, context, restart_extraction); } void PartitionController::restart_groveling( void *context) { ASSERT(context != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 2, (_T("time: %d\n"), invokation_time)); PartitionController *me = (PartitionController *)context; TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCrg -\tattempting to restart groveler on drive %s\n"), sis_drives.partition_mount_name(me->partition_index))); ASSERT(me->groveler_dead); me->groveler->close(); eventlog.report_event(GROVMSG_GROVELER_RETRY, ERROR_SUCCESS, 1,sis_drives.partition_mount_name(me->partition_index)); GrovelStatus status = me->groveler->open( sis_drives.partition_guid_name(me->partition_index), sis_drives.partition_mount_name(me->partition_index), me->partition_index == log_drive->drive_index(), me->read_report_discard_threshold, me->min_file_size, me->min_file_age, me->allow_compressed_files, me->allow_encrypted_files, me->allow_hidden_files, me->allow_offline_files, me->allow_temporary_files, GrovelAllPaths, me->num_excluded_paths, me->excluded_paths, me->base_regrovel_interval, me->max_regrovel_interval); if (status == Grovel_ok) { TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCrg -\topen() returned ok\n"))); log_drive->partition_initialized(me->partition_index); eventlog.report_event(GROVMSG_GROVELER_STARTED, ERROR_SUCCESS, 1,sis_drives.partition_mount_name(me->partition_index)); } else if (status == Grovel_new) { TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCrg -\topen() returned new\n"))); } else { ASSERT(status == Grovel_error); TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCrg -\topen() returned error\n"))); } me->restart_volume_scan = true; me->initialize_groveling(status); } void PartitionController::update_partition_info( void *context) { ASSERT(context != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), invokation_time)); PartitionController *me = (PartitionController *)context; TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCupi -\tupdating partition info for drive %s\n"), sis_drives.partition_mount_name(me->partition_index))); ULARGE_INTEGER my_free_bytes; ULARGE_INTEGER total_bytes; ULARGE_INTEGER free_bytes; int ok = GetDiskFreeSpaceEx( sis_drives.partition_guid_name(me->partition_index), &my_free_bytes, &total_bytes, &free_bytes); if (ok) { me->volume_total_bytes = double(__int64(total_bytes.QuadPart)); me->volume_free_bytes = double(__int64(free_bytes.QuadPart)); ASSERT(me->volume_total_bytes > 0.0); ASSERT(me->volume_free_bytes >= 0.0); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCupi -\tvolume total bytes = %f\n"), me->volume_total_bytes)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCupi -\tvolume free bytes = %f\n"), me->volume_free_bytes)); } else { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCupi -\tGetDiskFreeSpaceEx() returned error.\n"))); DWORD err = GetLastError(); PRINT_DEBUG_MSG((_T("GROVELER: GetDiskFreeSpaceEx() failed with error %d\n"), err)); } ASSERT(me->partition_info_update_interval > 0); event_timer.schedule(invokation_time + me->partition_info_update_interval, context, update_partition_info); } void PartitionController::initialize_groveling( GrovelStatus groveler_status) { ASSERT(this != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), invokation_time)); TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCig -\tinitializing groveling for drive %s\n"), sis_drives.partition_mount_name(partition_index))); if (groveler_status == Grovel_ok || groveler_status == Grovel_new) { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCig -\tgroveler_status indicates success\n"))); groveler_dead = false; DWORD serial_number; int ok = GetVolumeInformation( sis_drives.partition_guid_name(partition_index), 0, 0, &serial_number, 0, 0, 0, 0); if (!ok) { DWORD err = GetLastError(); TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCig -\tGetVolumeInformation() returned error %d\n"), err)); PRINT_DEBUG_MSG((_T("GROVELER: GetVolumeInformation() failed with error %d\n"), err)); } else { TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\tGetVolumeInformation() returned ") _T("serial number %d\n"), serial_number)); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\trecorded serial number is %d\n"), *volume_serial_number)); if (volume_serial_number == 0) { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCig -\tGetVolumeInformation() returned ") _T("serial number 0\n"))); PRINT_DEBUG_MSG((_T("GROVELER: GetVolumeInformation() returned ") _T("volume serial number 0\n"))); } } if (ok && int(serial_number) != *volume_serial_number) { TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\tresetting read time filters\n"))); read_time_filter[RT_hash]->reset(); read_time_filter[RT_compare]->reset(); read_time_confidence_estimator.reset(); *read_time_estimate[RT_hash] = read_time_filter[RT_hash]->retrieve_value(); ASSERT(*read_time_estimate[RT_hash] == 0.0); *read_time_estimate[RT_compare] = read_time_filter[RT_compare]->retrieve_value(); ASSERT(*read_time_estimate[RT_compare] == 0.0); *read_time_confidence = read_time_confidence_estimator.confidence(); ASSERT(*read_time_confidence == 0.0); *volume_serial_number = serial_number; } extraction_interval = max_extraction_interval; free_space_ratio = 0.0; calculate_effective_max_grovel_interval(); ASSERT(effective_max_grovel_interval > 0); ASSERT(effective_max_grovel_interval <= max_grovel_interval); grovel_interval = base_grovel_interval; remaining_grovel_interval = grovel_interval; ok_to_record_measurement = true; next_untrusted_measurement_time = invokation_time; restart_extraction_required = true; if (groveler_status == Grovel_new) { TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCig -\tinitiating full volume scan\n"))); initiate_full_volume_scan = true; performing_full_volume_scan = false; extended_restart_in_progress = true; } else { TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCig -\tcontinuing full volume scan\n"))); initiate_full_volume_scan = false; performing_full_volume_scan = true; extended_restart_in_progress = false; restart_groveling_interval = base_restart_groveling_interval; } TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\trestart groveling interval = %d\n"), restart_groveling_interval)); } else { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCig -\tgroveler_status indicates error or disable\n"))); ASSERT(groveler_status == Grovel_error || groveler_status == Grovel_disable); groveler_dead = true; if (groveler_status != Grovel_disable && error_retry_groveling) { TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\tscheduling restart groveling\n"))); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\trestart groveling interval = %d\n"), restart_groveling_interval)); event_timer.schedule(invokation_time + restart_groveling_interval, (void *)this, restart_groveling); } restart_groveling_interval *= 2; if (restart_groveling_interval > max_restart_groveling_interval) { restart_groveling_interval = max_restart_groveling_interval; } TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCig -\tnext restart groveling interval = %d\n"), restart_groveling_interval)); } } bool PartitionController::control_groveling( DWORD grovel_duration, DWORD *count_of_files_hashed, DWORDLONG *bytes_of_files_hashed, DWORD *count_of_files_matching, DWORDLONG *bytes_of_files_matching, DWORD *count_of_files_compared, DWORDLONG *bytes_of_files_compared, DWORD *count_of_files_merged, DWORDLONG *bytes_of_files_merged, DWORD *count_of_files_enqueued, DWORD *count_of_files_dequeued, double cpu_load) { ASSERT(this != 0); ASSERT(grovel_duration > 0); ASSERT(count_of_files_hashed != 0); ASSERT(bytes_of_files_hashed != 0); ASSERT(count_of_files_matching != 0); ASSERT(bytes_of_files_matching != 0); ASSERT(count_of_files_compared != 0); ASSERT(bytes_of_files_compared != 0); ASSERT(count_of_files_merged != 0); ASSERT(bytes_of_files_merged != 0); ASSERT(count_of_files_enqueued != 0); ASSERT(count_of_files_dequeued != 0); ASSERT(cpu_load >= 0.0); ASSERT(cpu_load <= 1.0); ASSERT(!groveler_dead); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 2, (_T("time: %d\n"), invokation_time)); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCcg -\tgroveling on drive %s\n"), sis_drives.partition_mount_name(partition_index))); DWORD hash_read_ops; DWORD hash_read_time; DWORD compare_read_ops; DWORD compare_read_time; DWORD merge_time; GrovelStatus status = groveler->grovel(grovel_duration, &hash_read_ops, &hash_read_time, count_of_files_hashed, bytes_of_files_hashed, &compare_read_ops, &compare_read_time, count_of_files_compared, bytes_of_files_compared, count_of_files_matching, bytes_of_files_matching, &merge_time, count_of_files_merged, bytes_of_files_merged, count_of_files_enqueued, count_of_files_dequeued); unsigned int completion_time = GET_TICK_COUNT(); if (status != Grovel_ok && status != Grovel_pending) { ASSERT(status == Grovel_error); *count_of_files_hashed = 0; *bytes_of_files_hashed = 0; *count_of_files_matching = 0; *bytes_of_files_matching = 0; *count_of_files_compared = 0; *bytes_of_files_compared = 0; *count_of_files_merged = 0; *bytes_of_files_merged = 0; *count_of_files_enqueued = 0; *count_of_files_dequeued = 0; TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCcg -\tgrovel() returned error -- groveler dead\n"))); eventlog.report_event(GROVMSG_GROVELER_DEAD, ERROR_SUCCESS, 1,sis_drives.partition_mount_name(partition_index)); groveler_dead = true; if (error_retry_groveling) { restart_groveling_interval = base_restart_groveling_interval; event_timer.schedule(invokation_time + restart_groveling_interval, (void *)this, restart_groveling); return true; } else { return false; } } ASSERT(hash_read_ops > 0 || hash_read_time == 0); ASSERT(compare_read_ops > 0 || compare_read_time == 0); ASSERT(*bytes_of_files_hashed >= *count_of_files_hashed); ASSERT(*bytes_of_files_matching >= *count_of_files_matching); ASSERT(*bytes_of_files_compared >= *count_of_files_compared); ASSERT(*bytes_of_files_merged >= *count_of_files_merged); ASSERT(*count_of_files_hashed >= *count_of_files_matching); ASSERT(*bytes_of_files_hashed >= *bytes_of_files_matching); ASSERT(*count_of_files_compared >= *count_of_files_merged); ASSERT(*bytes_of_files_compared >= *bytes_of_files_merged); ASSERT(*count_of_files_dequeued >= *count_of_files_hashed); unsigned int grovel_time = completion_time - invokation_time; ASSERT(signed(grovel_time) >= 0); shared_data->increment_value(partition_index, SDF_grovel_time, grovel_time); shared_data->increment_value(partition_index, SDF_working_time, grovel_time); shared_data->increment_value(partition_index, SDF_files_hashed, *count_of_files_hashed); shared_data->increment_value(partition_index, SDF_files_compared, *count_of_files_compared); shared_data->increment_value(partition_index, SDF_files_merged, *count_of_files_merged); int files_in_queue = groveler->count_of_files_in_queue(); ASSERT(files_in_queue >= 0); int files_to_compare = groveler->count_of_files_to_compare(); ASSERT(files_to_compare >= 0); shared_data->set_value(partition_index, SDF_queue_length, files_in_queue); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\thash read ops = %d\n"), hash_read_ops)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\thash read time = %d\n"), hash_read_time)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\tcompare read ops = %d\n"), compare_read_ops)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\tcompare read time = %d\n"), compare_read_time)); shared_data->increment_value(partition_index, SDF_hash_read_time, hash_read_time); shared_data->increment_value(partition_index, SDF_hash_read_ops, hash_read_ops); shared_data->increment_value(partition_index, SDF_compare_read_time, compare_read_time); shared_data->increment_value(partition_index, SDF_compare_read_ops, compare_read_ops); update_peak_finder(RT_hash, hash_read_time, hash_read_ops); update_peak_finder(RT_compare, compare_read_time, compare_read_ops); shared_data->set_value(partition_index, SDF_hash_read_estimate, __int64(*read_time_estimate[RT_hash])); shared_data->set_value(partition_index, SDF_compare_read_estimate, __int64(*read_time_estimate[RT_compare])); int count_of_files_groveled = *count_of_files_hashed + *count_of_files_compared; ASSERT(mean_file_size != 0); if (count_of_files_groveled > 0) { __int64 bytes_of_files_groveled = *bytes_of_files_hashed + *bytes_of_files_compared; double sample_mean_file_size = double(bytes_of_files_groveled) / double(count_of_files_groveled); ASSERT(sample_mean_file_size > 0.0); file_size_filter.update_value(sample_mean_file_size, count_of_files_groveled); *mean_file_size = file_size_filter.retrieve_value(); ASSERT(*mean_file_size > 0.0); } ASSERT(dequeue_hash_ratio != 0); ASSERT(*dequeue_hash_ratio >= 0.0); ASSERT(*dequeue_hash_ratio <= 1.0); double files_to_hash = *dequeue_hash_ratio * double(files_in_queue); ASSERT(*mean_file_size >= 0.0); double bytes_to_hash = *mean_file_size * files_to_hash; double bytes_to_compare = *mean_file_size * double(files_to_compare); double expected_bytes_to_free = *compare_match_ratio * (bytes_to_compare + *hash_match_ratio * bytes_to_hash); double expected_free_bytes = volume_free_bytes + expected_bytes_to_free; free_space_ratio = 0; if (expected_free_bytes > 0) { free_space_ratio = expected_bytes_to_free / expected_free_bytes; } ASSERT(free_space_ratio >= 0.0); ASSERT(free_space_ratio <= 1.0); calculate_effective_max_grovel_interval(); ASSERT(effective_max_grovel_interval > 0); ASSERT(effective_max_grovel_interval <= max_grovel_interval); double use_multiplier = base_use_multiplier + use_multiplier_slope * free_space_ratio; ASSERT(use_multiplier >= 0.0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcg -\tuse multiplier = %f\n"), use_multiplier)); double hash_comparison_time = use_multiplier * *read_time_estimate[RT_hash]; ASSERT(hash_comparison_time >= 0.0); double compare_comparison_time = use_multiplier * *read_time_estimate[RT_compare]; ASSERT(compare_comparison_time >= 0.0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcg -\thash comparison time = %f\n"), hash_comparison_time)); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcg -\tcompare comparison time = %f\n"), compare_comparison_time)); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcg -\tcpu load = %f\n"), cpu_load)); double cpu_load_threshold = base_cpu_load_threshold + cpu_load_threshold_slope * free_space_ratio; TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcg -\tcpu load threshold = %f\n"), cpu_load_threshold)); ASSERT(cpu_load_threshold >= 0.0); ASSERT(cpu_load_threshold <= 1.0); if (cpu_load > cpu_load_threshold || read_mean_comparator.exceeds( hash_comparison_time, compare_comparison_time)) { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCcg -\tread time exceeds acceptable bounds\n"))); TRACE_PRINTF(TC_partctrl, 3, (_T("\t\tor CPU load exceeds threshold\n"))); ASSERT(grovel_interval > 0); remaining_grovel_interval = grovel_interval; read_mean_comparator.reset(); grovel_interval *= 2; ok_to_record_measurement = false; } else if (read_mean_comparator.within(hash_comparison_time, compare_comparison_time)) { TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCcg -\tread time within acceptable bounds\n"))); ASSERT(base_grovel_interval > 0); grovel_interval = base_grovel_interval; ok_to_record_measurement = true; } if (grovel_interval > effective_max_grovel_interval) { ASSERT(effective_max_grovel_interval > 0); grovel_interval = effective_max_grovel_interval; } TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\tgrovel interval = %d\n"), grovel_interval)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcg -\tremaining grovel interval = %d\n"), remaining_grovel_interval)); return true; } bool PartitionController::control_volume_scan( int scan_duration, DWORD *count_of_files_enqueued) { ASSERT(this != 0); ASSERT(scan_duration > 0); ASSERT(count_of_files_enqueued != 0); ASSERT(!groveler_dead); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 2, (_T("time: %d\n"), invokation_time)); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCcvs -\tscanning volume on drive %s\n"), sis_drives.partition_mount_name(partition_index))); DWORD time_consumed; DWORD findfirst_count; DWORD findnext_count; GrovelStatus status = groveler->scan_volume(scan_duration, restart_volume_scan, &time_consumed, &findfirst_count, &findnext_count, count_of_files_enqueued); unsigned int completion_time = GET_TICK_COUNT(); if (status == Grovel_ok || status == Grovel_pending) { if (extended_restart_in_progress) { log_drive->partition_initialized(partition_index); eventlog.report_event(GROVMSG_GROVELER_STARTED, ERROR_SUCCESS, 1,sis_drives.partition_mount_name(partition_index)); extended_restart_in_progress = false; } if (status == Grovel_ok) { TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCcvs -\tcompleted volume scan\n"))); performing_full_volume_scan = false; } } else { ASSERT(status == Grovel_error); *count_of_files_enqueued = 0; TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCcvs -\tscan_volume() returned error -- groveler dead\n"))); if (!extended_restart_in_progress) { eventlog.report_event(GROVMSG_GROVELER_DEAD, ERROR_SUCCESS, 1,sis_drives.partition_mount_name(partition_index)); } groveler_dead = true; if (error_retry_groveling) { if (!extended_restart_in_progress) { restart_groveling_interval = base_restart_groveling_interval; } event_timer.schedule(invokation_time + restart_groveling_interval, (void *)this, restart_groveling); if (extended_restart_in_progress) { restart_groveling_interval *= 2; if (restart_groveling_interval > max_restart_groveling_interval) { restart_groveling_interval = max_restart_groveling_interval; } extended_restart_in_progress = false; } return true; } else { extended_restart_in_progress = false; return false; } } ASSERT(signed(time_consumed) >= 0); ASSERT(signed(findfirst_count) >= 0); ASSERT(signed(findnext_count) >= 0); ASSERT(signed(*count_of_files_enqueued) >= 0); unsigned int scan_time = completion_time - invokation_time; ASSERT(signed(scan_time) >= 0); shared_data->increment_value(partition_index, SDF_volscan_time, scan_time); shared_data->increment_value(partition_index, SDF_working_time, scan_time); shared_data->increment_value(partition_index, SDF_files_scanned, findfirst_count + findnext_count); int queue_length = groveler->count_of_files_in_queue(); ASSERT(queue_length >= 0); shared_data->set_value(partition_index, SDF_queue_length, queue_length); restart_volume_scan = false; TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcvs -\ttime consumed = %d\n"), time_consumed)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcvs -\tfindfirst count = %d\n"), findfirst_count)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcvs -\tfindnext count = %d\n"), findnext_count)); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcvs -\tcount of files enqueued = %d\n"), *count_of_files_enqueued)); return true; } void PartitionController::update_peak_finder( ReadType read_type, DWORD read_time, DWORD read_ops) { ASSERT(this != 0); unsigned int invokation_time = GET_TICK_COUNT(); TRACE_PRINTF(TC_partctrl, 3, (_T("time: %d\n"), invokation_time)); TRACE_PRINTF(TC_partctrl, 3, (_T("\tPCupf -\tupdating peak finder for drive %s\n"), sis_drives.partition_mount_name(partition_index))); ASSERT(read_type == RT_hash || read_type == RT_compare); ASSERT(signed(read_time) >= 0); ASSERT(signed(read_ops) >= 0); if (read_ops > 0) { double time_per_read = double(read_time)/double(read_ops); ASSERT(time_per_read >= 0.0); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCupf -\ttime per %s read = %f\n"), (read_type == RT_hash ? _T("hash") : _T("compare")), time_per_read)); read_mean_comparator.sample(read_type, time_per_read); ASSERT(read_peak_finder[read_type] != 0); ASSERT(read_time_filter[read_type] != 0); ASSERT(read_time_estimate[read_type] != 0); ASSERT(read_time_confidence != 0); if (ok_to_record_measurement || signed(invokation_time - next_untrusted_measurement_time) >= 0) { TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCupf -\trecording %s measurement for drive %s\n"), (read_type == RT_hash ? _T("hash") : _T("compare")), sis_drives.partition_mount_name(partition_index))); read_peak_finder[read_type]->sample(time_per_read, read_ops); ASSERT(untrusted_measurement_interval > 0); ok_to_record_measurement = true; next_untrusted_measurement_time = invokation_time + untrusted_measurement_interval; TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCupf -\tnext untrusted measurement time = %d\n"), next_untrusted_measurement_time)); } if (read_peak_finder[read_type]->found()) { double peak = read_peak_finder[read_type]->median(); TRACE_PRINTF(TC_partctrl, 1, (_T("\tPCupf -\t%s read peak found: %f\n"), (read_type == RT_hash ? _T("hash") : _T("compare")), peak)); if (peak > 0.0) { read_time_filter[read_type]->update_value(peak); } else { PRINT_DEBUG_MSG((_T("GROVELER: update_peak_finder() peak finder returned peak of zero\n"))); } read_peak_finder[read_type]->reset(); *read_time_estimate[read_type] = read_time_filter[read_type]->retrieve_value(); ASSERT(*read_time_estimate[read_type] > 0.0); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCupf -\t%s read time estimate = %f\n"), (read_type == RT_hash ? _T("hash") : _T("compare")), *read_time_estimate[read_type])); double sample_peak_ratio = *read_time_estimate[read_type] / peak; double sample_read_time_confidence = (sample_peak_ratio + peak_finder_accuracy - 1.0) / peak_finder_accuracy; TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCupf -\tsample read time confidence = %f\n"), sample_read_time_confidence)); read_time_confidence_estimator.update(read_type, sample_read_time_confidence); *read_time_confidence = read_time_confidence_estimator.confidence(); ASSERT(*read_time_confidence >= 0.0); ASSERT(*read_time_confidence <= 1.0); TRACE_PRINTF(TC_partctrl, 2, (_T("\tPCupf -\tread time confidence = %f\n"), *read_time_confidence)); calculate_effective_max_grovel_interval(); ASSERT(effective_max_grovel_interval > 0); ASSERT(effective_max_grovel_interval <= max_grovel_interval); } } } void PartitionController::calculate_effective_max_grovel_interval() { ASSERT(this != 0); ASSERT(read_time_confidence != 0); ASSERT(*read_time_confidence >= 0.0); ASSERT(*read_time_confidence <= 1.0); TRACE_PRINTF(TC_partctrl, 4, (_T("\tPCcemgi -\tread time confidence = %f\n"), *read_time_confidence)); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcemgi -\tfree space ratio = %f\n"), free_space_ratio)); double log_untrusted_measurement_interval = log_max_grovel_interval - (1.0 - *read_time_confidence) * log_low_confidence_slope; untrusted_measurement_interval = int(exp(log_untrusted_measurement_interval)); ASSERT(untrusted_measurement_interval > 0); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcemgi -\tuntrusted measurement interval = %d\n"), untrusted_measurement_interval)); int low_disk_space_interval = max_grovel_interval - int(free_space_ratio * low_disk_space_slope); ASSERT(low_disk_space_interval > 0); effective_max_grovel_interval = __min(untrusted_measurement_interval, low_disk_space_interval); ASSERT(effective_max_grovel_interval > 0); ASSERT(effective_max_grovel_interval <= max_grovel_interval); TRACE_PRINTF(TC_partctrl, 5, (_T("\tPCcemgi -\teffective max grovel interval = %d\n"), effective_max_grovel_interval)); }