use std::path::Path;
use crate::stats::bivariate::regression::Slope;
use crate::stats::bivariate::Data;
use crate::stats::univariate::outliers::tukey;
use crate::stats::univariate::Sample;
use crate::stats::{Distribution, Tails};
use crate::benchmark::BenchmarkConfig;
use crate::connection::OutgoingMessage;
use crate::estimate::{
build_estimates, ConfidenceInterval, Distributions, Estimate, Estimates, PointEstimates,
};
use crate::fs;
use crate::measurement::Measurement;
use crate::report::{BenchmarkId, Report, ReportContext};
use crate::routine::Routine;
use crate::{Baseline, Criterion, SavedSample, Throughput};
macro_rules! elapsed {
($msg:expr, $block:expr) => {{
let start = ::std::time::Instant::now();
let out = $block;
let elapsed = &start.elapsed();
info!(
"{} took {}",
$msg,
crate::format::time(elapsed.as_nanos() as f64)
);
out
}};
}
mod compare;
pub(crate) fn common<M: Measurement, T: ?Sized>(
id: &BenchmarkId,
routine: &mut dyn Routine<M, T>,
config: &BenchmarkConfig,
criterion: &Criterion<M>,
report_context: &ReportContext,
parameter: &T,
throughput: Option<Throughput>,
) {
criterion.report.benchmark_start(id, report_context);
if let Baseline::CompareStrict = criterion.baseline {
if !base_dir_exists(
id,
&criterion.baseline_directory,
&criterion.output_directory,
) {
panic!(
"Baseline '{base}' must exist before comparison is allowed; try --save-baseline {base}",
base=criterion.baseline_directory,
);
}
}
let (sampling_mode, iters, times);
if let Some(baseline) = &criterion.load_baseline {
let mut sample_path = criterion.output_directory.clone();
sample_path.push(id.as_directory_name());
sample_path.push(baseline);
sample_path.push("sample.json");
let loaded = fs::load::<SavedSample, _>(&sample_path);
match loaded {
Err(err) => panic!(
"Baseline '{base}' must exist before it can be loaded; try --save-baseline {base}. Error: {err}",
base = baseline, err = err
),
Ok(samples) => {
sampling_mode = samples.sampling_mode;
iters = samples.iters.into_boxed_slice();
times = samples.times.into_boxed_slice();
}
}
} else {
let sample = routine.sample(
&criterion.measurement,
id,
config,
criterion,
report_context,
parameter,
);
sampling_mode = sample.0;
iters = sample.1;
times = sample.2;
if let Some(conn) = &criterion.connection {
conn.send(&OutgoingMessage::MeasurementComplete {
id: id.into(),
iters: &iters,
times: ×,
plot_config: (&report_context.plot_config).into(),
sampling_method: sampling_mode.into(),
benchmark_config: config.into(),
})
.unwrap();
conn.serve_value_formatter(criterion.measurement.formatter())
.unwrap();
return;
}
}
criterion.report.analysis(id, report_context);
if times.iter().any(|&f| f == 0.0) {
error!(
"At least one measurement of benchmark {} took zero time per \
iteration. This should not be possible. If using iter_custom, please verify \
that your routine is correctly measured.",
id.as_title()
);
return;
}
let avg_times = iters
.iter()
.zip(times.iter())
.map(|(&iters, &elapsed)| elapsed / iters)
.collect::<Vec<f64>>();
let avg_times = Sample::new(&avg_times);
if criterion.should_save_baseline() {
log_if_err!({
let mut new_dir = criterion.output_directory.clone();
new_dir.push(id.as_directory_name());
new_dir.push("new");
fs::mkdirp(&new_dir)
});
}
let data = Data::new(&iters, ×);
let labeled_sample = tukey::classify(avg_times);
if criterion.should_save_baseline() {
log_if_err!({
let mut tukey_file = criterion.output_directory.to_owned();
tukey_file.push(id.as_directory_name());
tukey_file.push("new");
tukey_file.push("tukey.json");
fs::save(&labeled_sample.fences(), &tukey_file)
});
}
let (mut distributions, mut estimates) = estimates(avg_times, config);
if sampling_mode.is_linear() {
let (distribution, slope) = regression(&data, config);
estimates.slope = Some(slope);
distributions.slope = Some(distribution);
}
if criterion.should_save_baseline() {
log_if_err!({
let mut sample_file = criterion.output_directory.clone();
sample_file.push(id.as_directory_name());
sample_file.push("new");
sample_file.push("sample.json");
fs::save(
&SavedSample {
sampling_mode,
iters: data.x().as_ref().to_vec(),
times: data.y().as_ref().to_vec(),
},
&sample_file,
)
});
log_if_err!({
let mut estimates_file = criterion.output_directory.clone();
estimates_file.push(id.as_directory_name());
estimates_file.push("new");
estimates_file.push("estimates.json");
fs::save(&estimates, &estimates_file)
});
}
let compare_data = if base_dir_exists(
id,
&criterion.baseline_directory,
&criterion.output_directory,
) {
let result = compare::common(id, avg_times, config, criterion);
match result {
Ok((
t_value,
t_distribution,
relative_estimates,
relative_distributions,
base_iter_counts,
base_sample_times,
base_avg_times,
base_estimates,
)) => {
let p_value = t_distribution.p_value(t_value, &Tails::Two);
Some(crate::report::ComparisonData {
p_value,
t_distribution,
t_value,
relative_estimates,
relative_distributions,
significance_threshold: config.significance_level,
noise_threshold: config.noise_threshold,
base_iter_counts,
base_sample_times,
base_avg_times,
base_estimates,
})
}
Err(e) => {
crate::error::log_error(&e);
None
}
}
} else {
None
};
let measurement_data = crate::report::MeasurementData {
data: Data::new(&iters, ×),
avg_times: labeled_sample,
absolute_estimates: estimates,
distributions,
comparison: compare_data,
throughput,
};
criterion.report.measurement_complete(
id,
report_context,
&measurement_data,
criterion.measurement.formatter(),
);
if criterion.should_save_baseline() {
log_if_err!({
let mut benchmark_file = criterion.output_directory.clone();
benchmark_file.push(id.as_directory_name());
benchmark_file.push("new");
benchmark_file.push("benchmark.json");
fs::save(&id, &benchmark_file)
});
}
if criterion.connection.is_none() {
if let Baseline::Save = criterion.baseline {
copy_new_dir_to_base(
id.as_directory_name(),
&criterion.baseline_directory,
&criterion.output_directory,
);
}
}
}
fn base_dir_exists(id: &BenchmarkId, baseline: &str, output_directory: &Path) -> bool {
let mut base_dir = output_directory.to_owned();
base_dir.push(id.as_directory_name());
base_dir.push(baseline);
base_dir.exists()
}
fn regression(
data: &Data<'_, f64, f64>,
config: &BenchmarkConfig,
) -> (Distribution<f64>, Estimate) {
let cl = config.confidence_level;
let distribution = elapsed!(
"Bootstrapped linear regression",
data.bootstrap(config.nresamples, |d| (Slope::fit(&d).0,))
)
.0;
let point = Slope::fit(data);
let (lb, ub) = distribution.confidence_interval(config.confidence_level);
let se = distribution.std_dev(None);
(
distribution,
Estimate {
confidence_interval: ConfidenceInterval {
confidence_level: cl,
lower_bound: lb,
upper_bound: ub,
},
point_estimate: point.0,
standard_error: se,
},
)
}
fn estimates(avg_times: &Sample<f64>, config: &BenchmarkConfig) -> (Distributions, Estimates) {
fn stats(sample: &Sample<f64>) -> (f64, f64, f64, f64) {
let mean = sample.mean();
let std_dev = sample.std_dev(Some(mean));
let median = sample.percentiles().median();
let mad = sample.median_abs_dev(Some(median));
(mean, std_dev, median, mad)
}
let cl = config.confidence_level;
let nresamples = config.nresamples;
let (mean, std_dev, median, mad) = stats(avg_times);
let points = PointEstimates {
mean,
median,
std_dev,
median_abs_dev: mad,
};
let (dist_mean, dist_stddev, dist_median, dist_mad) = elapsed!(
"Bootstrapping the absolute statistics.",
avg_times.bootstrap(nresamples, stats)
);
let distributions = Distributions {
mean: dist_mean,
slope: None,
median: dist_median,
median_abs_dev: dist_mad,
std_dev: dist_stddev,
};
let estimates = build_estimates(&distributions, &points, cl);
(distributions, estimates)
}
fn copy_new_dir_to_base(id: &str, baseline: &str, output_directory: &Path) {
let root_dir = Path::new(output_directory).join(id);
let base_dir = root_dir.join(baseline);
let new_dir = root_dir.join("new");
if !new_dir.exists() {
return;
};
if !base_dir.exists() {
try_else_return!(fs::mkdirp(&base_dir));
}
try_else_return!(fs::cp(
&new_dir.join("estimates.json"),
&base_dir.join("estimates.json")
));
try_else_return!(fs::cp(
&new_dir.join("sample.json"),
&base_dir.join("sample.json")
));
try_else_return!(fs::cp(
&new_dir.join("tukey.json"),
&base_dir.join("tukey.json")
));
try_else_return!(fs::cp(
&new_dir.join("benchmark.json"),
&base_dir.join("benchmark.json")
));
#[cfg(feature = "csv_output")]
try_else_return!(fs::cp(&new_dir.join("raw.csv"), &base_dir.join("raw.csv")));
}