summaryrefslogtreecommitdiff
path: root/utils/google-benchmark/test/reporter_output_test.cc
diff options
context:
space:
mode:
Diffstat (limited to 'utils/google-benchmark/test/reporter_output_test.cc')
-rw-r--r--utils/google-benchmark/test/reporter_output_test.cc191
1 files changed, 159 insertions, 32 deletions
diff --git a/utils/google-benchmark/test/reporter_output_test.cc b/utils/google-benchmark/test/reporter_output_test.cc
index cb52aec0c0820..1662fcb8b5433 100644
--- a/utils/google-benchmark/test/reporter_output_test.cc
+++ b/utils/google-benchmark/test/reporter_output_test.cc
@@ -9,20 +9,55 @@
// ---------------------- Testing Prologue Output -------------------------- //
// ========================================================================= //
-ADD_CASES(TC_ConsoleOut,
- {{"^[-]+$", MR_Next},
- {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
- {"^[-]+$", MR_Next}});
-ADD_CASES(TC_CSVOut,
- {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
- "items_per_second,label,error_occurred,error_message"}});
+ADD_CASES(TC_ConsoleOut, {{"^[-]+$", MR_Next},
+ {"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
+ {"^[-]+$", MR_Next}});
+static int AddContextCases() {
+ AddCases(TC_ConsoleErr,
+ {
+ {"%int[-/]%int[-/]%int %int:%int:%int$", MR_Default},
+ {"Running .*/reporter_output_test(\\.exe)?$", MR_Next},
+ {"Run on \\(%int X %float MHz CPU s\\)", MR_Next},
+ });
+ AddCases(TC_JSONOut,
+ {{"^\\{", MR_Default},
+ {"\"context\":", MR_Next},
+ {"\"date\": \"", MR_Next},
+ {"\"executable\": \".*/reporter_output_test(\\.exe)?\",", MR_Next},
+ {"\"num_cpus\": %int,$", MR_Next},
+ {"\"mhz_per_cpu\": %float,$", MR_Next},
+ {"\"cpu_scaling_enabled\": ", MR_Next},
+ {"\"caches\": \\[$", MR_Next}});
+ auto const& Caches = benchmark::CPUInfo::Get().caches;
+ if (!Caches.empty()) {
+ AddCases(TC_ConsoleErr, {{"CPU Caches:$", MR_Next}});
+ }
+ for (size_t I = 0; I < Caches.size(); ++I) {
+ std::string num_caches_str =
+ Caches[I].num_sharing != 0 ? " \\(x%int\\)$" : "$";
+ AddCases(
+ TC_ConsoleErr,
+ {{"L%int (Data|Instruction|Unified) %intK" + num_caches_str, MR_Next}});
+ AddCases(TC_JSONOut, {{"\\{$", MR_Next},
+ {"\"type\": \"", MR_Next},
+ {"\"level\": %int,$", MR_Next},
+ {"\"size\": %int,$", MR_Next},
+ {"\"num_sharing\": %int$", MR_Next},
+ {"}[,]{0,1}$", MR_Next}});
+ }
+
+ AddCases(TC_JSONOut, {{"],$"}});
+ return 0;
+}
+int dummy_register = AddContextCases();
+ADD_CASES(TC_CSVOut, {{"%csv_header"}});
// ========================================================================= //
// ------------------------ Testing Basic Output --------------------------- //
// ========================================================================= //
void BM_basic(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_basic);
@@ -30,8 +65,8 @@ BENCHMARK(BM_basic);
ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
{"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %int,$", MR_Next},
- {"\"cpu_time\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\"$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
@@ -41,20 +76,20 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
// ========================================================================= //
void BM_bytes_per_second(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
state.SetBytesProcessed(1);
}
BENCHMARK(BM_bytes_per_second);
ADD_CASES(TC_ConsoleOut,
- {{"^BM_bytes_per_second %console_report +%floatB/s$"}});
+ {{"^BM_bytes_per_second %console_report +%float[kM]{0,1}B/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
{"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %int,$", MR_Next},
- {"\"cpu_time\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
- {"\"bytes_per_second\": %int$", MR_Next},
+ {"\"bytes_per_second\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
@@ -63,20 +98,20 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
// ========================================================================= //
void BM_items_per_second(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
state.SetItemsProcessed(1);
}
BENCHMARK(BM_items_per_second);
ADD_CASES(TC_ConsoleOut,
- {{"^BM_items_per_second %console_report +%float items/s$"}});
+ {{"^BM_items_per_second %console_report +%float[kM]{0,1} items/s$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
{"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %int,$", MR_Next},
- {"\"cpu_time\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
- {"\"items_per_second\": %int$", MR_Next},
+ {"\"items_per_second\": %float$", MR_Next},
{"}", MR_Next}});
ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
@@ -85,7 +120,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
// ========================================================================= //
void BM_label(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
state.SetLabel("some label");
}
@@ -94,8 +129,8 @@ BENCHMARK(BM_label);
ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
{"\"iterations\": %int,$", MR_Next},
- {"\"real_time\": %int,$", MR_Next},
- {"\"cpu_time\": %int,$", MR_Next},
+ {"\"real_time\": %float,$", MR_Next},
+ {"\"cpu_time\": %float,$", MR_Next},
{"\"time_unit\": \"ns\",$", MR_Next},
{"\"label\": \"some label\"$", MR_Next},
{"}", MR_Next}});
@@ -108,7 +143,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
void BM_error(benchmark::State& state) {
state.SkipWithError("message");
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_error);
@@ -125,7 +160,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
// ========================================================================= //
void BM_no_arg_name(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_no_arg_name)->Arg(3);
@@ -138,7 +173,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
// ========================================================================= //
void BM_arg_name(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
@@ -151,7 +186,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
// ========================================================================= //
void BM_arg_names(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
@@ -165,7 +200,7 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
// ========================================================================= //
void BM_Complexity_O1(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
state.SetComplexityN(state.range(0));
}
@@ -181,30 +216,74 @@ ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
// Test that non-aggregate data is printed by default
void BM_Repeat(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
+// need two repetitions min to be able to output any aggregate output
+BENCHMARK(BM_Repeat)->Repetitions(2);
+ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:2 %console_report$"},
+ {"^BM_Repeat/repeats:2 %console_report$"},
+ {"^BM_Repeat/repeats:2_mean %console_report$"},
+ {"^BM_Repeat/repeats:2_median %console_report$"},
+ {"^BM_Repeat/repeats:2_stddev %console_report$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:2\",$"},
+ {"\"name\": \"BM_Repeat/repeats:2\",$"},
+ {"\"name\": \"BM_Repeat/repeats:2_mean\",$"},
+ {"\"name\": \"BM_Repeat/repeats:2_median\",$"},
+ {"\"name\": \"BM_Repeat/repeats:2_stddev\",$"}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:2\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:2\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:2_mean\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:2_median\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:2_stddev\",%csv_report$"}});
+// but for two repetitions, mean and median is the same, so let's repeat..
BENCHMARK(BM_Repeat)->Repetitions(3);
ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3 %console_report$"},
{"^BM_Repeat/repeats:3_mean %console_report$"},
+ {"^BM_Repeat/repeats:3_median %console_report$"},
{"^BM_Repeat/repeats:3_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"name\": \"BM_Repeat/repeats:3\",$"},
{"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
+ {"\"name\": \"BM_Repeat/repeats:3_median\",$"},
{"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:3_median\",%csv_report$"},
{"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
+// median differs between even/odd number of repetitions, so just to be sure
+BENCHMARK(BM_Repeat)->Repetitions(4);
+ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4 %console_report$"},
+ {"^BM_Repeat/repeats:4_mean %console_report$"},
+ {"^BM_Repeat/repeats:4_median %console_report$"},
+ {"^BM_Repeat/repeats:4_stddev %console_report$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4_mean\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4_median\",$"},
+ {"\"name\": \"BM_Repeat/repeats:4_stddev\",$"}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:4\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4_mean\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4_median\",%csv_report$"},
+ {"^\"BM_Repeat/repeats:4_stddev\",%csv_report$"}});
// Test that a non-repeated test still prints non-aggregate results even when
// only-aggregate reports have been requested
void BM_RepeatOnce(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
@@ -214,23 +293,26 @@ ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
// Test that non-aggregate data is not reported
void BM_SummaryRepeat(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
ADD_CASES(TC_ConsoleOut,
{{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
+ {"^BM_SummaryRepeat/repeats:3_median %console_report$"},
{"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
+ {"\"name\": \"BM_SummaryRepeat/repeats:3_median\",$"},
{"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
{"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
+ {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
{"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
void BM_RepeatTimeUnit(benchmark::State& state) {
- while (state.KeepRunning()) {
+ for (auto _ : state) {
}
}
BENCHMARK(BM_RepeatTimeUnit)
@@ -240,18 +322,63 @@ BENCHMARK(BM_RepeatTimeUnit)
ADD_CASES(TC_ConsoleOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
+ {"^BM_RepeatTimeUnit/repeats:3_median %console_us_report$"},
{"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
{"\"time_unit\": \"us\",?$"},
+ {"\"name\": \"BM_RepeatTimeUnit/repeats:3_median\",$"},
+ {"\"time_unit\": \"us\",?$"},
{"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
{"\"time_unit\": \"us\",?$"}});
ADD_CASES(TC_CSVOut,
{{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
{"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
+ {"^\"BM_RepeatTimeUnit/repeats:3_median\",%csv_us_report$"},
{"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
// ========================================================================= //
+// -------------------- Testing user-provided statistics ------------------- //
+// ========================================================================= //
+
+const auto UserStatistics = [](const std::vector<double>& v) {
+ return v.back();
+};
+void BM_UserStats(benchmark::State& state) {
+ for (auto _ : state) {
+ }
+}
+// clang-format off
+BENCHMARK(BM_UserStats)
+ ->Repetitions(3)
+ ->ComputeStatistics("", UserStatistics);
+// clang-format on
+
+// check that user-provided stats is calculated, and is after the default-ones
+// empty string as name is intentional, it would sort before anything else
+ADD_CASES(TC_ConsoleOut, {{"^BM_UserStats/repeats:3 %console_report$"},
+ {"^BM_UserStats/repeats:3 %console_report$"},
+ {"^BM_UserStats/repeats:3 %console_report$"},
+ {"^BM_UserStats/repeats:3_mean %console_report$"},
+ {"^BM_UserStats/repeats:3_median %console_report$"},
+ {"^BM_UserStats/repeats:3_stddev %console_report$"},
+ {"^BM_UserStats/repeats:3_ %console_report$"}});
+ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_UserStats/repeats:3\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3_mean\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3_median\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3_stddev\",$"},
+ {"\"name\": \"BM_UserStats/repeats:3_\",$"}});
+ADD_CASES(TC_CSVOut, {{"^\"BM_UserStats/repeats:3\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3_mean\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3_median\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3_stddev\",%csv_report$"},
+ {"^\"BM_UserStats/repeats:3_\",%csv_report$"}});
+
+// ========================================================================= //
// --------------------------- TEST CASES END ------------------------------ //
// ========================================================================= //
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-12 14:07:39 +0000