1 | // Copyright 2015 Google Inc. All rights reserved. |
2 | // |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | // you may not use this file except in compliance with the License. |
5 | // You may obtain a copy of the License at |
6 | // |
7 | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | // |
9 | // Unless required by applicable law or agreed to in writing, software |
10 | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | // See the License for the specific language governing permissions and |
13 | // limitations under the License. |
14 | |
15 | #include "benchmark/benchmark.h" |
16 | |
17 | #include "benchmark_api_internal.h" |
18 | #include "benchmark_runner.h" |
19 | #include "internal_macros.h" |
20 | |
21 | #ifndef BENCHMARK_OS_WINDOWS |
22 | #ifndef BENCHMARK_OS_FUCHSIA |
23 | #include <sys/resource.h> |
24 | #endif |
25 | #include <sys/time.h> |
26 | #include <unistd.h> |
27 | #endif |
28 | |
29 | #include <algorithm> |
30 | #include <atomic> |
31 | #include <condition_variable> |
32 | #include <cstdio> |
33 | #include <cstdlib> |
34 | #include <fstream> |
35 | #include <iostream> |
36 | #include <map> |
37 | #include <memory> |
38 | #include <string> |
39 | #include <thread> |
40 | #include <utility> |
41 | |
42 | #include "check.h" |
43 | #include "colorprint.h" |
44 | #include "commandlineflags.h" |
45 | #include "complexity.h" |
46 | #include "counter.h" |
47 | #include "internal_macros.h" |
48 | #include "log.h" |
49 | #include "mutex.h" |
50 | #include "perf_counters.h" |
51 | #include "re.h" |
52 | #include "statistics.h" |
53 | #include "string_util.h" |
54 | #include "thread_manager.h" |
55 | #include "thread_timer.h" |
56 | |
57 | // Print a list of benchmarks. This option overrides all other options. |
58 | DEFINE_bool(benchmark_list_tests, false); |
59 | |
60 | // A regular expression that specifies the set of benchmarks to execute. If |
61 | // this flag is empty, or if this flag is the string \"all\", all benchmarks |
62 | // linked into the binary are run. |
63 | DEFINE_string(benchmark_filter, "." ); |
64 | |
65 | // Minimum number of seconds we should run benchmark before results are |
66 | // considered significant. For cpu-time based tests, this is the lower bound |
67 | // on the total cpu time used by all threads that make up the test. For |
68 | // real-time based tests, this is the lower bound on the elapsed time of the |
69 | // benchmark execution, regardless of number of threads. |
70 | DEFINE_double(benchmark_min_time, 0.5); |
71 | |
72 | // The number of runs of each benchmark. If greater than 1, the mean and |
73 | // standard deviation of the runs will be reported. |
74 | DEFINE_int32(benchmark_repetitions, 1); |
75 | |
76 | // Report the result of each benchmark repetitions. When 'true' is specified |
77 | // only the mean, standard deviation, and other statistics are reported for |
78 | // repeated benchmarks. Affects all reporters. |
79 | DEFINE_bool(benchmark_report_aggregates_only, false); |
80 | |
81 | // Display the result of each benchmark repetitions. When 'true' is specified |
82 | // only the mean, standard deviation, and other statistics are displayed for |
83 | // repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects |
84 | // the display reporter, but *NOT* file reporter, which will still contain |
85 | // all the output. |
86 | DEFINE_bool(benchmark_display_aggregates_only, false); |
87 | |
88 | // The format to use for console output. |
89 | // Valid values are 'console', 'json', or 'csv'. |
90 | DEFINE_string(benchmark_format, "console" ); |
91 | |
92 | // The format to use for file output. |
93 | // Valid values are 'console', 'json', or 'csv'. |
94 | DEFINE_string(benchmark_out_format, "json" ); |
95 | |
96 | // The file to write additional output to. |
97 | DEFINE_string(benchmark_out, "" ); |
98 | |
99 | // Whether to use colors in the output. Valid values: |
100 | // 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if |
101 | // the output is being sent to a terminal and the TERM environment variable is |
102 | // set to a terminal type that supports colors. |
103 | DEFINE_string(benchmark_color, "auto" ); |
104 | |
105 | // Whether to use tabular format when printing user counters to the console. |
106 | // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. |
107 | DEFINE_bool(benchmark_counters_tabular, false); |
108 | |
109 | // The level of verbose logging to output |
110 | DEFINE_int32(v, 0); |
111 | |
112 | // List of additional perf counters to collect, in libpfm format. For more |
113 | // information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html |
114 | DEFINE_string(benchmark_perf_counters, "" ); |
115 | |
116 | namespace benchmark { |
117 | namespace internal { |
118 | |
119 | // Extra context to include in the output formatted as comma-separated key-value |
120 | // pairs. Kept internal as it's only used for parsing from env/command line. |
121 | DEFINE_kvpairs(benchmark_context, {}); |
122 | |
123 | std::map<std::string, std::string>* global_context = nullptr; |
124 | |
125 | // FIXME: wouldn't LTO mess this up? |
126 | void UseCharPointer(char const volatile*) {} |
127 | |
128 | } // namespace internal |
129 | |
130 | State::State(IterationCount max_iters, const std::vector<int64_t>& ranges, |
131 | int thread_i, int n_threads, internal::ThreadTimer* timer, |
132 | internal::ThreadManager* manager, |
133 | internal::PerfCountersMeasurement* perf_counters_measurement) |
134 | : total_iterations_(0), |
135 | batch_leftover_(0), |
136 | max_iterations(max_iters), |
137 | started_(false), |
138 | finished_(false), |
139 | error_occurred_(false), |
140 | range_(ranges), |
141 | complexity_n_(0), |
142 | counters(), |
143 | thread_index(thread_i), |
144 | threads(n_threads), |
145 | timer_(timer), |
146 | manager_(manager), |
147 | perf_counters_measurement_(perf_counters_measurement) { |
148 | CHECK(max_iterations != 0) << "At least one iteration must be run" ; |
149 | CHECK_LT(thread_index, threads) << "thread_index must be less than threads" ; |
150 | |
151 | // Note: The use of offsetof below is technically undefined until C++17 |
152 | // because State is not a standard layout type. However, all compilers |
153 | // currently provide well-defined behavior as an extension (which is |
154 | // demonstrated since constexpr evaluation must diagnose all undefined |
155 | // behavior). However, GCC and Clang also warn about this use of offsetof, |
156 | // which must be suppressed. |
157 | #if defined(__INTEL_COMPILER) |
158 | #pragma warning push |
159 | #pragma warning(disable : 1875) |
160 | #elif defined(__GNUC__) |
161 | #pragma GCC diagnostic push |
162 | #pragma GCC diagnostic ignored "-Winvalid-offsetof" |
163 | #endif |
164 | // Offset tests to ensure commonly accessed data is on the first cache line. |
165 | const int cache_line_size = 64; |
166 | static_assert(offsetof(State, error_occurred_) <= |
167 | (cache_line_size - sizeof(error_occurred_)), |
168 | "" ); |
169 | #if defined(__INTEL_COMPILER) |
170 | #pragma warning pop |
171 | #elif defined(__GNUC__) |
172 | #pragma GCC diagnostic pop |
173 | #endif |
174 | } |
175 | |
176 | void State::PauseTiming() { |
177 | // Add in time accumulated so far |
178 | CHECK(started_ && !finished_ && !error_occurred_); |
179 | timer_->StopTimer(); |
180 | if (perf_counters_measurement_) { |
181 | auto measurements = perf_counters_measurement_->StopAndGetMeasurements(); |
182 | for (const auto& name_and_measurement : measurements) { |
183 | auto name = name_and_measurement.first; |
184 | auto measurement = name_and_measurement.second; |
185 | CHECK_EQ(counters[name], 0.0); |
186 | counters[name] = Counter(measurement, Counter::kAvgIterations); |
187 | } |
188 | } |
189 | } |
190 | |
191 | void State::ResumeTiming() { |
192 | CHECK(started_ && !finished_ && !error_occurred_); |
193 | timer_->StartTimer(); |
194 | if (perf_counters_measurement_) { |
195 | perf_counters_measurement_->Start(); |
196 | } |
197 | } |
198 | |
199 | void State::SkipWithError(const char* msg) { |
200 | CHECK(msg); |
201 | error_occurred_ = true; |
202 | { |
203 | MutexLock l(manager_->GetBenchmarkMutex()); |
204 | if (manager_->results.has_error_ == false) { |
205 | manager_->results.error_message_ = msg; |
206 | manager_->results.has_error_ = true; |
207 | } |
208 | } |
209 | total_iterations_ = 0; |
210 | if (timer_->running()) timer_->StopTimer(); |
211 | } |
212 | |
213 | void State::SetIterationTime(double seconds) { |
214 | timer_->SetIterationTime(seconds); |
215 | } |
216 | |
217 | void State::SetLabel(const char* label) { |
218 | MutexLock l(manager_->GetBenchmarkMutex()); |
219 | manager_->results.report_label_ = label; |
220 | } |
221 | |
222 | void State::StartKeepRunning() { |
223 | CHECK(!started_ && !finished_); |
224 | started_ = true; |
225 | total_iterations_ = error_occurred_ ? 0 : max_iterations; |
226 | manager_->StartStopBarrier(); |
227 | if (!error_occurred_) ResumeTiming(); |
228 | } |
229 | |
230 | void State::FinishKeepRunning() { |
231 | CHECK(started_ && (!finished_ || error_occurred_)); |
232 | if (!error_occurred_) { |
233 | PauseTiming(); |
234 | } |
235 | // Total iterations has now wrapped around past 0. Fix this. |
236 | total_iterations_ = 0; |
237 | finished_ = true; |
238 | manager_->StartStopBarrier(); |
239 | } |
240 | |
241 | namespace internal { |
242 | namespace { |
243 | |
244 | void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, |
245 | BenchmarkReporter* display_reporter, |
246 | BenchmarkReporter* file_reporter) { |
247 | // Note the file_reporter can be null. |
248 | CHECK(display_reporter != nullptr); |
249 | |
250 | // Determine the width of the name field using a minimum width of 10. |
251 | bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; |
252 | size_t name_field_width = 10; |
253 | size_t stat_field_width = 0; |
254 | for (const BenchmarkInstance& benchmark : benchmarks) { |
255 | name_field_width = |
256 | std::max<size_t>(name_field_width, benchmark.name().str().size()); |
257 | might_have_aggregates |= benchmark.repetitions() > 1; |
258 | |
259 | for (const auto& Stat : benchmark.statistics()) |
260 | stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size()); |
261 | } |
262 | if (might_have_aggregates) name_field_width += 1 + stat_field_width; |
263 | |
264 | // Print header here |
265 | BenchmarkReporter::Context context; |
266 | context.name_field_width = name_field_width; |
267 | |
268 | // Keep track of running times of all instances of current benchmark |
269 | std::vector<BenchmarkReporter::Run> complexity_reports; |
270 | |
271 | // We flush streams after invoking reporter methods that write to them. This |
272 | // ensures users get timely updates even when streams are not line-buffered. |
273 | auto flushStreams = [](BenchmarkReporter* reporter) { |
274 | if (!reporter) return; |
275 | std::flush(reporter->GetOutputStream()); |
276 | std::flush(reporter->GetErrorStream()); |
277 | }; |
278 | |
279 | if (display_reporter->ReportContext(context) && |
280 | (!file_reporter || file_reporter->ReportContext(context))) { |
281 | flushStreams(display_reporter); |
282 | flushStreams(file_reporter); |
283 | |
284 | for (const auto& benchmark : benchmarks) { |
285 | RunResults run_results = RunBenchmark(benchmark, &complexity_reports); |
286 | |
287 | auto report = [&run_results](BenchmarkReporter* reporter, |
288 | bool report_aggregates_only) { |
289 | assert(reporter); |
290 | // If there are no aggregates, do output non-aggregates. |
291 | report_aggregates_only &= !run_results.aggregates_only.empty(); |
292 | if (!report_aggregates_only) |
293 | reporter->ReportRuns(run_results.non_aggregates); |
294 | if (!run_results.aggregates_only.empty()) |
295 | reporter->ReportRuns(run_results.aggregates_only); |
296 | }; |
297 | |
298 | report(display_reporter, run_results.display_report_aggregates_only); |
299 | if (file_reporter) |
300 | report(file_reporter, run_results.file_report_aggregates_only); |
301 | |
302 | flushStreams(display_reporter); |
303 | flushStreams(file_reporter); |
304 | } |
305 | } |
306 | display_reporter->Finalize(); |
307 | if (file_reporter) file_reporter->Finalize(); |
308 | flushStreams(display_reporter); |
309 | flushStreams(file_reporter); |
310 | } |
311 | |
312 | // Disable deprecated warnings temporarily because we need to reference |
313 | // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations |
314 | #ifdef __GNUC__ |
315 | #pragma GCC diagnostic push |
316 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" |
317 | #endif |
318 | |
319 | std::unique_ptr<BenchmarkReporter> CreateReporter( |
320 | std::string const& name, ConsoleReporter::OutputOptions output_opts) { |
321 | typedef std::unique_ptr<BenchmarkReporter> PtrType; |
322 | if (name == "console" ) { |
323 | return PtrType(new ConsoleReporter(output_opts)); |
324 | } else if (name == "json" ) { |
325 | return PtrType(new JSONReporter); |
326 | } else if (name == "csv" ) { |
327 | return PtrType(new CSVReporter); |
328 | } else { |
329 | std::cerr << "Unexpected format: '" << name << "'\n" ; |
330 | std::exit(1); |
331 | } |
332 | } |
333 | |
334 | #ifdef __GNUC__ |
335 | #pragma GCC diagnostic pop |
336 | #endif |
337 | |
338 | } // end namespace |
339 | |
340 | bool IsZero(double n) { |
341 | return std::abs(n) < std::numeric_limits<double>::epsilon(); |
342 | } |
343 | |
344 | ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { |
345 | int output_opts = ConsoleReporter::OO_Defaults; |
346 | auto is_benchmark_color = [force_no_color]() -> bool { |
347 | if (force_no_color) { |
348 | return false; |
349 | } |
350 | if (FLAGS_benchmark_color == "auto" ) { |
351 | return IsColorTerminal(); |
352 | } |
353 | return IsTruthyFlagValue(FLAGS_benchmark_color); |
354 | }; |
355 | if (is_benchmark_color()) { |
356 | output_opts |= ConsoleReporter::OO_Color; |
357 | } else { |
358 | output_opts &= ~ConsoleReporter::OO_Color; |
359 | } |
360 | if (FLAGS_benchmark_counters_tabular) { |
361 | output_opts |= ConsoleReporter::OO_Tabular; |
362 | } else { |
363 | output_opts &= ~ConsoleReporter::OO_Tabular; |
364 | } |
365 | return static_cast<ConsoleReporter::OutputOptions>(output_opts); |
366 | } |
367 | |
368 | } // end namespace internal |
369 | |
370 | size_t RunSpecifiedBenchmarks() { |
371 | return RunSpecifiedBenchmarks(nullptr, nullptr); |
372 | } |
373 | |
374 | size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) { |
375 | return RunSpecifiedBenchmarks(display_reporter, nullptr); |
376 | } |
377 | |
378 | size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, |
379 | BenchmarkReporter* file_reporter) { |
380 | std::string spec = FLAGS_benchmark_filter; |
381 | if (spec.empty() || spec == "all" ) |
382 | spec = "." ; // Regexp that matches all benchmarks |
383 | |
384 | // Setup the reporters |
385 | std::ofstream output_file; |
386 | std::unique_ptr<BenchmarkReporter> default_display_reporter; |
387 | std::unique_ptr<BenchmarkReporter> default_file_reporter; |
388 | if (!display_reporter) { |
389 | default_display_reporter = internal::CreateReporter( |
390 | FLAGS_benchmark_format, internal::GetOutputOptions()); |
391 | display_reporter = default_display_reporter.get(); |
392 | } |
393 | auto& Out = display_reporter->GetOutputStream(); |
394 | auto& Err = display_reporter->GetErrorStream(); |
395 | |
396 | std::string const& fname = FLAGS_benchmark_out; |
397 | if (fname.empty() && file_reporter) { |
398 | Err << "A custom file reporter was provided but " |
399 | "--benchmark_out=<file> was not specified." |
400 | << std::endl; |
401 | std::exit(1); |
402 | } |
403 | if (!fname.empty()) { |
404 | output_file.open(fname); |
405 | if (!output_file.is_open()) { |
406 | Err << "invalid file name: '" << fname << "'" << std::endl; |
407 | std::exit(1); |
408 | } |
409 | if (!file_reporter) { |
410 | default_file_reporter = internal::CreateReporter( |
411 | FLAGS_benchmark_out_format, ConsoleReporter::OO_None); |
412 | file_reporter = default_file_reporter.get(); |
413 | } |
414 | file_reporter->SetOutputStream(&output_file); |
415 | file_reporter->SetErrorStream(&output_file); |
416 | } |
417 | |
418 | std::vector<internal::BenchmarkInstance> benchmarks; |
419 | if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; |
420 | |
421 | if (benchmarks.empty()) { |
422 | Err << "Failed to match any benchmarks against regex: " << spec << "\n" ; |
423 | return 0; |
424 | } |
425 | |
426 | if (FLAGS_benchmark_list_tests) { |
427 | for (auto const& benchmark : benchmarks) |
428 | Out << benchmark.name().str() << "\n" ; |
429 | } else { |
430 | internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); |
431 | } |
432 | |
433 | return benchmarks.size(); |
434 | } |
435 | |
436 | void RegisterMemoryManager(MemoryManager* manager) { |
437 | internal::memory_manager = manager; |
438 | } |
439 | |
440 | void AddCustomContext(const std::string& key, const std::string& value) { |
441 | if (internal::global_context == nullptr) { |
442 | internal::global_context = new std::map<std::string, std::string>(); |
443 | } |
444 | if (!internal::global_context->emplace(key, value).second) { |
445 | std::cerr << "Failed to add custom context \"" << key << "\" as it already " |
446 | << "exists with value \"" << value << "\"\n" ; |
447 | } |
448 | } |
449 | |
450 | namespace internal { |
451 | |
452 | void PrintUsageAndExit() { |
453 | fprintf(stdout, |
454 | "benchmark" |
455 | " [--benchmark_list_tests={true|false}]\n" |
456 | " [--benchmark_filter=<regex>]\n" |
457 | " [--benchmark_min_time=<min_time>]\n" |
458 | " [--benchmark_repetitions=<num_repetitions>]\n" |
459 | " [--benchmark_report_aggregates_only={true|false}]\n" |
460 | " [--benchmark_display_aggregates_only={true|false}]\n" |
461 | " [--benchmark_format=<console|json|csv>]\n" |
462 | " [--benchmark_out=<filename>]\n" |
463 | " [--benchmark_out_format=<json|console|csv>]\n" |
464 | " [--benchmark_color={auto|true|false}]\n" |
465 | " [--benchmark_counters_tabular={true|false}]\n" |
466 | " [--benchmark_context=<key>=<value>,...]\n" |
467 | " [--v=<verbosity>]\n" ); |
468 | exit(0); |
469 | } |
470 | |
471 | void ParseCommandLineFlags(int* argc, char** argv) { |
472 | using namespace benchmark; |
473 | BenchmarkReporter::Context::executable_name = |
474 | (argc && *argc > 0) ? argv[0] : "unknown" ; |
475 | for (int i = 1; argc && i < *argc; ++i) { |
476 | if (ParseBoolFlag(argv[i], "benchmark_list_tests" , |
477 | &FLAGS_benchmark_list_tests) || |
478 | ParseStringFlag(argv[i], "benchmark_filter" , &FLAGS_benchmark_filter) || |
479 | ParseDoubleFlag(argv[i], "benchmark_min_time" , |
480 | &FLAGS_benchmark_min_time) || |
481 | ParseInt32Flag(argv[i], "benchmark_repetitions" , |
482 | &FLAGS_benchmark_repetitions) || |
483 | ParseBoolFlag(argv[i], "benchmark_report_aggregates_only" , |
484 | &FLAGS_benchmark_report_aggregates_only) || |
485 | ParseBoolFlag(argv[i], "benchmark_display_aggregates_only" , |
486 | &FLAGS_benchmark_display_aggregates_only) || |
487 | ParseStringFlag(argv[i], "benchmark_format" , &FLAGS_benchmark_format) || |
488 | ParseStringFlag(argv[i], "benchmark_out" , &FLAGS_benchmark_out) || |
489 | ParseStringFlag(argv[i], "benchmark_out_format" , |
490 | &FLAGS_benchmark_out_format) || |
491 | ParseStringFlag(argv[i], "benchmark_color" , &FLAGS_benchmark_color) || |
492 | // "color_print" is the deprecated name for "benchmark_color". |
493 | // TODO: Remove this. |
494 | ParseStringFlag(argv[i], "color_print" , &FLAGS_benchmark_color) || |
495 | ParseBoolFlag(argv[i], "benchmark_counters_tabular" , |
496 | &FLAGS_benchmark_counters_tabular) || |
497 | ParseStringFlag(argv[i], "benchmark_perf_counters" , |
498 | &FLAGS_benchmark_perf_counters) || |
499 | ParseKeyValueFlag(argv[i], "benchmark_context" , |
500 | &FLAGS_benchmark_context) || |
501 | ParseInt32Flag(argv[i], "v" , &FLAGS_v)) { |
502 | for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; |
503 | |
504 | --(*argc); |
505 | --i; |
506 | } else if (IsFlag(argv[i], "help" )) { |
507 | PrintUsageAndExit(); |
508 | } |
509 | } |
510 | for (auto const* flag : |
511 | {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) { |
512 | if (*flag != "console" && *flag != "json" && *flag != "csv" ) { |
513 | PrintUsageAndExit(); |
514 | } |
515 | } |
516 | if (FLAGS_benchmark_color.empty()) { |
517 | PrintUsageAndExit(); |
518 | } |
519 | for (const auto& kv : FLAGS_benchmark_context) { |
520 | AddCustomContext(kv.first, kv.second); |
521 | } |
522 | } |
523 | |
524 | int InitializeStreams() { |
525 | static std::ios_base::Init init; |
526 | return 0; |
527 | } |
528 | |
529 | } // end namespace internal |
530 | |
531 | void Initialize(int* argc, char** argv) { |
532 | internal::ParseCommandLineFlags(argc, argv); |
533 | internal::LogLevel() = FLAGS_v; |
534 | } |
535 | |
536 | bool ReportUnrecognizedArguments(int argc, char** argv) { |
537 | for (int i = 1; i < argc; ++i) { |
538 | fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n" , argv[0], |
539 | argv[i]); |
540 | } |
541 | return argc > 1; |
542 | } |
543 | |
544 | } // end namespace benchmark |
545 | |