1// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "benchmark/benchmark.h"
16
17#include "benchmark_api_internal.h"
18#include "benchmark_runner.h"
19#include "internal_macros.h"
20
21#ifndef BENCHMARK_OS_WINDOWS
22#ifndef BENCHMARK_OS_FUCHSIA
23#include <sys/resource.h>
24#endif
25#include <sys/time.h>
26#include <unistd.h>
27#endif
28
29#include <algorithm>
30#include <atomic>
31#include <condition_variable>
32#include <cstdio>
33#include <cstdlib>
34#include <fstream>
35#include <iostream>
36#include <map>
37#include <memory>
38#include <string>
39#include <thread>
40#include <utility>
41
42#include "check.h"
43#include "colorprint.h"
44#include "commandlineflags.h"
45#include "complexity.h"
46#include "counter.h"
47#include "internal_macros.h"
48#include "log.h"
49#include "mutex.h"
50#include "perf_counters.h"
51#include "re.h"
52#include "statistics.h"
53#include "string_util.h"
54#include "thread_manager.h"
55#include "thread_timer.h"
56
57// Print a list of benchmarks. This option overrides all other options.
58DEFINE_bool(benchmark_list_tests, false);
59
60// A regular expression that specifies the set of benchmarks to execute. If
61// this flag is empty, or if this flag is the string \"all\", all benchmarks
62// linked into the binary are run.
63DEFINE_string(benchmark_filter, ".");
64
65// Minimum number of seconds we should run benchmark before results are
66// considered significant. For cpu-time based tests, this is the lower bound
67// on the total cpu time used by all threads that make up the test. For
68// real-time based tests, this is the lower bound on the elapsed time of the
69// benchmark execution, regardless of number of threads.
70DEFINE_double(benchmark_min_time, 0.5);
71
72// The number of runs of each benchmark. If greater than 1, the mean and
73// standard deviation of the runs will be reported.
74DEFINE_int32(benchmark_repetitions, 1);
75
76// Report the result of each benchmark repetitions. When 'true' is specified
77// only the mean, standard deviation, and other statistics are reported for
78// repeated benchmarks. Affects all reporters.
79DEFINE_bool(benchmark_report_aggregates_only, false);
80
81// Display the result of each benchmark repetitions. When 'true' is specified
82// only the mean, standard deviation, and other statistics are displayed for
83// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
84// the display reporter, but *NOT* file reporter, which will still contain
85// all the output.
86DEFINE_bool(benchmark_display_aggregates_only, false);
87
88// The format to use for console output.
89// Valid values are 'console', 'json', or 'csv'.
90DEFINE_string(benchmark_format, "console");
91
92// The format to use for file output.
93// Valid values are 'console', 'json', or 'csv'.
94DEFINE_string(benchmark_out_format, "json");
95
96// The file to write additional output to.
97DEFINE_string(benchmark_out, "");
98
99// Whether to use colors in the output. Valid values:
100// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
101// the output is being sent to a terminal and the TERM environment variable is
102// set to a terminal type that supports colors.
103DEFINE_string(benchmark_color, "auto");
104
105// Whether to use tabular format when printing user counters to the console.
106// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
107DEFINE_bool(benchmark_counters_tabular, false);
108
109// The level of verbose logging to output
110DEFINE_int32(v, 0);
111
112// List of additional perf counters to collect, in libpfm format. For more
113// information about libpfm: https://man7.org/linux/man-pages/man3/libpfm.3.html
114DEFINE_string(benchmark_perf_counters, "");
115
116namespace benchmark {
117namespace internal {
118
119// Extra context to include in the output formatted as comma-separated key-value
120// pairs. Kept internal as it's only used for parsing from env/command line.
121DEFINE_kvpairs(benchmark_context, {});
122
123std::map<std::string, std::string>* global_context = nullptr;
124
125// FIXME: wouldn't LTO mess this up?
126void UseCharPointer(char const volatile*) {}
127
128} // namespace internal
129
130State::State(IterationCount max_iters, const std::vector<int64_t>& ranges,
131 int thread_i, int n_threads, internal::ThreadTimer* timer,
132 internal::ThreadManager* manager,
133 internal::PerfCountersMeasurement* perf_counters_measurement)
134 : total_iterations_(0),
135 batch_leftover_(0),
136 max_iterations(max_iters),
137 started_(false),
138 finished_(false),
139 error_occurred_(false),
140 range_(ranges),
141 complexity_n_(0),
142 counters(),
143 thread_index(thread_i),
144 threads(n_threads),
145 timer_(timer),
146 manager_(manager),
147 perf_counters_measurement_(perf_counters_measurement) {
148 CHECK(max_iterations != 0) << "At least one iteration must be run";
149 CHECK_LT(thread_index, threads) << "thread_index must be less than threads";
150
151 // Note: The use of offsetof below is technically undefined until C++17
152 // because State is not a standard layout type. However, all compilers
153 // currently provide well-defined behavior as an extension (which is
154 // demonstrated since constexpr evaluation must diagnose all undefined
155 // behavior). However, GCC and Clang also warn about this use of offsetof,
156 // which must be suppressed.
157#if defined(__INTEL_COMPILER)
158#pragma warning push
159#pragma warning(disable : 1875)
160#elif defined(__GNUC__)
161#pragma GCC diagnostic push
162#pragma GCC diagnostic ignored "-Winvalid-offsetof"
163#endif
164 // Offset tests to ensure commonly accessed data is on the first cache line.
165 const int cache_line_size = 64;
166 static_assert(offsetof(State, error_occurred_) <=
167 (cache_line_size - sizeof(error_occurred_)),
168 "");
169#if defined(__INTEL_COMPILER)
170#pragma warning pop
171#elif defined(__GNUC__)
172#pragma GCC diagnostic pop
173#endif
174}
175
176void State::PauseTiming() {
177 // Add in time accumulated so far
178 CHECK(started_ && !finished_ && !error_occurred_);
179 timer_->StopTimer();
180 if (perf_counters_measurement_) {
181 auto measurements = perf_counters_measurement_->StopAndGetMeasurements();
182 for (const auto& name_and_measurement : measurements) {
183 auto name = name_and_measurement.first;
184 auto measurement = name_and_measurement.second;
185 CHECK_EQ(counters[name], 0.0);
186 counters[name] = Counter(measurement, Counter::kAvgIterations);
187 }
188 }
189}
190
191void State::ResumeTiming() {
192 CHECK(started_ && !finished_ && !error_occurred_);
193 timer_->StartTimer();
194 if (perf_counters_measurement_) {
195 perf_counters_measurement_->Start();
196 }
197}
198
199void State::SkipWithError(const char* msg) {
200 CHECK(msg);
201 error_occurred_ = true;
202 {
203 MutexLock l(manager_->GetBenchmarkMutex());
204 if (manager_->results.has_error_ == false) {
205 manager_->results.error_message_ = msg;
206 manager_->results.has_error_ = true;
207 }
208 }
209 total_iterations_ = 0;
210 if (timer_->running()) timer_->StopTimer();
211}
212
213void State::SetIterationTime(double seconds) {
214 timer_->SetIterationTime(seconds);
215}
216
217void State::SetLabel(const char* label) {
218 MutexLock l(manager_->GetBenchmarkMutex());
219 manager_->results.report_label_ = label;
220}
221
222void State::StartKeepRunning() {
223 CHECK(!started_ && !finished_);
224 started_ = true;
225 total_iterations_ = error_occurred_ ? 0 : max_iterations;
226 manager_->StartStopBarrier();
227 if (!error_occurred_) ResumeTiming();
228}
229
230void State::FinishKeepRunning() {
231 CHECK(started_ && (!finished_ || error_occurred_));
232 if (!error_occurred_) {
233 PauseTiming();
234 }
235 // Total iterations has now wrapped around past 0. Fix this.
236 total_iterations_ = 0;
237 finished_ = true;
238 manager_->StartStopBarrier();
239}
240
241namespace internal {
242namespace {
243
244void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
245 BenchmarkReporter* display_reporter,
246 BenchmarkReporter* file_reporter) {
247 // Note the file_reporter can be null.
248 CHECK(display_reporter != nullptr);
249
250 // Determine the width of the name field using a minimum width of 10.
251 bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
252 size_t name_field_width = 10;
253 size_t stat_field_width = 0;
254 for (const BenchmarkInstance& benchmark : benchmarks) {
255 name_field_width =
256 std::max<size_t>(name_field_width, benchmark.name().str().size());
257 might_have_aggregates |= benchmark.repetitions() > 1;
258
259 for (const auto& Stat : benchmark.statistics())
260 stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size());
261 }
262 if (might_have_aggregates) name_field_width += 1 + stat_field_width;
263
264 // Print header here
265 BenchmarkReporter::Context context;
266 context.name_field_width = name_field_width;
267
268 // Keep track of running times of all instances of current benchmark
269 std::vector<BenchmarkReporter::Run> complexity_reports;
270
271 // We flush streams after invoking reporter methods that write to them. This
272 // ensures users get timely updates even when streams are not line-buffered.
273 auto flushStreams = [](BenchmarkReporter* reporter) {
274 if (!reporter) return;
275 std::flush(reporter->GetOutputStream());
276 std::flush(reporter->GetErrorStream());
277 };
278
279 if (display_reporter->ReportContext(context) &&
280 (!file_reporter || file_reporter->ReportContext(context))) {
281 flushStreams(display_reporter);
282 flushStreams(file_reporter);
283
284 for (const auto& benchmark : benchmarks) {
285 RunResults run_results = RunBenchmark(benchmark, &complexity_reports);
286
287 auto report = [&run_results](BenchmarkReporter* reporter,
288 bool report_aggregates_only) {
289 assert(reporter);
290 // If there are no aggregates, do output non-aggregates.
291 report_aggregates_only &= !run_results.aggregates_only.empty();
292 if (!report_aggregates_only)
293 reporter->ReportRuns(run_results.non_aggregates);
294 if (!run_results.aggregates_only.empty())
295 reporter->ReportRuns(run_results.aggregates_only);
296 };
297
298 report(display_reporter, run_results.display_report_aggregates_only);
299 if (file_reporter)
300 report(file_reporter, run_results.file_report_aggregates_only);
301
302 flushStreams(display_reporter);
303 flushStreams(file_reporter);
304 }
305 }
306 display_reporter->Finalize();
307 if (file_reporter) file_reporter->Finalize();
308 flushStreams(display_reporter);
309 flushStreams(file_reporter);
310}
311
312// Disable deprecated warnings temporarily because we need to reference
313// CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations
314#ifdef __GNUC__
315#pragma GCC diagnostic push
316#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
317#endif
318
319std::unique_ptr<BenchmarkReporter> CreateReporter(
320 std::string const& name, ConsoleReporter::OutputOptions output_opts) {
321 typedef std::unique_ptr<BenchmarkReporter> PtrType;
322 if (name == "console") {
323 return PtrType(new ConsoleReporter(output_opts));
324 } else if (name == "json") {
325 return PtrType(new JSONReporter);
326 } else if (name == "csv") {
327 return PtrType(new CSVReporter);
328 } else {
329 std::cerr << "Unexpected format: '" << name << "'\n";
330 std::exit(1);
331 }
332}
333
334#ifdef __GNUC__
335#pragma GCC diagnostic pop
336#endif
337
338} // end namespace
339
340bool IsZero(double n) {
341 return std::abs(n) < std::numeric_limits<double>::epsilon();
342}
343
344ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {
345 int output_opts = ConsoleReporter::OO_Defaults;
346 auto is_benchmark_color = [force_no_color]() -> bool {
347 if (force_no_color) {
348 return false;
349 }
350 if (FLAGS_benchmark_color == "auto") {
351 return IsColorTerminal();
352 }
353 return IsTruthyFlagValue(FLAGS_benchmark_color);
354 };
355 if (is_benchmark_color()) {
356 output_opts |= ConsoleReporter::OO_Color;
357 } else {
358 output_opts &= ~ConsoleReporter::OO_Color;
359 }
360 if (FLAGS_benchmark_counters_tabular) {
361 output_opts |= ConsoleReporter::OO_Tabular;
362 } else {
363 output_opts &= ~ConsoleReporter::OO_Tabular;
364 }
365 return static_cast<ConsoleReporter::OutputOptions>(output_opts);
366}
367
368} // end namespace internal
369
370size_t RunSpecifiedBenchmarks() {
371 return RunSpecifiedBenchmarks(nullptr, nullptr);
372}
373
374size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) {
375 return RunSpecifiedBenchmarks(display_reporter, nullptr);
376}
377
378size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
379 BenchmarkReporter* file_reporter) {
380 std::string spec = FLAGS_benchmark_filter;
381 if (spec.empty() || spec == "all")
382 spec = "."; // Regexp that matches all benchmarks
383
384 // Setup the reporters
385 std::ofstream output_file;
386 std::unique_ptr<BenchmarkReporter> default_display_reporter;
387 std::unique_ptr<BenchmarkReporter> default_file_reporter;
388 if (!display_reporter) {
389 default_display_reporter = internal::CreateReporter(
390 FLAGS_benchmark_format, internal::GetOutputOptions());
391 display_reporter = default_display_reporter.get();
392 }
393 auto& Out = display_reporter->GetOutputStream();
394 auto& Err = display_reporter->GetErrorStream();
395
396 std::string const& fname = FLAGS_benchmark_out;
397 if (fname.empty() && file_reporter) {
398 Err << "A custom file reporter was provided but "
399 "--benchmark_out=<file> was not specified."
400 << std::endl;
401 std::exit(1);
402 }
403 if (!fname.empty()) {
404 output_file.open(fname);
405 if (!output_file.is_open()) {
406 Err << "invalid file name: '" << fname << "'" << std::endl;
407 std::exit(1);
408 }
409 if (!file_reporter) {
410 default_file_reporter = internal::CreateReporter(
411 FLAGS_benchmark_out_format, ConsoleReporter::OO_None);
412 file_reporter = default_file_reporter.get();
413 }
414 file_reporter->SetOutputStream(&output_file);
415 file_reporter->SetErrorStream(&output_file);
416 }
417
418 std::vector<internal::BenchmarkInstance> benchmarks;
419 if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0;
420
421 if (benchmarks.empty()) {
422 Err << "Failed to match any benchmarks against regex: " << spec << "\n";
423 return 0;
424 }
425
426 if (FLAGS_benchmark_list_tests) {
427 for (auto const& benchmark : benchmarks)
428 Out << benchmark.name().str() << "\n";
429 } else {
430 internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);
431 }
432
433 return benchmarks.size();
434}
435
436void RegisterMemoryManager(MemoryManager* manager) {
437 internal::memory_manager = manager;
438}
439
440void AddCustomContext(const std::string& key, const std::string& value) {
441 if (internal::global_context == nullptr) {
442 internal::global_context = new std::map<std::string, std::string>();
443 }
444 if (!internal::global_context->emplace(key, value).second) {
445 std::cerr << "Failed to add custom context \"" << key << "\" as it already "
446 << "exists with value \"" << value << "\"\n";
447 }
448}
449
450namespace internal {
451
452void PrintUsageAndExit() {
453 fprintf(stdout,
454 "benchmark"
455 " [--benchmark_list_tests={true|false}]\n"
456 " [--benchmark_filter=<regex>]\n"
457 " [--benchmark_min_time=<min_time>]\n"
458 " [--benchmark_repetitions=<num_repetitions>]\n"
459 " [--benchmark_report_aggregates_only={true|false}]\n"
460 " [--benchmark_display_aggregates_only={true|false}]\n"
461 " [--benchmark_format=<console|json|csv>]\n"
462 " [--benchmark_out=<filename>]\n"
463 " [--benchmark_out_format=<json|console|csv>]\n"
464 " [--benchmark_color={auto|true|false}]\n"
465 " [--benchmark_counters_tabular={true|false}]\n"
466 " [--benchmark_context=<key>=<value>,...]\n"
467 " [--v=<verbosity>]\n");
468 exit(0);
469}
470
471void ParseCommandLineFlags(int* argc, char** argv) {
472 using namespace benchmark;
473 BenchmarkReporter::Context::executable_name =
474 (argc && *argc > 0) ? argv[0] : "unknown";
475 for (int i = 1; argc && i < *argc; ++i) {
476 if (ParseBoolFlag(argv[i], "benchmark_list_tests",
477 &FLAGS_benchmark_list_tests) ||
478 ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) ||
479 ParseDoubleFlag(argv[i], "benchmark_min_time",
480 &FLAGS_benchmark_min_time) ||
481 ParseInt32Flag(argv[i], "benchmark_repetitions",
482 &FLAGS_benchmark_repetitions) ||
483 ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
484 &FLAGS_benchmark_report_aggregates_only) ||
485 ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
486 &FLAGS_benchmark_display_aggregates_only) ||
487 ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
488 ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
489 ParseStringFlag(argv[i], "benchmark_out_format",
490 &FLAGS_benchmark_out_format) ||
491 ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) ||
492 // "color_print" is the deprecated name for "benchmark_color".
493 // TODO: Remove this.
494 ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) ||
495 ParseBoolFlag(argv[i], "benchmark_counters_tabular",
496 &FLAGS_benchmark_counters_tabular) ||
497 ParseStringFlag(argv[i], "benchmark_perf_counters",
498 &FLAGS_benchmark_perf_counters) ||
499 ParseKeyValueFlag(argv[i], "benchmark_context",
500 &FLAGS_benchmark_context) ||
501 ParseInt32Flag(argv[i], "v", &FLAGS_v)) {
502 for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1];
503
504 --(*argc);
505 --i;
506 } else if (IsFlag(argv[i], "help")) {
507 PrintUsageAndExit();
508 }
509 }
510 for (auto const* flag :
511 {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) {
512 if (*flag != "console" && *flag != "json" && *flag != "csv") {
513 PrintUsageAndExit();
514 }
515 }
516 if (FLAGS_benchmark_color.empty()) {
517 PrintUsageAndExit();
518 }
519 for (const auto& kv : FLAGS_benchmark_context) {
520 AddCustomContext(kv.first, kv.second);
521 }
522}
523
524int InitializeStreams() {
525 static std::ios_base::Init init;
526 return 0;
527}
528
529} // end namespace internal
530
531void Initialize(int* argc, char** argv) {
532 internal::ParseCommandLineFlags(argc, argv);
533 internal::LogLevel() = FLAGS_v;
534}
535
536bool ReportUnrecognizedArguments(int argc, char** argv) {
537 for (int i = 1; i < argc; ++i) {
538 fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0],
539 argv[i]);
540 }
541 return argc > 1;
542}
543
544} // end namespace benchmark
545