1// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15#include "benchmark/benchmark.h"
16#include "benchmark_api_internal.h"
17#include "benchmark_runner.h"
18#include "internal_macros.h"
19
20#ifndef BENCHMARK_OS_WINDOWS
21#ifndef BENCHMARK_OS_FUCHSIA
22#include <sys/resource.h>
23#endif
24#include <sys/time.h>
25#include <unistd.h>
26#endif
27
28#include <algorithm>
29#include <atomic>
30#include <condition_variable>
31#include <cstdio>
32#include <cstdlib>
33#include <fstream>
34#include <iostream>
35#include <memory>
36#include <string>
37#include <thread>
38#include <utility>
39
40#include "check.h"
41#include "colorprint.h"
42#include "commandlineflags.h"
43#include "complexity.h"
44#include "counter.h"
45#include "internal_macros.h"
46#include "log.h"
47#include "mutex.h"
48#include "re.h"
49#include "statistics.h"
50#include "string_util.h"
51#include "thread_manager.h"
52#include "thread_timer.h"
53
54// Print a list of benchmarks. This option overrides all other options.
55DEFINE_bool(benchmark_list_tests, false);
56
57// A regular expression that specifies the set of benchmarks to execute. If
58// this flag is empty, or if this flag is the string \"all\", all benchmarks
59// linked into the binary are run.
60DEFINE_string(benchmark_filter, ".");
61
62// Minimum number of seconds we should run benchmark before results are
63// considered significant. For cpu-time based tests, this is the lower bound
64// on the total cpu time used by all threads that make up the test. For
65// real-time based tests, this is the lower bound on the elapsed time of the
66// benchmark execution, regardless of number of threads.
67DEFINE_double(benchmark_min_time, 0.5);
68
69// The number of runs of each benchmark. If greater than 1, the mean and
70// standard deviation of the runs will be reported.
71DEFINE_int32(benchmark_repetitions, 1);
72
73// Report the result of each benchmark repetitions. When 'true' is specified
74// only the mean, standard deviation, and other statistics are reported for
75// repeated benchmarks. Affects all reporters.
76DEFINE_bool(benchmark_report_aggregates_only, false);
77
78// Display the result of each benchmark repetitions. When 'true' is specified
79// only the mean, standard deviation, and other statistics are displayed for
80// repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects
81// the display reporter, but *NOT* file reporter, which will still contain
82// all the output.
83DEFINE_bool(benchmark_display_aggregates_only, false);
84
85// The format to use for console output.
86// Valid values are 'console', 'json', or 'csv'.
87DEFINE_string(benchmark_format, "console");
88
89// The format to use for file output.
90// Valid values are 'console', 'json', or 'csv'.
91DEFINE_string(benchmark_out_format, "json");
92
93// The file to write additional output to.
94DEFINE_string(benchmark_out, "");
95
96// Whether to use colors in the output. Valid values:
97// 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if
98// the output is being sent to a terminal and the TERM environment variable is
99// set to a terminal type that supports colors.
100DEFINE_string(benchmark_color, "auto");
101
102// Whether to use tabular format when printing user counters to the console.
103// Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false.
104DEFINE_bool(benchmark_counters_tabular, false);
105
106// The level of verbose logging to output
107DEFINE_int32(v, 0);
108
109namespace benchmark {
110
111namespace internal {
112
113// FIXME: wouldn't LTO mess this up?
114void UseCharPointer(char const volatile*) {}
115
116} // namespace internal
117
118State::State(IterationCount max_iters, const std::vector<int64_t>& ranges,
119 int thread_i, int n_threads, internal::ThreadTimer* timer,
120 internal::ThreadManager* manager)
121 : total_iterations_(0),
122 batch_leftover_(0),
123 max_iterations(max_iters),
124 started_(false),
125 finished_(false),
126 error_occurred_(false),
127 range_(ranges),
128 complexity_n_(0),
129 counters(),
130 thread_index(thread_i),
131 threads(n_threads),
132 timer_(timer),
133 manager_(manager) {
134 CHECK(max_iterations != 0) << "At least one iteration must be run";
135 CHECK_LT(thread_index, threads) << "thread_index must be less than threads";
136
137 // Note: The use of offsetof below is technically undefined until C++17
138 // because State is not a standard layout type. However, all compilers
139 // currently provide well-defined behavior as an extension (which is
140 // demonstrated since constexpr evaluation must diagnose all undefined
141 // behavior). However, GCC and Clang also warn about this use of offsetof,
142 // which must be suppressed.
143#if defined(__INTEL_COMPILER)
144#pragma warning push
145#pragma warning(disable : 1875)
146#elif defined(__GNUC__)
147#pragma GCC diagnostic push
148#pragma GCC diagnostic ignored "-Winvalid-offsetof"
149#endif
150 // Offset tests to ensure commonly accessed data is on the first cache line.
151 const int cache_line_size = 64;
152 static_assert(offsetof(State, error_occurred_) <=
153 (cache_line_size - sizeof(error_occurred_)),
154 "");
155#if defined(__INTEL_COMPILER)
156#pragma warning pop
157#elif defined(__GNUC__)
158#pragma GCC diagnostic pop
159#endif
160}
161
162void State::PauseTiming() {
163 // Add in time accumulated so far
164 CHECK(started_ && !finished_ && !error_occurred_);
165 timer_->StopTimer();
166}
167
168void State::ResumeTiming() {
169 CHECK(started_ && !finished_ && !error_occurred_);
170 timer_->StartTimer();
171}
172
173void State::SkipWithError(const char* msg) {
174 CHECK(msg);
175 error_occurred_ = true;
176 {
177 MutexLock l(manager_->GetBenchmarkMutex());
178 if (manager_->results.has_error_ == false) {
179 manager_->results.error_message_ = msg;
180 manager_->results.has_error_ = true;
181 }
182 }
183 total_iterations_ = 0;
184 if (timer_->running()) timer_->StopTimer();
185}
186
187void State::SetIterationTime(double seconds) {
188 timer_->SetIterationTime(seconds);
189}
190
191void State::SetLabel(const char* label) {
192 MutexLock l(manager_->GetBenchmarkMutex());
193 manager_->results.report_label_ = label;
194}
195
196void State::StartKeepRunning() {
197 CHECK(!started_ && !finished_);
198 started_ = true;
199 total_iterations_ = error_occurred_ ? 0 : max_iterations;
200 manager_->StartStopBarrier();
201 if (!error_occurred_) ResumeTiming();
202}
203
204void State::FinishKeepRunning() {
205 CHECK(started_ && (!finished_ || error_occurred_));
206 if (!error_occurred_) {
207 PauseTiming();
208 }
209 // Total iterations has now wrapped around past 0. Fix this.
210 total_iterations_ = 0;
211 finished_ = true;
212 manager_->StartStopBarrier();
213}
214
215namespace internal {
216namespace {
217
218void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks,
219 BenchmarkReporter* display_reporter,
220 BenchmarkReporter* file_reporter) {
221 // Note the file_reporter can be null.
222 CHECK(display_reporter != nullptr);
223
224 // Determine the width of the name field using a minimum width of 10.
225 bool might_have_aggregates = FLAGS_benchmark_repetitions > 1;
226 size_t name_field_width = 10;
227 size_t stat_field_width = 0;
228 for (const BenchmarkInstance& benchmark : benchmarks) {
229 name_field_width =
230 std::max<size_t>(name_field_width, benchmark.name.str().size());
231 might_have_aggregates |= benchmark.repetitions > 1;
232
233 for (const auto& Stat : *benchmark.statistics)
234 stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size());
235 }
236 if (might_have_aggregates) name_field_width += 1 + stat_field_width;
237
238 // Print header here
239 BenchmarkReporter::Context context;
240 context.name_field_width = name_field_width;
241
242 // Keep track of running times of all instances of current benchmark
243 std::vector<BenchmarkReporter::Run> complexity_reports;
244
245 // We flush streams after invoking reporter methods that write to them. This
246 // ensures users get timely updates even when streams are not line-buffered.
247 auto flushStreams = [](BenchmarkReporter* reporter) {
248 if (!reporter) return;
249 std::flush(reporter->GetOutputStream());
250 std::flush(reporter->GetErrorStream());
251 };
252
253 if (display_reporter->ReportContext(context) &&
254 (!file_reporter || file_reporter->ReportContext(context))) {
255 flushStreams(display_reporter);
256 flushStreams(file_reporter);
257
258 for (const auto& benchmark : benchmarks) {
259 RunResults run_results = RunBenchmark(benchmark, &complexity_reports);
260
261 auto report = [&run_results](BenchmarkReporter* reporter,
262 bool report_aggregates_only) {
263 assert(reporter);
264 // If there are no aggregates, do output non-aggregates.
265 report_aggregates_only &= !run_results.aggregates_only.empty();
266 if (!report_aggregates_only)
267 reporter->ReportRuns(run_results.non_aggregates);
268 if (!run_results.aggregates_only.empty())
269 reporter->ReportRuns(run_results.aggregates_only);
270 };
271
272 report(display_reporter, run_results.display_report_aggregates_only);
273 if (file_reporter)
274 report(file_reporter, run_results.file_report_aggregates_only);
275
276 flushStreams(display_reporter);
277 flushStreams(file_reporter);
278 }
279 }
280 display_reporter->Finalize();
281 if (file_reporter) file_reporter->Finalize();
282 flushStreams(display_reporter);
283 flushStreams(file_reporter);
284}
285
286// Disable deprecated warnings temporarily because we need to reference
287// CSVReporter but don't want to trigger -Werror=-Wdeprecated
288#ifdef __GNUC__
289#pragma GCC diagnostic push
290#pragma GCC diagnostic ignored "-Wdeprecated"
291#endif
292
293std::unique_ptr<BenchmarkReporter> CreateReporter(
294 std::string const& name, ConsoleReporter::OutputOptions output_opts) {
295 typedef std::unique_ptr<BenchmarkReporter> PtrType;
296 if (name == "console") {
297 return PtrType(new ConsoleReporter(output_opts));
298 } else if (name == "json") {
299 return PtrType(new JSONReporter);
300 } else if (name == "csv") {
301 return PtrType(new CSVReporter);
302 } else {
303 std::cerr << "Unexpected format: '" << name << "'\n";
304 std::exit(1);
305 }
306}
307
308#ifdef __GNUC__
309#pragma GCC diagnostic pop
310#endif
311
312} // end namespace
313
314bool IsZero(double n) {
315 return std::abs(n) < std::numeric_limits<double>::epsilon();
316}
317
318ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) {
319 int output_opts = ConsoleReporter::OO_Defaults;
320 auto is_benchmark_color = [force_no_color]() -> bool {
321 if (force_no_color) {
322 return false;
323 }
324 if (FLAGS_benchmark_color == "auto") {
325 return IsColorTerminal();
326 }
327 return IsTruthyFlagValue(FLAGS_benchmark_color);
328 };
329 if (is_benchmark_color()) {
330 output_opts |= ConsoleReporter::OO_Color;
331 } else {
332 output_opts &= ~ConsoleReporter::OO_Color;
333 }
334 if (FLAGS_benchmark_counters_tabular) {
335 output_opts |= ConsoleReporter::OO_Tabular;
336 } else {
337 output_opts &= ~ConsoleReporter::OO_Tabular;
338 }
339 return static_cast<ConsoleReporter::OutputOptions>(output_opts);
340}
341
342} // end namespace internal
343
344size_t RunSpecifiedBenchmarks() {
345 return RunSpecifiedBenchmarks(nullptr, nullptr);
346}
347
348size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) {
349 return RunSpecifiedBenchmarks(display_reporter, nullptr);
350}
351
352size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter,
353 BenchmarkReporter* file_reporter) {
354 std::string spec = FLAGS_benchmark_filter;
355 if (spec.empty() || spec == "all")
356 spec = "."; // Regexp that matches all benchmarks
357
358 // Setup the reporters
359 std::ofstream output_file;
360 std::unique_ptr<BenchmarkReporter> default_display_reporter;
361 std::unique_ptr<BenchmarkReporter> default_file_reporter;
362 if (!display_reporter) {
363 default_display_reporter = internal::CreateReporter(
364 FLAGS_benchmark_format, internal::GetOutputOptions());
365 display_reporter = default_display_reporter.get();
366 }
367 auto& Out = display_reporter->GetOutputStream();
368 auto& Err = display_reporter->GetErrorStream();
369
370 std::string const& fname = FLAGS_benchmark_out;
371 if (fname.empty() && file_reporter) {
372 Err << "A custom file reporter was provided but "
373 "--benchmark_out=<file> was not specified."
374 << std::endl;
375 std::exit(1);
376 }
377 if (!fname.empty()) {
378 output_file.open(fname);
379 if (!output_file.is_open()) {
380 Err << "invalid file name: '" << fname << std::endl;
381 std::exit(1);
382 }
383 if (!file_reporter) {
384 default_file_reporter = internal::CreateReporter(
385 FLAGS_benchmark_out_format, ConsoleReporter::OO_None);
386 file_reporter = default_file_reporter.get();
387 }
388 file_reporter->SetOutputStream(&output_file);
389 file_reporter->SetErrorStream(&output_file);
390 }
391
392 std::vector<internal::BenchmarkInstance> benchmarks;
393 if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0;
394
395 if (benchmarks.empty()) {
396 Err << "Failed to match any benchmarks against regex: " << spec << "\n";
397 return 0;
398 }
399
400 if (FLAGS_benchmark_list_tests) {
401 for (auto const& benchmark : benchmarks)
402 Out << benchmark.name.str() << "\n";
403 } else {
404 internal::RunBenchmarks(benchmarks, display_reporter, file_reporter);
405 }
406
407 return benchmarks.size();
408}
409
410void RegisterMemoryManager(MemoryManager* manager) {
411 internal::memory_manager = manager;
412}
413
414namespace internal {
415
416void PrintUsageAndExit() {
417 fprintf(stdout,
418 "benchmark"
419 " [--benchmark_list_tests={true|false}]\n"
420 " [--benchmark_filter=<regex>]\n"
421 " [--benchmark_min_time=<min_time>]\n"
422 " [--benchmark_repetitions=<num_repetitions>]\n"
423 " [--benchmark_report_aggregates_only={true|false}]\n"
424 " [--benchmark_display_aggregates_only={true|false}]\n"
425 " [--benchmark_format=<console|json|csv>]\n"
426 " [--benchmark_out=<filename>]\n"
427 " [--benchmark_out_format=<json|console|csv>]\n"
428 " [--benchmark_color={auto|true|false}]\n"
429 " [--benchmark_counters_tabular={true|false}]\n"
430 " [--v=<verbosity>]\n");
431 exit(0);
432}
433
434void ParseCommandLineFlags(int* argc, char** argv) {
435 using namespace benchmark;
436 BenchmarkReporter::Context::executable_name =
437 (argc && *argc > 0) ? argv[0] : "unknown";
438 for (int i = 1; argc && i < *argc; ++i) {
439 if (ParseBoolFlag(argv[i], "benchmark_list_tests",
440 &FLAGS_benchmark_list_tests) ||
441 ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) ||
442 ParseDoubleFlag(argv[i], "benchmark_min_time",
443 &FLAGS_benchmark_min_time) ||
444 ParseInt32Flag(argv[i], "benchmark_repetitions",
445 &FLAGS_benchmark_repetitions) ||
446 ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
447 &FLAGS_benchmark_report_aggregates_only) ||
448 ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
449 &FLAGS_benchmark_display_aggregates_only) ||
450 ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
451 ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
452 ParseStringFlag(argv[i], "benchmark_out_format",
453 &FLAGS_benchmark_out_format) ||
454 ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) ||
455 // "color_print" is the deprecated name for "benchmark_color".
456 // TODO: Remove this.
457 ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) ||
458 ParseBoolFlag(argv[i], "benchmark_counters_tabular",
459 &FLAGS_benchmark_counters_tabular) ||
460 ParseInt32Flag(argv[i], "v", &FLAGS_v)) {
461 for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1];
462
463 --(*argc);
464 --i;
465 } else if (IsFlag(argv[i], "help")) {
466 PrintUsageAndExit();
467 }
468 }
469 for (auto const* flag :
470 {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format})
471 if (*flag != "console" && *flag != "json" && *flag != "csv") {
472 PrintUsageAndExit();
473 }
474 if (FLAGS_benchmark_color.empty()) {
475 PrintUsageAndExit();
476 }
477}
478
479int InitializeStreams() {
480 static std::ios_base::Init init;
481 return 0;
482}
483
484} // end namespace internal
485
486void Initialize(int* argc, char** argv) {
487 internal::ParseCommandLineFlags(argc, argv);
488 internal::LogLevel() = FLAGS_v;
489}
490
491bool ReportUnrecognizedArguments(int argc, char** argv) {
492 for (int i = 1; i < argc; ++i) {
493 fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0],
494 argv[i]);
495 }
496 return argc > 1;
497}
498
499} // end namespace benchmark
500