1/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15
16#ifndef TENSORFLOW_TSL_UTIL_REPORTER_H_
17#define TENSORFLOW_TSL_UTIL_REPORTER_H_
18
19#include <cstdlib>
20#include <memory>
21#include <string>
22#include <unordered_set>
23
24#include "tensorflow/core/util/test_log.pb.h"
25#include "tensorflow/tsl/platform/env.h"
26#include "tensorflow/tsl/platform/macros.h"
27#include "tensorflow/tsl/platform/mutex.h"
28#include "tensorflow/tsl/platform/types.h"
29
30namespace tsl {
31
32// The TestReportFile provides a file abstraction for TF tests to use.
33class TestReportFile {
34 public:
35 // Create a TestReportFile with the test name 'test_name'.
36 TestReportFile(const string& fname, const string& test_name);
37
38 // Initialize the TestReportFile. If the reporting env flag is set,
39 // try to create the reporting file. Fails if the file already exists.
40 Status Initialize();
41
42 // Append the report file w/ 'content'.
43 Status Append(const string& content);
44
45 // Close the report file.
46 Status Close();
47
48 bool IsClosed() const { return closed_; }
49
50 ~TestReportFile() { Close().IgnoreError(); } // Autoclose in destructor.
51
52 private:
53 bool closed_;
54 string fname_;
55 string test_name_;
56 std::unique_ptr<WritableFile> log_file_;
57 TF_DISALLOW_COPY_AND_ASSIGN(TestReportFile);
58};
59
60// The TestReporter writes test / benchmark output to binary Protobuf files when
61// the environment variable "TEST_REPORT_FILE_PREFIX" is defined.
62//
63// If this environment variable is not defined, no logging is performed.
64//
65// The intended use is via the following lines:
66//
67// TestReporter reporter(test_name);
68// TF_CHECK_OK(reporter.Initialize()));
69// TF_CHECK_OK(reporter.Benchmark(iters, cpu_time, wall_time, throughput));
70// TF_CHECK_OK(reporter.SetProperty("some_string_property", "some_value");
71// TF_CHECK_OK(reporter.SetProperty("some_double_property", double_value);
72// TF_CHECK_OK(reporter.Close());
73//
74// For example, if the environment variable
75// TEST_REPORT_FILE_PREFIX="/tmp/run_"
76// is set, and test_name is "BM_Foo/1/2", then a BenchmarkEntries pb
77// with a single entry is written to file:
78// /tmp/run_BM_Foo__1__2
79//
80class TestReporter {
81 public:
82 static constexpr const char* kTestReporterEnv = "TEST_REPORT_FILE_PREFIX";
83
84 // Create a TestReporter with the test name 'test_name'.
85 explicit TestReporter(const string& test_name)
86 : TestReporter(GetLogEnv(), test_name) {}
87
88 // Provide a prefix filename, mostly used for testing this class.
89 TestReporter(const string& fname, const string& test_name);
90
91 // Initialize the TestReporter. If the reporting env flag is set,
92 // try to create the reporting file. Fails if the file already exists.
93 Status Initialize();
94
95 // Finalize the report. If the reporting env flag is set,
96 // flush the reporting file and close it.
97 // Once Close is called, no other methods should be called other
98 // than Close and the destructor.
99 Status Close();
100
101 // Set the report to be a Benchmark and log the given parameters.
102 // Only does something if the reporting env flag is set.
103 // Does not guarantee the report is written. Use Close() to
104 // enforce I/O operations.
105 Status Benchmark(int64_t iters, double cpu_time, double wall_time,
106 double throughput);
107
108 // Set property on Benchmark to the given value.
109 Status SetProperty(const string& name, double value);
110
111 // Set property on Benchmark to the given value.
112 Status SetProperty(const string& name, const string& value);
113
114 // Add the given value to the metrics on the Benchmark.
115 Status AddMetric(const string& name, double value);
116
117 // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object!
118 ~TestReporter() { Close().IgnoreError(); } // Autoclose in destructor.
119
120 private:
121 static string GetLogEnv() {
122 const char* fname_ptr = getenv(kTestReporterEnv);
123 return (fname_ptr != nullptr) ? fname_ptr : "";
124 }
125 TestReportFile report_file_;
126 tensorflow::BenchmarkEntry benchmark_entry_;
127 TF_DISALLOW_COPY_AND_ASSIGN(TestReporter);
128};
129
130} // namespace tsl
131
132#endif // TENSORFLOW_TSL_UTIL_REPORTER_H_
133