1/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15#ifndef TENSORFLOW_LITE_CORE_API_PROFILER_H_
16#define TENSORFLOW_LITE_CORE_API_PROFILER_H_
17
18#include <cstdint>
19
20namespace tflite {
21
22// A simple utility for enabling profiled event tracing in TensorFlow Lite.
23class Profiler {
24 public:
25 // As certain Profiler instance might be only interested in certain event
26 // types, we define each event type value to allow a Profiler to use
27 // bitmasking bitwise operations to determine whether an event should be
28 // recorded or not.
29 enum class EventType {
30 // Default event type, the metadata field has no special significance.
31 DEFAULT = 1,
32
33 // The event is an operator invocation and the event_metadata field is the
34 // index of operator node.
35 OPERATOR_INVOKE_EVENT = 1 << 1,
36
37 // The event is an invocation for an internal operator of a TFLite delegate.
38 // The event_metadata field is the index of operator node that's specific to
39 // the delegate.
40 DELEGATE_OPERATOR_INVOKE_EVENT = 1 << 2,
41
42 // The event is a recording of runtime instrumentation such as the overall
43 // TFLite runtime status, the TFLite delegate status (if a delegate
44 // is applied), and the overall model inference latency etc.
45 // Note, the delegate status and overall status are stored as separate
46 // event_metadata fields. In particular, the delegate status is encoded
47 // as DelegateStatus::full_status().
48 GENERAL_RUNTIME_INSTRUMENTATION_EVENT = 1 << 3,
49
50 // Telemetry events. Users and code instrumentations should invoke Telemetry
51 // calls instead of using the following types directly.
52 // See experimental/telemetry:profiler for definition of each metadata.
53 //
54 // A telemetry event that reports model and interpreter level events.
55 TELEMETRY_EVENT = 1 << 4,
56 // A telemetry event that reports model and interpreter level settings.
57 TELEMETRY_REPORT_SETTINGS = 1 << 5,
58 // A telemetry event that reports delegate level events.
59 TELEMETRY_DELEGATE_EVENT = 1 << 6,
60 // A telemetry event that reports delegate settings.
61 TELEMETRY_DELEGATE_REPORT_SETTINGS = 1 << 7,
62 };
63
64 virtual ~Profiler() {}
65
66 // Signals the beginning of an event and returns a handle to the profile
67 // event. The `event_metadata1` and `event_metadata2` have different
68 // interpretations based on the actual Profiler instance and the `event_type`.
69 // For example, as for the 'SubgraphAwareProfiler' defined in
70 // lite/core/subgraph.h, when the event_type is OPERATOR_INVOKE_EVENT,
71 // `event_metadata1` represents the index of a TFLite node, and
72 // `event_metadata2` represents the index of the subgraph that this event
73 // comes from.
74 virtual uint32_t BeginEvent(const char* tag, EventType event_type,
75 int64_t event_metadata1,
76 int64_t event_metadata2) = 0;
77 // Similar w/ the above, but `event_metadata2` defaults to 0.
78 uint32_t BeginEvent(const char* tag, EventType event_type,
79 int64_t event_metadata) {
80 return BeginEvent(tag, event_type, event_metadata, /*event_metadata2*/ 0);
81 }
82
83 // Signals an end to the specified profile event with 'event_metadata's, This
84 // is useful when 'event_metadata's are not available when the event begins
85 // or when one wants to overwrite the 'event_metadata's set at the beginning.
86 virtual void EndEvent(uint32_t event_handle, int64_t event_metadata1,
87 int64_t event_metadata2) {
88 // By default discards the metadata.
89 EndEvent(event_handle);
90 }
91 // Signals an end to the specified profile event.
92 virtual void EndEvent(uint32_t event_handle) = 0;
93
94 // Appends an event of type 'event_type' with 'tag' and 'event_metadata'
95 // which ran for elapsed_time.
96 // Note:
97 // In cases were ProfileSummarizer and tensorflow::StatsCalculator are used
98 // they assume the value is in "usec", if in any case subclasses
99 // didn't put usec, then the values are not meaningful.
100 // TODO(karimnosseir): karimnosseir: Revisit and make the function more clear.
101 void AddEvent(const char* tag, EventType event_type, uint64_t elapsed_time,
102 int64_t event_metadata) {
103 AddEvent(tag, event_type, elapsed_time, event_metadata,
104 /*event_metadata2*/ 0);
105 }
106
107 // Adds a profiler event.
108 // `metric` field has different intreptation based on `event_type`.
109 // e.g. it means elapsed time for [DELEGATE_]OPERATOR_INVOKE_EVENT types,
110 // and interprets as source and status code for TELEMETRY_[DELEGATE_]EVENT
111 // event types. If the concrete profiler does not provide an implementation,
112 // does nothing.
113 // TODO(b/241982974): Clean up dependencies and make it pure virtual.
114 virtual void AddEvent(const char* tag, EventType event_type, uint64_t metric,
115 int64_t event_metadata1, int64_t event_metadata2) {}
116
117 // Adds a profiler event with data.
118 // Data will be a const TelemetrySettings* for TELEMETRY_REPORT_SETTINGS
119 // and TELEMETRY_DELEGATE_REPORT_SETTINGS.
120 // If the concrete profiler does not provide an implementation, does nothing.
121 // TODO(b/241982974): Clean up dependencies and make it pure virtual.
122 virtual void AddEventWithData(const char* tag, EventType event_type,
123 const void* data) {}
124
125 protected:
126 friend class ScopedProfile;
127};
128
129// Adds a profile event to `profiler` that begins with the construction
130// of the object and ends when the object goes out of scope.
131// The lifetime of tag should be at least the lifetime of `profiler`.
132// `profiler` may be null, in which case nothing is profiled.
133class ScopedProfile {
134 public:
135 ScopedProfile(Profiler* profiler, const char* tag,
136 Profiler::EventType event_type = Profiler::EventType::DEFAULT,
137 int64_t event_metadata = 0)
138 : profiler_(profiler), event_handle_(0) {
139 if (profiler) {
140 event_handle_ = profiler_->BeginEvent(tag, event_type, event_metadata);
141 }
142 }
143
144 ~ScopedProfile() {
145 if (profiler_) {
146 profiler_->EndEvent(event_handle_);
147 }
148 }
149
150 protected:
151 Profiler* profiler_;
152 uint32_t event_handle_;
153};
154
155class ScopedOperatorProfile : public ScopedProfile {
156 public:
157 ScopedOperatorProfile(Profiler* profiler, const char* tag, int node_index)
158 : ScopedProfile(profiler, tag, Profiler::EventType::OPERATOR_INVOKE_EVENT,
159 static_cast<uint32_t>(node_index)) {}
160};
161
162class ScopedDelegateOperatorProfile : public ScopedProfile {
163 public:
164 ScopedDelegateOperatorProfile(Profiler* profiler, const char* tag,
165 int node_index)
166 : ScopedProfile(profiler, tag,
167 Profiler::EventType::DELEGATE_OPERATOR_INVOKE_EVENT,
168 static_cast<uint32_t>(node_index)) {}
169};
170
171// Similar to ScopedProfile but has extra event metadata for EndEvent.
172class ScopedRuntimeInstrumentationProfile {
173 public:
174 ScopedRuntimeInstrumentationProfile(Profiler* profiler, const char* tag)
175 : profiler_(profiler), event_handle_(0) {
176 if (profiler) {
177 event_handle_ = profiler_->BeginEvent(
178 tag, Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT,
179 /*event_metadata=*/-1);
180 }
181 }
182
183 void set_runtime_status(int64_t delegate_status, int64_t interpreter_status) {
184 if (profiler_) {
185 delegate_status_ = delegate_status;
186 interpreter_status_ = interpreter_status;
187 }
188 }
189
190 ~ScopedRuntimeInstrumentationProfile() {
191 if (profiler_) {
192 profiler_->EndEvent(event_handle_, delegate_status_, interpreter_status_);
193 }
194 }
195
196 private:
197 Profiler* profiler_ = nullptr;
198 uint32_t event_handle_ = 0;
199 int64_t delegate_status_ = 0;
200 int64_t interpreter_status_ = 0;
201};
202
203} // namespace tflite
204
205#define TFLITE_VARNAME_UNIQ_IMPL(name, ctr) name##ctr
206#define TFLITE_VARNAME_UNIQ(name, ctr) TFLITE_VARNAME_UNIQ_IMPL(name, ctr)
207
208#define TFLITE_SCOPED_TAGGED_DEFAULT_PROFILE(profiler, tag) \
209 tflite::ScopedProfile TFLITE_VARNAME_UNIQ(_profile_, __COUNTER__)( \
210 (profiler), (tag))
211
212#define TFLITE_SCOPED_TAGGED_OPERATOR_PROFILE(profiler, tag, node_index) \
213 tflite::ScopedOperatorProfile TFLITE_VARNAME_UNIQ(_profile_, __COUNTER__)( \
214 (profiler), (tag), (node_index))
215
216#define TFLITE_SCOPED_DELEGATE_OPERATOR_PROFILE(profiler, tag, node_index) \
217 tflite::ScopedDelegateOperatorProfile TFLITE_VARNAME_UNIQ( \
218 _profile_, __COUNTER__)((profiler), (tag), (node_index))
219
220#define TFLITE_ADD_RUNTIME_INSTRUMENTATION_EVENT( \
221 profiler, tag, event_metadata1, event_metadata2) \
222 do { \
223 if (profiler) { \
224 const auto handle = profiler->BeginEvent( \
225 tag, Profiler::EventType::GENERAL_RUNTIME_INSTRUMENTATION_EVENT, \
226 event_metadata1, event_metadata2); \
227 profiler->EndEvent(handle); \
228 } \
229 } while (false);
230
231#endif // TENSORFLOW_LITE_CORE_API_PROFILER_H_
232