1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | |
16 | #include "tensorflow/core/util/events_writer.h" |
17 | |
18 | #include <stddef.h> // for NULL |
19 | |
20 | #include "tensorflow/core/lib/core/errors.h" |
21 | #include "tensorflow/core/lib/core/status.h" |
22 | #include "tensorflow/core/lib/io/path.h" |
23 | #include "tensorflow/core/lib/strings/strcat.h" |
24 | #include "tensorflow/core/lib/strings/stringprintf.h" |
25 | #include "tensorflow/core/platform/env.h" |
26 | #include "tensorflow/core/platform/host_info.h" |
27 | #include "tensorflow/core/platform/logging.h" |
28 | #include "tensorflow/core/platform/types.h" |
29 | #include "tensorflow/core/util/event.pb.h" |
30 | |
31 | namespace tensorflow { |
32 | |
33 | EventsWriter::EventsWriter(const string& file_prefix) |
34 | // TODO(jeff,sanjay): Pass in env and use that here instead of Env::Default |
35 | : env_(Env::Default()), |
36 | file_prefix_(file_prefix), |
37 | num_outstanding_events_(0) {} |
38 | |
39 | EventsWriter::~EventsWriter() { |
40 | Close().IgnoreError(); // Autoclose in destructor. |
41 | } |
42 | |
43 | Status EventsWriter::Init() { return InitWithSuffix("" ); } |
44 | |
45 | Status EventsWriter::InitWithSuffix(const string& suffix) { |
46 | file_suffix_ = suffix; |
47 | return InitIfNeeded(); |
48 | } |
49 | |
50 | Status EventsWriter::InitIfNeeded() { |
51 | if (recordio_writer_ != nullptr) { |
52 | CHECK(!filename_.empty()); |
53 | if (!FileStillExists().ok()) { |
54 | // Warn user of data loss and let .reset() below do basic cleanup. |
55 | if (num_outstanding_events_ > 0) { |
56 | LOG(WARNING) << "Re-initialization, attempting to open a new file, " |
57 | << num_outstanding_events_ << " events will be lost." ; |
58 | } |
59 | } else { |
60 | // No-op: File is present and writer is initialized. |
61 | return OkStatus(); |
62 | } |
63 | } |
64 | |
65 | int64_t time_in_seconds = env_->NowMicros() / 1000000; |
66 | |
67 | filename_ = |
68 | strings::Printf("%s.out.tfevents.%010lld.%s%s" , file_prefix_.c_str(), |
69 | static_cast<long long>(time_in_seconds), |
70 | port::Hostname().c_str(), file_suffix_.c_str()); |
71 | |
72 | // Reset recordio_writer (which has a reference to recordio_file_) so final |
73 | // Flush() and Close() call have access to recordio_file_. |
74 | recordio_writer_.reset(); |
75 | |
76 | TF_RETURN_WITH_CONTEXT_IF_ERROR( |
77 | env_->NewWritableFile(filename_, &recordio_file_), |
78 | "Creating writable file " , filename_); |
79 | recordio_writer_.reset(new io::RecordWriter(recordio_file_.get())); |
80 | if (recordio_writer_ == nullptr) { |
81 | return errors::Unknown("Could not create record writer" ); |
82 | } |
83 | num_outstanding_events_ = 0; |
84 | VLOG(1) << "Successfully opened events file: " << filename_; |
85 | { |
86 | // Write the first event with the current version, and flush |
87 | // right away so the file contents will be easily determined. |
88 | |
89 | Event event; |
90 | event.set_wall_time(time_in_seconds); |
91 | event.set_file_version(strings::StrCat(kVersionPrefix, kCurrentVersion)); |
92 | WriteEvent(event); |
93 | TF_RETURN_WITH_CONTEXT_IF_ERROR(Flush(), "Flushing first event." ); |
94 | } |
95 | return OkStatus(); |
96 | } |
97 | |
98 | string EventsWriter::FileName() { |
99 | if (filename_.empty()) { |
100 | InitIfNeeded().IgnoreError(); |
101 | } |
102 | return filename_; |
103 | } |
104 | |
105 | void EventsWriter::WriteSerializedEvent(StringPiece event_str) { |
106 | if (recordio_writer_ == nullptr) { |
107 | if (!InitIfNeeded().ok()) { |
108 | LOG(ERROR) << "Write failed because file could not be opened." ; |
109 | return; |
110 | } |
111 | } |
112 | num_outstanding_events_++; |
113 | recordio_writer_->WriteRecord(event_str).IgnoreError(); |
114 | } |
115 | |
116 | // NOTE(touts); This is NOT the function called by the Python code. |
117 | // Python calls WriteSerializedEvent(), see events_writer.i. |
118 | void EventsWriter::WriteEvent(const Event& event) { |
119 | string record; |
120 | event.AppendToString(&record); |
121 | WriteSerializedEvent(record); |
122 | } |
123 | |
124 | Status EventsWriter::Flush() { |
125 | if (num_outstanding_events_ == 0) return OkStatus(); |
126 | CHECK(recordio_file_ != nullptr) << "Unexpected NULL file" ; |
127 | |
128 | TF_RETURN_WITH_CONTEXT_IF_ERROR(recordio_writer_->Flush(), "Failed to flush " , |
129 | num_outstanding_events_, " events to " , |
130 | filename_); |
131 | TF_RETURN_WITH_CONTEXT_IF_ERROR(recordio_file_->Sync(), "Failed to sync " , |
132 | num_outstanding_events_, " events to " , |
133 | filename_); |
134 | VLOG(1) << "Wrote " << num_outstanding_events_ << " events to disk." ; |
135 | num_outstanding_events_ = 0; |
136 | return OkStatus(); |
137 | } |
138 | |
139 | Status EventsWriter::Close() { |
140 | Status status = Flush(); |
141 | if (recordio_file_ != nullptr) { |
142 | Status close_status = recordio_file_->Close(); |
143 | if (!close_status.ok()) { |
144 | status = close_status; |
145 | } |
146 | recordio_writer_.reset(nullptr); |
147 | recordio_file_.reset(nullptr); |
148 | } |
149 | num_outstanding_events_ = 0; |
150 | return status; |
151 | } |
152 | |
153 | Status EventsWriter::FileStillExists() { |
154 | if (env_->FileExists(filename_).ok()) { |
155 | return OkStatus(); |
156 | } |
157 | // This can happen even with non-null recordio_writer_ if some other |
158 | // process has removed the file. |
159 | return errors::Unknown("The events file " , filename_, " has disappeared." ); |
160 | } |
161 | |
162 | } // namespace tensorflow |
163 | |