1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | |
16 | #include "tensorflow/core/common_runtime/allocator_retry.h" |
17 | |
18 | #include "absl/types/optional.h" |
19 | #include "tensorflow/core/framework/metrics.h" |
20 | #include "tensorflow/core/platform/env.h" |
21 | #include "tensorflow/core/platform/logging.h" |
22 | #include "tensorflow/core/platform/mutex.h" |
23 | #include "tensorflow/core/platform/types.h" |
24 | |
25 | namespace tensorflow { |
26 | |
27 | namespace { |
28 | class ScopedTimeTracker { |
29 | public: |
30 | explicit ScopedTimeTracker(Env* env) : env_(env) {} |
31 | void Enable() { |
32 | if (!start_us_) { // Only override start_us when not set yet. |
33 | start_us_ = env_->NowMicros(); |
34 | } |
35 | } |
36 | ~ScopedTimeTracker() { |
37 | if (start_us_) { |
38 | uint64 end_us = env_->NowMicros(); |
39 | metrics::UpdateBfcAllocatorDelayTime(end_us - *start_us_); |
40 | } |
41 | } |
42 | |
43 | private: |
44 | Env* env_; |
45 | absl::optional<uint64> start_us_; |
46 | }; |
47 | } // namespace |
48 | |
49 | AllocatorRetry::AllocatorRetry() : env_(Env::Default()) {} |
50 | |
51 | void* AllocatorRetry::AllocateRaw( |
52 | std::function<void*(size_t alignment, size_t num_bytes, |
53 | bool verbose_failure)> |
54 | alloc_func, |
55 | int max_millis_to_wait, size_t alignment, size_t num_bytes) { |
56 | if (num_bytes == 0) { |
57 | return nullptr; |
58 | } |
59 | ScopedTimeTracker tracker(env_); |
60 | uint64 deadline_micros = 0; |
61 | bool first = true; |
62 | void* ptr = nullptr; |
63 | while (ptr == nullptr) { |
64 | ptr = alloc_func(alignment, num_bytes, false); |
65 | if (ptr == nullptr) { |
66 | uint64 now = env_->NowMicros(); |
67 | if (first) { |
68 | deadline_micros = now + max_millis_to_wait * 1000; |
69 | first = false; |
70 | } |
71 | if (now < deadline_micros) { |
72 | tracker.Enable(); |
73 | mutex_lock l(mu_); |
74 | WaitForMilliseconds(&l, &memory_returned_, |
75 | (deadline_micros - now) / 1000); |
76 | } else { |
77 | return alloc_func(alignment, num_bytes, true); |
78 | } |
79 | } |
80 | } |
81 | return ptr; |
82 | } |
83 | |
84 | } // namespace tensorflow |
85 | |