1 | /* Copyright 2015 The TensorFlow Authors. All Rights Reserved. |
2 | |
3 | Licensed under the Apache License, Version 2.0 (the "License"); |
4 | you may not use this file except in compliance with the License. |
5 | You may obtain a copy of the License at |
6 | |
7 | http://www.apache.org/licenses/LICENSE-2.0 |
8 | |
9 | Unless required by applicable law or agreed to in writing, software |
10 | distributed under the License is distributed on an "AS IS" BASIS, |
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | See the License for the specific language governing permissions and |
13 | limitations under the License. |
14 | ==============================================================================*/ |
15 | |
16 | #ifndef TENSORFLOW_CORE_KERNELS_SOFTPLUS_OP_H_ |
17 | #define TENSORFLOW_CORE_KERNELS_SOFTPLUS_OP_H_ |
18 | // Functor definition for SoftplusOp and SoftplusGradOp, must be compilable by |
19 | // nvcc. |
20 | |
21 | // clang-format off |
22 | #include "tensorflow/core/platform/bfloat16.h" |
23 | #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" |
24 | // clang-format on |
25 | #include "tensorflow/core/framework/tensor_types.h" |
26 | |
27 | namespace tensorflow { |
28 | namespace functor { |
29 | |
30 | // Functor used by SoftplusOp to do the computations. |
31 | template <typename Device, typename T> |
32 | struct Softplus { |
33 | // Computes Softplus activation. |
34 | // |
35 | // features: any shape. |
36 | // activations: same shape as "features". |
37 | void operator()(const Device& d, typename TTypes<T>::ConstTensor features, |
38 | typename TTypes<T>::Tensor activations) { |
39 | // Choose a threshold on x below which exp(x) may underflow |
40 | // when added to 1, but for which exp(x) is always within epsilon of the |
41 | // true softplus(x). Offset of 2 from machine epsilon checked |
42 | // experimentally for float16, float32, float64. Checked against |
43 | // softplus implemented with numpy's log1p and numpy's logaddexp. |
44 | static const T threshold = |
45 | Eigen::numext::log(Eigen::NumTraits<T>::epsilon()) + T(2); |
46 | // Value above which exp(x) may overflow, but softplus(x) == x |
47 | // is within machine epsilon. |
48 | auto too_large = features > features.constant(-threshold); |
49 | // Value below which exp(x) may underflow, but softplus(x) == exp(x) |
50 | // is within machine epsilon. |
51 | auto too_small = features < features.constant(threshold); |
52 | auto features_exp = features.exp(); |
53 | activations.device(d) = too_large.select( |
54 | features, // softplus(x) ~= x for x large |
55 | too_small.select(features_exp, // softplus(x) ~= exp(x) for x small |
56 | features_exp.log1p())); |
57 | } |
58 | }; |
59 | |
60 | // Functor used by SoftplusGradOp to do the computations. |
61 | template <typename Device, typename T> |
62 | struct SoftplusGrad { |
63 | // Computes SoftplusGrad backprops. |
64 | // |
65 | // gradients: gradients backpropagated to the Softplus op. |
66 | // features: inputs that where passed to the Softplus op. |
67 | // backprops: gradients to backpropagate to the Softplus inputs. |
68 | void operator()(const Device& d, typename TTypes<T>::ConstTensor gradients, |
69 | typename TTypes<T>::ConstTensor features, |
70 | typename TTypes<T>::Tensor backprops) { |
71 | backprops.device(d) = |
72 | gradients / ((-features).exp() + features.constant(T(1))); |
73 | } |
74 | }; |
75 | |
76 | } // namespace functor |
77 | } // namespace tensorflow |
78 | |
79 | #endif // TENSORFLOW_CORE_KERNELS_SOFTPLUS_OP_H_ |
80 | |