1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "BackendTestUtils.h"
18
19#include "glow/Base/Tensor.h"
20#include "glow/ExecutionEngine/ExecutionEngine.h"
21#include "glow/Graph/Graph.h"
22#include "glow/Graph/Nodes.h"
23
24#include "gtest/gtest.h"
25
26using namespace glow;
27
28class OperatorGradTest : public BackendTest {
29protected:
30 /// Compute gradients of placeholders in bindings_. Given
31 /// outputNode, representing H(Vars), this function will train H(vars) to be
32 /// 0. It is achieved by creating RegressionNode between outputNode and 0,
33 /// which will minimize F, a total squared error of H divided by 2.
34 ///
35 /// Note that gradient value at the start of backpropagation is the same as
36 /// outputNode's forward value, because RegressionNode's grad is outputNode-0.
37 ///
38 /// \param outputNode Node that contains result of H(Vars).
39 VariableGradientsList computeVarGrads(NodeValue outputNode) {
40 auto *Exp = mod_.createPlaceholder(ElemKind::FloatTy, outputNode.dims(),
41 "exp", false);
42 bindings_.allocate(Exp)->zero();
43
44 auto *reg = F_->createRegression("reg", outputNode, Exp);
45 auto *result = F_->createSave("ret", reg);
46 bindings_.allocate(result->getPlaceholder());
47
48 // Create a version of the network that records the gradients to some side
49 // table instead of updating them.
50 VariableGradientsList varGrads;
51 TrainingConfig TC;
52 Function *recordNet = glow::differentiate(F_, TC, "record", &varGrads);
53 auto recordName = recordNet->getName();
54 allocateGrads(varGrads);
55 EE_.compile(CompilationMode::Train);
56
57 // Train the network just once to record the values of gradient for
58 // all variables.
59 EE_.run(bindings_, recordName);
60
61 return varGrads;
62 }
63
64 Tensor *getGradTensor(const VariableGradientsList &grads, Placeholder *V) {
65 for (auto &p : grads) {
66 if (p.first == V) {
67 return bindings_.get(p.second);
68 }
69 }
70 return nullptr;
71 }
72
73 void allocateGrads(const VariableGradientsList &grads) {
74 for (auto &p : grads) {
75 auto grad = p.second;
76 bindings_.allocate(grad);
77 }
78 }
79
80 PlaceholderBindings bindings_;
81};
82
83TEST_P(OperatorGradTest, concat) {
84 CHECK_IF_ENABLED();
85
86 dim_t numOutputElem = 4;
87
88 auto *A = mod_.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2},
89 "A", false);
90 bindings_.allocate(A)->getHandle() = {1, 2};
91 auto *B = mod_.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2},
92 "B", false);
93 bindings_.allocate(B)->getHandle() = {3, 4};
94
95 Node *O = F_->createConcat("concat", {A, B}, 1);
96 VariableGradientsList varGrads = computeVarGrads(O);
97
98 Tensor expectedLeft(ElemKind::FloatTy, {1, numOutputElem / 2});
99 expectedLeft.getHandle() = {1, 2};
100 EXPECT_TRUE(expectedLeft.isEqual(*getGradTensor(varGrads, A)));
101
102 Tensor expectedRight(ElemKind::FloatTy, {1, numOutputElem / 2});
103 expectedRight.getHandle() = {3, 4};
104 EXPECT_TRUE(expectedRight.isEqual(*getGradTensor(varGrads, B)));
105}
106
107TEST_P(OperatorGradTest, fc) {
108 CHECK_IF_ENABLED();
109
110 auto *x = mod_.createPlaceholder(ElemKind::FloatTy, {1, 2}, "x", false);
111 bindings_.allocate(x)->getHandle() = {1, 2};
112 auto *W = mod_.createPlaceholder(ElemKind::FloatTy, {2, 2}, "W", false);
113 bindings_.allocate(W)->getHandle() = {1, 2, 3, 4};
114 auto *b = mod_.createPlaceholder(ElemKind::FloatTy, {2}, "b", false);
115 bindings_.allocate(b)->getHandle() = {1, 1};
116
117 Node *O = F_->createFullyConnected("fc", x, W, b);
118 VariableGradientsList varGrads = computeVarGrads(O);
119
120 Tensor expected(ElemKind::FloatTy, {2, 2});
121 // x = [x1, x2]
122 // W = [[w1_1,w1_2],[w2_1,w2_2]]
123 // O = [O1, O2] = [x1*w1_1 + x2*w2_1 + b2, x1*w1_2 + x2*w2_2 + b2] = [8, 11]
124 // dO_k/dWi_j = (k == j ? 1 : 0) * x_i
125 // dF/dW_i_j = sum_of_all_k (dF/dO_k * dO_k/dW_i_j).
126 // dF/dO = [ 8, 11 ]
127 // dF/dW = [ [O1*x1, O2*x1], [O1*x2, O2*x2] ]
128 // dF/dW = [ [8*1, 11*1 ], [8*2, 11*2 ] ]
129 expected.getHandle() = {8 * 1, 11 * 1, 8 * 2, 11 * 2};
130 EXPECT_TRUE(expected.isEqual(*getGradTensor(varGrads, W)));
131 // TODO: Add checks for grads of x and b.
132}
133
134INSTANTIATE_BACKEND_TEST(OperatorGradTest);
135