1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #include "BackendTestUtils.h" |
18 | |
19 | #include "glow/Base/Tensor.h" |
20 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
21 | #include "glow/Graph/Graph.h" |
22 | #include "glow/Graph/Nodes.h" |
23 | |
24 | #include "gtest/gtest.h" |
25 | |
26 | using namespace glow; |
27 | |
28 | class OperatorGradTest : public BackendTest { |
29 | protected: |
30 | /// Compute gradients of placeholders in bindings_. Given |
31 | /// outputNode, representing H(Vars), this function will train H(vars) to be |
32 | /// 0. It is achieved by creating RegressionNode between outputNode and 0, |
33 | /// which will minimize F, a total squared error of H divided by 2. |
34 | /// |
35 | /// Note that gradient value at the start of backpropagation is the same as |
36 | /// outputNode's forward value, because RegressionNode's grad is outputNode-0. |
37 | /// |
38 | /// \param outputNode Node that contains result of H(Vars). |
39 | VariableGradientsList computeVarGrads(NodeValue outputNode) { |
40 | auto *Exp = mod_.createPlaceholder(ElemKind::FloatTy, outputNode.dims(), |
41 | "exp" , false); |
42 | bindings_.allocate(Exp)->zero(); |
43 | |
44 | auto *reg = F_->createRegression("reg" , outputNode, Exp); |
45 | auto *result = F_->createSave("ret" , reg); |
46 | bindings_.allocate(result->getPlaceholder()); |
47 | |
48 | // Create a version of the network that records the gradients to some side |
49 | // table instead of updating them. |
50 | VariableGradientsList varGrads; |
51 | TrainingConfig TC; |
52 | Function *recordNet = glow::differentiate(F_, TC, "record" , &varGrads); |
53 | auto recordName = recordNet->getName(); |
54 | allocateGrads(varGrads); |
55 | EE_.compile(CompilationMode::Train); |
56 | |
57 | // Train the network just once to record the values of gradient for |
58 | // all variables. |
59 | EE_.run(bindings_, recordName); |
60 | |
61 | return varGrads; |
62 | } |
63 | |
64 | Tensor *getGradTensor(const VariableGradientsList &grads, Placeholder *V) { |
65 | for (auto &p : grads) { |
66 | if (p.first == V) { |
67 | return bindings_.get(p.second); |
68 | } |
69 | } |
70 | return nullptr; |
71 | } |
72 | |
73 | void allocateGrads(const VariableGradientsList &grads) { |
74 | for (auto &p : grads) { |
75 | auto grad = p.second; |
76 | bindings_.allocate(grad); |
77 | } |
78 | } |
79 | |
80 | PlaceholderBindings bindings_; |
81 | }; |
82 | |
83 | TEST_P(OperatorGradTest, concat) { |
84 | CHECK_IF_ENABLED(); |
85 | |
86 | dim_t numOutputElem = 4; |
87 | |
88 | auto *A = mod_.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, |
89 | "A" , false); |
90 | bindings_.allocate(A)->getHandle() = {1, 2}; |
91 | auto *B = mod_.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, |
92 | "B" , false); |
93 | bindings_.allocate(B)->getHandle() = {3, 4}; |
94 | |
95 | Node *O = F_->createConcat("concat" , {A, B}, 1); |
96 | VariableGradientsList varGrads = computeVarGrads(O); |
97 | |
98 | Tensor expectedLeft(ElemKind::FloatTy, {1, numOutputElem / 2}); |
99 | expectedLeft.getHandle() = {1, 2}; |
100 | EXPECT_TRUE(expectedLeft.isEqual(*getGradTensor(varGrads, A))); |
101 | |
102 | Tensor expectedRight(ElemKind::FloatTy, {1, numOutputElem / 2}); |
103 | expectedRight.getHandle() = {3, 4}; |
104 | EXPECT_TRUE(expectedRight.isEqual(*getGradTensor(varGrads, B))); |
105 | } |
106 | |
107 | TEST_P(OperatorGradTest, fc) { |
108 | CHECK_IF_ENABLED(); |
109 | |
110 | auto *x = mod_.createPlaceholder(ElemKind::FloatTy, {1, 2}, "x" , false); |
111 | bindings_.allocate(x)->getHandle() = {1, 2}; |
112 | auto *W = mod_.createPlaceholder(ElemKind::FloatTy, {2, 2}, "W" , false); |
113 | bindings_.allocate(W)->getHandle() = {1, 2, 3, 4}; |
114 | auto *b = mod_.createPlaceholder(ElemKind::FloatTy, {2}, "b" , false); |
115 | bindings_.allocate(b)->getHandle() = {1, 1}; |
116 | |
117 | Node *O = F_->createFullyConnected("fc" , x, W, b); |
118 | VariableGradientsList varGrads = computeVarGrads(O); |
119 | |
120 | Tensor expected(ElemKind::FloatTy, {2, 2}); |
121 | // x = [x1, x2] |
122 | // W = [[w1_1,w1_2],[w2_1,w2_2]] |
123 | // O = [O1, O2] = [x1*w1_1 + x2*w2_1 + b2, x1*w1_2 + x2*w2_2 + b2] = [8, 11] |
124 | // dO_k/dWi_j = (k == j ? 1 : 0) * x_i |
125 | // dF/dW_i_j = sum_of_all_k (dF/dO_k * dO_k/dW_i_j). |
126 | // dF/dO = [ 8, 11 ] |
127 | // dF/dW = [ [O1*x1, O2*x1], [O1*x2, O2*x2] ] |
128 | // dF/dW = [ [8*1, 11*1 ], [8*2, 11*2 ] ] |
129 | expected.getHandle() = {8 * 1, 11 * 1, 8 * 2, 11 * 2}; |
130 | EXPECT_TRUE(expected.isEqual(*getGradTensor(varGrads, W))); |
131 | // TODO: Add checks for grads of x and b. |
132 | } |
133 | |
134 | INSTANTIATE_BACKEND_TEST(OperatorGradTest); |
135 | |