1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#include "BackendTestUtils.h"
17
18#include "glow/Base/Tensor.h"
19#include "glow/ExecutionEngine/ExecutionEngine.h"
20#include "glow/Graph/Graph.h"
21#include "glow/Graph/Nodes.h"
22#include "glow/IR/IR.h"
23#include "glow/IR/IRBuilder.h"
24#include "glow/IR/Instrs.h"
25
26#include "gtest/gtest.h"
27
28using namespace glow;
29using llvm::cast;
30
31class GradCheck : public ::testing::TestWithParam<std::string> {
32public:
33 ExecutionEngine EET_{GetParam()};
34 ExecutionEngine EEI_{GetParam()};
35 std::vector<ExecutionEngine *> engines_;
36 void SetUp() {
37 engines_.push_back(&EEI_);
38 engines_.push_back(&EET_);
39 }
40};
41
42/// \returns the regression loss for the tensor \p X with regard to \p Y.
43float computeL2Loss(Tensor *X, Tensor *Y) {
44 assert(X->dims() == Y->dims() && "Invalid input dims");
45 auto xH = X->getHandle<>();
46 auto yH = Y->getHandle<>();
47 float loss = 0;
48
49 for (size_t i = 0, e = X->size(); i < e; i++) {
50 float dy = (xH.raw(i) - yH.raw(i));
51 loss += 0.5 * dy * dy;
52 }
53
54 return loss;
55}
56
57/// \returns the error when comparing two grads: absolute or relative.
58float gradDiff(float G1, float G2) {
59 return std::min(std::abs(G1 - G2), std::abs(G1 - G2) / std::abs(G1 + G2 + 1));
60}
61
62Placeholder *getGrad(const VariableGradientsList &grads, Placeholder *V) {
63 for (auto &p : grads) {
64 if (p.first == V) {
65 return p.second;
66 }
67 }
68 return nullptr;
69}
70
71void allocateGrads(PlaceholderBindings &bindings,
72 const VariableGradientsList &grads) {
73 for (auto &p : grads) {
74 auto grad = p.second;
75 bindings.allocate(grad);
76 }
77}
78
79/// Performs gradient check by comparing analytical and numerical gradients.
80/// Numeric grad is calculated based on: f(x-delta) and f(x+delta) values.
81/// Analytical grad is based on the gradient output calculated during back
82/// propagation.
83///
84/// \param EET Execution engine to compile/run network for training.
85/// \param EEI Executiuon engine to compile/run network for inference.
86/// \param bindings Placeholder bindings.
87/// \param result Node that contains result of f(x).
88/// \param inputVar Placeholder which gradient is assessed.
89/// \param expVar Placeholder with expected value, only used during the
90/// training. \param inputs Tensor for \p inputVar placeholder. \param outputs
91/// Tensor for \p expVar placeholder. \param allowedError allowed delta between
92/// analytical and numerical gradients.
93void performGradCheck(ExecutionEngine &EET, ExecutionEngine &EEI,
94 PlaceholderBindings &bindings, Placeholder *result,
95 Placeholder *inputVar, Placeholder *expVar,
96 Tensor *inputs, Tensor *outputs, float delta,
97 float allowedError) {
98 TrainingConfig TC;
99
100 auto *F = EET.getModule().getFunction("main");
101
102 // Allocate result, inputVar and expVar.
103 bindings.allocate(result);
104 bindings.allocate(inputVar);
105 bindings.allocate(expVar);
106
107 // This variable records the number of the next sample to be used for
108 // training.
109 size_t sampleCounter = 0;
110
111 // Create a function that trains the network.
112 auto *TF = glow::differentiate(F, TC);
113
114 // Create a version of the network that records the gradients to some side
115 // table instead of updating them.
116 VariableGradientsList varGrads;
117 auto *RF = glow::differentiate(F, TC, "record", &varGrads);
118 auto rfName = RF->getName();
119 auto tfName = TF->getName();
120 EET.compile(CompilationMode::Train);
121 bindings.allocate(EET.getModule().getPlaceholders());
122
123 // The network might have variables, other than inputVar and expVar.
124 // Train the network until other variables reach some stable local minimum.
125 runBatch(EET, bindings, 300, sampleCounter, {inputVar, expVar},
126 {inputs, outputs}, tfName);
127
128 // Clear the gradients of the first layer.
129 auto gradVar = getGrad(varGrads, inputVar);
130 auto gradVarTensor = bindings.get(gradVar);
131 gradVarTensor->zero();
132
133 // Train the network just once to record the values of gradient for inputVar.
134 runBatch(EET, bindings, 1, sampleCounter, {inputVar, expVar},
135 {inputs, outputs}, rfName);
136 // Compile the original network in inference mode.
137 EEI.compile(CompilationMode::Infer);
138 PlaceholderBindings inferBindings;
139 inferBindings.allocate(EEI.getModule().getPlaceholders());
140 // Copy values to inferenceBindings. This is needed for any placeholders that
141 // were initialized during function creation.
142 for (auto PH : EEI.getModule().getPlaceholders()) {
143 bindings.copyToTarget(PH->getName(), inferBindings);
144 }
145 auto resultTensor = inferBindings.get(
146 inferBindings.getPlaceholderByNameSlow(result->getName()));
147
148 auto analyticalGradsH = gradVarTensor->getHandle();
149 auto inputsH = inputs->getHandle<>();
150 bool allZeroGradients = true;
151 auto inferInput = inferBindings.getPlaceholderByNameSlow(inputVar->getName());
152
153 for (size_t i = 0; i < analyticalGradsH.size(); i++) {
154 auto old = inputsH.raw(i);
155 // Calculate f(x+e):
156 inputsH.raw(i) = old + delta;
157 updateInputPlaceholders(inferBindings, {inferInput}, {inputs});
158 EEI.run(inferBindings);
159 auto plusLoss = computeL2Loss(outputs, resultTensor);
160
161 // Calculate f(x-e):
162 inputsH.raw(i) = old - delta;
163 updateInputPlaceholders(inferBindings, {inferInput}, {inputs});
164 EEI.run(inferBindings);
165
166 auto minusLoss = computeL2Loss(outputs, resultTensor);
167
168 // Restore value back.
169 inputsH.raw(i) = old;
170
171 auto numericGrad = (plusLoss - minusLoss) / (2 * delta);
172 auto analyticalGrad = analyticalGradsH.raw(i);
173 auto err = gradDiff(analyticalGrad, numericGrad);
174
175 if (numericGrad != 0.0) {
176 allZeroGradients = false;
177 }
178 // Make sure that the analytical and numerical gradients agree.
179 EXPECT_LE(err, allowedError);
180 }
181
182 // Make sure that some gradients are non-zero. If all gradients are zero,
183 // it means that the input doesn't affect the output, which also means
184 // that the operations being test is totally ignored.
185 EXPECT_FALSE(allZeroGradients);
186}
187
188TEST_P(GradCheck, gradientCheckConcat) {
189 CHECK_IF_ENABLED();
190 PlaceholderBindings bindings;
191 Placeholder *A, *Exp, *B;
192 SaveNode *result;
193 dim_t numOutputElem = 20;
194 for (auto *EE : engines_) {
195 auto &mod = EE->getModule();
196 Function *F = mod.createFunction("main");
197
198 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, "A",
199 false);
200 B = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, "B",
201 false);
202
203 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp",
204 false);
205 Node *O = F->createConcat("concat", {A, B}, 1);
206 O = F->createRegression("reg", O, Exp);
207 result = F->createSave("ret", O);
208 }
209 bindings.allocate(B)->zero();
210 Tensor inputs(ElemKind::FloatTy, {{1, numOutputElem / 2}});
211 Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}});
212
213 auto inputsH = inputs.getHandle<>();
214 auto outputsH = outputs.getHandle<>();
215 auto &trainingMod = EET_.getModule();
216 inputsH.randomize(-1, 1, trainingMod.getPRNG());
217 outputsH.randomize(-1, 1, trainingMod.getPRNG());
218
219 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
220 &inputs, &outputs, 0.001, 0.01);
221}
222
223TEST_P(GradCheck, gradientCheckMatMul) {
224 CHECK_IF_ENABLED();
225 PlaceholderBindings Bindings;
226 Placeholder *A, *Exp, *B;
227 SaveNode *Result;
228 dim_t NumDim = 10;
229 for (auto *EE : engines_) {
230 auto &Mod = EE->getModule();
231 Bindings.clear();
232 Function *F = Mod.createFunction("main");
233
234 A = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "A", false);
235 B = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "B", false);
236
237 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "exp",
238 false);
239 auto HandleB = Bindings.allocate(B)->getHandle<float>();
240 HandleB.randomize(-1, 1, Mod.getPRNG());
241 Node *MM = F->createMatMul("matmul", A, B);
242 auto *Reg = F->createRegression("reg", MM, Exp);
243 Result = F->createSave("save", Reg);
244 }
245
246 Tensor Inputs(ElemKind::FloatTy, {{NumDim, NumDim}});
247 Tensor Outputs(ElemKind::FloatTy, {{NumDim, NumDim}});
248
249 auto InputsH = Inputs.getHandle<>();
250 auto OutputsH = Outputs.getHandle<>();
251 auto &Mod = EET_.getModule();
252
253 InputsH.randomize(-1, 1, Mod.getPRNG());
254 OutputsH.randomize(-1, 1, Mod.getPRNG());
255
256 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
257 &Inputs, &Outputs, 0.001, 0.01);
258}
259
260TEST_P(GradCheck, gradientCheckBatchMatMul) {
261 CHECK_IF_ENABLED();
262 PlaceholderBindings Bindings;
263 Placeholder *A, *Exp, *B;
264 SaveNode *Result;
265
266 constexpr dim_t BatchSize{4}, P{5}, Q{6}, R{7};
267
268 for (auto *EE : engines_) {
269 auto &Mod = EE->getModule();
270 Bindings.clear();
271 Function *F = Mod.createFunction("main");
272
273 A = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, P, Q}, "A",
274 /*isTrainable=*/false);
275 B = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, Q, R}, "B",
276 /*isTrainable=*/false);
277
278 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, P, R}, "exp",
279 /*isTrainable=*/false);
280 auto HandleB = Bindings.allocate(B)->getHandle<float>();
281 HandleB.randomize(-1, 1, Mod.getPRNG());
282 Node *MM = F->createBatchMatMul("batchMatmul", A, B);
283 auto *Reg = F->createRegression("reg", MM, Exp);
284 Result = F->createSave("save", Reg);
285 }
286
287 Tensor Inputs(ElemKind::FloatTy, {{BatchSize, P, Q}});
288 Tensor Outputs(ElemKind::FloatTy, {{BatchSize, P, R}});
289
290 auto InputsH = Inputs.getHandle<>();
291 auto OutputsH = Outputs.getHandle<>();
292 auto &Mod = EET_.getModule();
293
294 InputsH.randomize(-1, 1, Mod.getPRNG());
295 OutputsH.randomize(-1, 1, Mod.getPRNG());
296
297 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
298 &Inputs, &Outputs, 0.001, 0.01);
299}
300
301TEST_P(GradCheck, gradientCheckTile) {
302 CHECK_IF_ENABLED();
303 PlaceholderBindings Bindings;
304 Placeholder *A, *Exp;
305 SaveNode *Result;
306
307 constexpr dim_t N{2}, C{3}, H{4}, W{5};
308 constexpr dim_t NumTiles{5};
309
310 for (auto *EE : engines_) {
311 auto &Mod = EE->getModule();
312 Function *F = Mod.createFunction("main");
313
314 A = Mod.createPlaceholder(ElemKind::FloatTy, {N, C, H, W}, "A",
315 /*isTrainable=*/false);
316 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {N, NumTiles * C, H, W},
317 "Exp", /*isTrainable=*/false);
318 auto *Tile = F->createTile("tile", A, NumTiles, /*axis=*/1, Exp->getType());
319 auto *Reg = F->createRegression("reg", Tile, Exp);
320 Result = F->createSave("save", Reg);
321 }
322
323 Tensor Inputs(ElemKind::FloatTy, A->getType()->dims());
324 Tensor Outputs(ElemKind::FloatTy, Exp->getType()->dims());
325
326 auto InputsH = Inputs.getHandle<>();
327 auto OutputsH = Outputs.getHandle<>();
328 auto &Mod = EET_.getModule();
329 InputsH.randomize(-1, 1, Mod.getPRNG());
330 OutputsH.randomize(-1, 1, Mod.getPRNG());
331
332 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
333 &Inputs, &Outputs, 0.001, 0.01);
334}
335
336TEST_P(GradCheck, gradientCheckBatchedReduceAddAxis0) {
337 CHECK_IF_ENABLED();
338 PlaceholderBindings Bindings;
339 Placeholder *A, *Exp;
340 SaveNode *Result;
341 dim_t BatchSize = 4;
342 dim_t NumRows = 3;
343 dim_t NumCols = 5;
344 for (auto *EE : engines_) {
345 auto &Mod = EE->getModule();
346 Function *F = Mod.createFunction("main");
347
348 A = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, NumRows, NumCols},
349 "A", false);
350
351 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, NumCols}, "exp",
352 false);
353
354 TypeRef Ty = Mod.uniqueType(ElemKind::FloatTy, {NumRows, NumCols});
355 Node *BRA = F->createBatchedReduceAdd("BRA", Ty, A, 0 /*axis*/);
356 auto *Reg = F->createRegression("reg", BRA, Exp);
357 Result = F->createSave("save", Reg);
358 }
359
360 Tensor Inputs(ElemKind::FloatTy, {{BatchSize, NumRows, NumCols}});
361 Tensor Outputs(ElemKind::FloatTy, {{NumRows, NumCols}});
362
363 auto InputsH = Inputs.getHandle<>();
364 auto OutputsH = Outputs.getHandle<>();
365 auto &Mod = EET_.getModule();
366 InputsH.randomize(-1, 1, Mod.getPRNG());
367 OutputsH.randomize(-1, 1, Mod.getPRNG());
368
369 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
370 &Inputs, &Outputs, 0.001, 0.01);
371}
372
373TEST_P(GradCheck, gradientCheckBatchedReduceAddAxis1) {
374 CHECK_IF_ENABLED();
375 PlaceholderBindings Bindings;
376 dim_t NumRows = 3;
377 dim_t BatchSize = 4;
378 dim_t NumCols = 5;
379 Placeholder *A, *Exp;
380 SaveNode *Result;
381 for (auto *EE : engines_) {
382 auto &Mod = EE->getModule();
383 Function *F = Mod.createFunction("main");
384
385 A = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, BatchSize, NumCols},
386 "A", false);
387
388 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, NumCols}, "exp",
389 false);
390
391 TypeRef Ty = Mod.uniqueType(ElemKind::FloatTy, {NumRows, NumCols});
392 Node *BRA = F->createBatchedReduceAdd("BRA", Ty, A, 1 /*axis*/);
393 auto *Reg = F->createRegression("reg", BRA, Exp);
394 Result = F->createSave("save", Reg);
395 }
396
397 Tensor Inputs(ElemKind::FloatTy, {{NumRows, BatchSize, NumCols}});
398 Tensor Outputs(ElemKind::FloatTy, {{NumRows, NumCols}});
399
400 auto InputsH = Inputs.getHandle<>();
401 auto OutputsH = Outputs.getHandle<>();
402 auto &Mod = EET_.getModule();
403 InputsH.randomize(-1, 1, Mod.getPRNG());
404 OutputsH.randomize(-1, 1, Mod.getPRNG());
405
406 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
407 &Inputs, &Outputs, 0.001, 0.01);
408}
409
410TEST_P(GradCheck, gradientCheckGatherVec) {
411 CHECK_IF_ENABLED();
412 PlaceholderBindings Bindings;
413 Placeholder *A, *Exp;
414 SaveNode *Result;
415 for (auto *EE : engines_) {
416 auto &Mod = EE->getModule();
417 Bindings.clear();
418 Function *F = Mod.createFunction("main");
419
420 A = Mod.createPlaceholder(ElemKind::FloatTy, {3, 4}, "A", false);
421 auto *Indices = Mod.createPlaceholder(ElemKind::Int32ITy, {2}, "I", false);
422 Bindings.allocate(Indices)->getHandle<int32_t>() = {0, 2};
423 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {2, 4}, "exp", false);
424
425 Node *G = F->createGather("gather", A, Indices, 0 /*batchDims*/);
426 auto *Reg = F->createRegression("reg", G, Exp);
427 Result = F->createSave("save", Reg);
428 }
429
430 Tensor Inputs(ElemKind::FloatTy, {{3, 4}});
431 Tensor Outputs(ElemKind::FloatTy, {{2, 4}});
432
433 auto InputsH = Inputs.getHandle<>();
434 auto OutputsH = Outputs.getHandle<>();
435 auto &Mod = EET_.getModule();
436 InputsH.randomize(-1, 1, Mod.getPRNG());
437 OutputsH.randomize(-1, 1, Mod.getPRNG());
438
439 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
440 &Inputs, &Outputs, 0.001, 0.01);
441}
442
443TEST_P(GradCheck, gradientCheckGatherDim) {
444 CHECK_IF_ENABLED();
445 PlaceholderBindings Bindings;
446 Placeholder *A, *Exp;
447 SaveNode *Result;
448 for (auto *EE : engines_) {
449 auto &Mod = EE->getModule();
450 Bindings.clear();
451 Function *F = Mod.createFunction("main");
452
453 A = Mod.createPlaceholder(ElemKind::FloatTy, {8, 4}, "A", false);
454 auto *Indices =
455 Mod.createPlaceholder(ElemKind::Int32ITy, {2, 2}, "I", false);
456 Bindings.allocate(Indices)->getHandle<int32_t>() = {0, 2, 3, 1};
457 Exp = Mod.createPlaceholder(ElemKind::FloatTy, {2, 2, 4}, "exp", false);
458
459 Node *G = F->createGather("gather", A, Indices, 0 /*batchDims*/);
460 auto *Reg = F->createRegression("reg", G, Exp);
461 Result = F->createSave("save", Reg);
462 }
463
464 Tensor Inputs(ElemKind::FloatTy, {{8, 4}});
465 Tensor Outputs(ElemKind::FloatTy, {{2, 2, 4}});
466
467 auto InputsH = Inputs.getHandle<>();
468 auto OutputsH = Outputs.getHandle<>();
469 auto &Mod = EET_.getModule();
470 InputsH.randomize(-1, 1, Mod.getPRNG());
471 OutputsH.randomize(-1, 1, Mod.getPRNG());
472
473 performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp,
474 &Inputs, &Outputs, 0.001, 0.01);
475}
476
477static void gradientCheckGroupConv(dim_t depth, dim_t group,
478 ExecutionEngine &EET_,
479 ExecutionEngine &EEI_) {
480 PlaceholderBindings bindings;
481 dim_t numDim = 10;
482 std::vector<ExecutionEngine *> engines;
483 engines.push_back(&EEI_);
484 engines.push_back(&EET_);
485 Placeholder *A, *Ex;
486 SaveNode *result;
487 for (auto *EE : engines) {
488 auto &mod = EE->getModule();
489 bindings.clear();
490 Function *F = mod.createFunction("main");
491 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth},
492 "A", false);
493 Ex = mod.createPlaceholder(
494 ElemKind::FloatTy, {1, numDim + 1, numDim + 1, depth}, "exp", false);
495
496 Node *O = F->createConv(bindings, "conv", A, depth, 2, 1, 1, group);
497 O = F->createRegression("reg", O, Ex);
498 result = F->createSave("ret", O);
499 }
500
501 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, depth});
502 Tensor outputs(ElemKind::FloatTy, {1, numDim + 1, numDim + 1, depth});
503
504 auto inputsH = inputs.getHandle<>();
505 auto outputsH = outputs.getHandle<>();
506 auto &mod = EET_.getModule();
507 inputsH.randomize(-1, 1, mod.getPRNG());
508 outputsH.randomize(-1, 1, mod.getPRNG());
509
510 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex,
511 &inputs, &outputs, 0.001, 0.04);
512}
513
514TEST_P(GradCheck, gradientCheckConv) {
515 CHECK_IF_ENABLED();
516 gradientCheckGroupConv(1, 1, EET_, EEI_);
517}
518
519TEST_P(GradCheck, gradientCheckDepthwiseConv) {
520 CHECK_IF_ENABLED();
521 gradientCheckGroupConv(4, 4, EET_, EEI_);
522}
523
524TEST_P(GradCheck, gradientCheckGroupConv) {
525 CHECK_IF_ENABLED();
526 gradientCheckGroupConv(4, 2, EET_, EEI_);
527}
528
529static void gradientCheckDilatedConv(dim_t depth, dim_t group,
530 unsigned_t dilation, ExecutionEngine &EET_,
531 ExecutionEngine &EEI_) {
532 PlaceholderBindings bindings;
533 dim_t numDim = 10;
534 std::vector<ExecutionEngine *> engines;
535 engines.push_back(&EEI_);
536 engines.push_back(&EET_);
537 Placeholder *A, *Ex;
538 SaveNode *result;
539 for (auto *EE : engines) {
540 auto &mod = EE->getModule();
541 bindings.clear();
542 Function *F = mod.createFunction("main");
543 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth},
544 "A", false);
545 Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth},
546 "exp", false);
547
548 Node *O = F->createConv(bindings, "conv", A, depth, 2, 1, 1, group,
549 {dilation, dilation});
550 O = F->createRegression("reg", O, Ex);
551 result = F->createSave("ret", O);
552 }
553
554 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, depth});
555 Tensor outputs(ElemKind::FloatTy, {1, numDim, numDim, depth});
556
557 auto inputsH = inputs.getHandle<>();
558 auto outputsH = outputs.getHandle<>();
559 auto &mod = EET_.getModule();
560 inputsH.randomize(-1, 1, mod.getPRNG());
561 outputsH.randomize(-1, 1, mod.getPRNG());
562
563 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex,
564 &inputs, &outputs, 0.001, 0.04);
565}
566
567TEST_P(GradCheck, gradientCheckDilatedConv) {
568 CHECK_IF_ENABLED();
569 gradientCheckDilatedConv(1, 1, 2, EET_, EEI_);
570}
571
572TEST_P(GradCheck, gradientCheckAvgPool) {
573 CHECK_IF_ENABLED();
574 PlaceholderBindings bindings;
575 dim_t numDim = 10;
576 dim_t numOutputElem = 10;
577 Placeholder *A, *Exp;
578 SaveNode *result;
579 for (auto *EE : engines_) {
580 auto &mod = EE->getModule();
581 bindings.clear();
582 Function *F = mod.createFunction("main");
583 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A",
584 false);
585 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
586 false);
587
588 Node *O = F->createAvgPool("pool", A, 3, 3, 1);
589 O = F->createTanh("tanh", O);
590 O = F->createFullyConnected(bindings, "fc", O, numOutputElem);
591 O = F->createRegression("reg", O, Exp);
592 result = F->createSave("ret", O);
593 }
594
595 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2});
596 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
597
598 auto inputsH = inputs.getHandle<>();
599 auto outputsH = outputs.getHandle<>();
600 auto &mod = EET_.getModule();
601 inputsH.initXavier(1, mod.getPRNG());
602 outputsH.initXavier(1, mod.getPRNG());
603
604 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
605 &inputs, &outputs, 0.001, 0.004);
606}
607
608TEST_P(GradCheck, gradientCheckAvgPoolCountExcludePads) {
609 CHECK_IF_ENABLED();
610 PlaceholderBindings bindings;
611 dim_t numDim = 10;
612 dim_t numOutputElem = 10;
613 Placeholder *A, *Exp;
614 SaveNode *result;
615 for (auto *EE : engines_) {
616 auto &mod = EE->getModule();
617 bindings.clear();
618 Function *F = mod.createFunction("main");
619 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A",
620 false);
621 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
622 false);
623
624 Node *O = F->createAvgPool("pool", A, 3, 3, 1, NHWC,
625 /* countIncludePads */ false);
626 O = F->createTanh("tanh", O);
627 O = F->createFullyConnected(bindings, "fc", O, numOutputElem);
628 O = F->createRegression("reg", O, Exp);
629 result = F->createSave("ret", O);
630 }
631
632 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2});
633 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
634
635 auto inputsH = inputs.getHandle<>();
636 auto outputsH = outputs.getHandle<>();
637 auto &mod = EET_.getModule();
638 inputsH.initXavier(1, mod.getPRNG());
639 outputsH.initXavier(1, mod.getPRNG());
640
641 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
642 &inputs, &outputs, 0.001, 0.004);
643}
644
645TEST_P(GradCheck, gradientCheckMaxPool) {
646 CHECK_IF_ENABLED();
647 PlaceholderBindings bindings;
648 dim_t numDim = 10;
649 dim_t numOutputElem = 10;
650 Placeholder *A, *Exp;
651 SaveNode *result;
652 for (auto *EE : engines_) {
653 auto &mod = EE->getModule();
654 bindings.clear();
655 Function *F = mod.createFunction("main");
656 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A",
657 false);
658 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
659 false);
660
661 MaxPoolNode *P = F->createMaxPool("pool", A, 3, 3, 1);
662 Node *O = F->createTanh("tanh", P->getResult());
663 O = F->createFullyConnected(bindings, "fc", O, numOutputElem);
664 O = F->createRegression("reg", O, Exp);
665 result = F->createSave("ret", O);
666 }
667
668 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2});
669 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
670
671 auto inputsH = inputs.getHandle<>();
672 auto outputsH = outputs.getHandle<>();
673 auto &mod = EET_.getModule();
674 inputsH.initXavier(1, mod.getPRNG());
675 outputsH.initXavier(1, mod.getPRNG());
676
677 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
678 &inputs, &outputs, 0.001, 0.004);
679}
680
681TEST_P(GradCheck, gradientCheckAdaptiveAvgPool) {
682 CHECK_IF_ENABLED();
683 PlaceholderBindings bindings;
684 dim_t numDim = 10;
685 dim_t numOutputElem = 10;
686 Placeholder *A, *Exp;
687 SaveNode *result;
688 for (auto *EE : engines_) {
689 auto &mod = EE->getModule();
690 bindings.clear();
691 Function *F = mod.createFunction("main");
692 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 1}, "A",
693 false);
694 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
695 false);
696
697 auto outTy = mod.uniqueType(ElemKind::FloatTy, {1, 3, 3, 1});
698 Node *O = F->createAdaptiveAvgPool("pool", A, outTy);
699 O = F->createTanh("tanh", O);
700 O = F->createFullyConnected(bindings, "fc", O, numOutputElem);
701 O = F->createRegression("reg", O, Exp);
702 result = F->createSave("ret", O);
703 }
704
705 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 1});
706 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
707
708 auto inputsH = inputs.getHandle<>();
709 auto outputsH = outputs.getHandle<>();
710 auto &mod = EET_.getModule();
711 inputsH.initXavier(1, mod.getPRNG());
712 outputsH.initXavier(1, mod.getPRNG());
713
714 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
715 &inputs, &outputs, 0.001, 0.005);
716}
717
718TEST_P(GradCheck, gradientCheckBatchNorm) {
719 CHECK_IF_ENABLED();
720 PlaceholderBindings bindings;
721 dim_t numDim = 5;
722 dim_t numOutputElem = numDim * numDim * 3;
723 Placeholder *A, *Ex;
724 SaveNode *result;
725 for (auto *EE : engines_) {
726 auto &mod = EE->getModule();
727 bindings.clear();
728 Function *F = mod.createFunction("main");
729 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 3}, "A",
730 false);
731 Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp",
732 false);
733
734 Node *O = F->createBatchNormalization(bindings, "batch", A, 3, 0.0001, 0.9);
735 O = F->createReshape("reshape", O, {1, numDim * numDim * 3});
736 O = F->createRegression("reg", O, Ex);
737 result = F->createSave("ret", O);
738 }
739
740 Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 3});
741 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
742
743 auto inputsH = inputs.getHandle<>();
744 auto outputsH = outputs.getHandle<>();
745 auto &mod = EET_.getModule();
746 inputsH.initXavier(1, mod.getPRNG());
747 outputsH.initXavier(1, mod.getPRNG());
748
749 for (auto &elem : inputsH) {
750 elem = elem * 6 + 4;
751 }
752
753 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex,
754 &inputs, &outputs, 0.001, 0.004);
755}
756
757TEST_P(GradCheck, gradientCheckArithmeticDiv) {
758 CHECK_IF_ENABLED();
759 // The test creates a net: A / B = Exp. Where A is trainable weight,
760 // B and Exp are external data (initialized randomly once). SGD will find
761 // correct value for A, and then gradient check will be performed.
762 PlaceholderBindings bindings;
763 dim_t numDim = 10;
764 Placeholder *B, *Exp;
765 SaveNode *result;
766 for (auto *EE : engines_) {
767 auto &mod = EE->getModule();
768 bindings.clear();
769 Function *F = mod.createFunction("main");
770 auto *A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "A", true);
771 B = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "B", false);
772 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "exp", false);
773
774 auto *ATensor = bindings.allocate(A);
775 ATensor->init(Tensor::InitKind::Xavier, 1.0, mod.getPRNG());
776
777 Node *O = F->createDiv("div", A, B);
778 O = F->createRegression("reg", O, Exp);
779 result = F->createSave("ret", O);
780 }
781
782 Tensor BValues(ElemKind::FloatTy, {1, numDim});
783 Tensor ExpValues(ElemKind::FloatTy, {1, numDim});
784 // Random values are in the range, so that all intermediate computations are
785 // not too small and not too large.
786 auto &mod = EET_.getModule();
787 BValues.getHandle().randomize(0.1, 1, mod.getPRNG());
788 ExpValues.getHandle().randomize(0.1, 1, mod.getPRNG());
789
790 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), B, Exp,
791 &BValues, &ExpValues, 0.0001, 0.001);
792}
793
794TEST_P(GradCheck, gradientCheckArithmetic) {
795 CHECK_IF_ENABLED();
796 PlaceholderBindings bindings;
797 dim_t numDim = 20;
798 Placeholder *A, *Exp;
799 SaveNode *result;
800 for (auto *EE : engines_) {
801 auto &mod = EE->getModule();
802 bindings.clear();
803 Function *F = mod.createFunction("main");
804 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "A", false);
805 auto *B = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "B", true);
806 auto *C = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "C", false);
807 auto *D = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "D", false);
808 auto *E = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "E", false);
809 bindings.allocate(B)->zero();
810 bindings.allocate(C)->zero();
811 bindings.allocate(D)->zero();
812
813 // Randomize E to avoid div by zero.
814 auto *ETensor = bindings.allocate(E);
815 ETensor->getHandle().randomize(-1, 1, mod.getPRNG());
816
817 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "exp", false);
818
819 Node *O = F->createMul("mul", A, B);
820 O = F->createAdd("add", O, C);
821 O = F->createSub("sub", D, O);
822 O = F->createDiv("div", O, E);
823 O = F->createRegression("reg", O, Exp);
824 result = F->createSave("ret", O);
825 }
826
827 Tensor inputs(ElemKind::FloatTy, {1, numDim});
828 Tensor outputs(ElemKind::FloatTy, {1, numDim});
829
830 auto inputsH = inputs.getHandle<>();
831 auto outputsH = outputs.getHandle<>();
832 auto &mod = EET_.getModule();
833 inputsH.randomize(-1, 1, mod.getPRNG());
834 outputsH.randomize(-1, 1, mod.getPRNG());
835
836 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
837 &inputs, &outputs, 0.01, 0.004);
838}
839
840TEST_P(GradCheck, gradientCheckFCConcatTanh) {
841 CHECK_IF_ENABLED();
842 // Using the same gradient check test setup as gradientCheck_FC_Concat_RELU
843 PlaceholderBindings bindings;
844 dim_t numInputElem = 20;
845 dim_t numOutputElem = 10;
846 Placeholder *A, *Exp;
847 SaveNode *result;
848 for (auto *EE : engines_) {
849 auto &mod = EE->getModule();
850 bindings.clear();
851 Function *F = mod.createFunction("main");
852 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A", false);
853 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
854 false);
855
856 Node *FA = F->createFullyConnected(bindings, "fc", A, numOutputElem);
857 FA = F->createTanh("tanh", FA);
858 FA = F->createRegression("reg", FA, Exp);
859 result = F->createSave("ret", FA);
860 }
861 Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}});
862 Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}});
863
864 auto inputsH = inputs.getHandle<>();
865 auto outputsH = outputs.getHandle<>();
866 auto &mod = EET_.getModule();
867 inputsH.initXavier(1, mod.getPRNG());
868 outputsH.initXavier(1, mod.getPRNG());
869
870 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
871 &inputs, &outputs, 0.0001, 0.001);
872}
873
874TEST_P(GradCheck, gradientCheckFC) {
875 CHECK_IF_ENABLED();
876 PlaceholderBindings bindings;
877 dim_t numInputElem = 20;
878 dim_t numOutputElem = 10;
879 Placeholder *A, *Exp;
880 SaveNode *result;
881 for (auto *EE : engines_) {
882 auto &mod = EE->getModule();
883 bindings.clear();
884 Function *F = mod.createFunction("main");
885 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A", false);
886 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
887 false);
888
889 Node *FA = F->createFullyConnected(bindings, "fc", A, numOutputElem);
890 FA = F->createRegression("reg", FA, Exp);
891 result = F->createSave("ret", FA);
892 }
893
894 Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}});
895 Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}});
896
897 auto inputsH = inputs.getHandle<>();
898 auto outputsH = outputs.getHandle<>();
899 auto &mod = EET_.getModule();
900 inputsH.initXavier(1, mod.getPRNG());
901 outputsH.initXavier(1, mod.getPRNG());
902
903 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
904 &inputs, &outputs, 0.0001, 0.0001);
905}
906
907TEST_P(GradCheck, gradientCheckSigmoid) {
908 CHECK_IF_ENABLED();
909 PlaceholderBindings bindings;
910 dim_t numInputElem = 20;
911 dim_t numOutputElem = 20;
912 Placeholder *A, *Exp;
913 SaveNode *result;
914 for (auto *EE : engines_) {
915 auto &mod = EE->getModule();
916 Function *F = mod.createFunction("main");
917 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A", false);
918 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
919 false);
920
921 Node *FA = F->createSigmoid("sig", A);
922 FA = F->createRegression("reg", FA, Exp);
923 result = F->createSave("ret", FA);
924 }
925
926 Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}});
927 Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}});
928
929 auto inputsH = inputs.getHandle<>();
930 auto outputsH = outputs.getHandle<>();
931 auto &mod = EET_.getModule();
932 inputsH.initXavier(1, mod.getPRNG());
933 outputsH.initXavier(1, mod.getPRNG());
934
935 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
936 &inputs, &outputs, 0.001, 0.01);
937}
938
939TEST_P(GradCheck, gradientCheckRelu) {
940 CHECK_IF_ENABLED();
941 PlaceholderBindings bindings;
942 dim_t numInputElem = 20;
943 dim_t numOutputElem = 20;
944 Placeholder *A, *Exp;
945 SaveNode *result;
946 for (auto *EE : engines_) {
947 auto &mod = EE->getModule();
948 Function *F = mod.createFunction("main");
949 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A", false);
950 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp",
951 false);
952
953 Node *FA = F->createRELU("relu", A);
954 FA = F->createRegression("reg", FA, Exp);
955 result = F->createSave("ret", FA);
956 }
957
958 Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}});
959 Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}});
960
961 auto inputsH = inputs.getHandle<>();
962 auto outputsH = outputs.getHandle<>();
963 auto &mod = EET_.getModule();
964 inputsH.initXavier(1, mod.getPRNG());
965 outputsH.initXavier(1, mod.getPRNG());
966
967 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
968 &inputs, &outputs, 0.001, 0.01);
969}
970
971TEST_P(GradCheck, gradientCheckTranspose) {
972 CHECK_IF_ENABLED();
973 // Using the same gradient check test setup as gradientCheck_FC_Concat_RELU
974 PlaceholderBindings bindings;
975 dim_t numOutputElem = 10;
976 Placeholder *A, *Exp;
977 SaveNode *result;
978 for (auto *EE : engines_) {
979 auto &mod = EE->getModule();
980 bindings.clear();
981 Function *F = mod.createFunction("main");
982 A = mod.createPlaceholder(ElemKind::FloatTy, {1, 5, 10, 5}, "input", false,
983 "NHWC");
984 Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp",
985 false);
986 Node *TA = F->createTranspose("transpose", A, NHWC2NCHW);
987 TA = F->createFullyConnected(bindings, "fc", TA, numOutputElem);
988 TA = F->createRegression("regress", TA, Exp);
989 result = F->createSave("ret", TA);
990 }
991
992 Tensor inputs(ElemKind::FloatTy, {1, 5, 10, 5});
993 Tensor outputs(ElemKind::FloatTy, {1, numOutputElem});
994
995 auto inputsH = inputs.getHandle<>();
996 auto outputsH = outputs.getHandle<>();
997 auto &mod = EET_.getModule();
998 inputsH.randomize(-1, 1, mod.getPRNG());
999 outputsH.randomize(-1, 1, mod.getPRNG());
1000
1001 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
1002 &inputs, &outputs, 0.0001, 0.001);
1003}
1004
1005TEST_P(GradCheck, gradientCheckCrossEntropyLoss) {
1006 CHECK_IF_ENABLED();
1007 const dim_t batchSize = 6;
1008 const int testSamples = 5;
1009 const float stepSize = 1e-4;
1010 const float delta = 0.015;
1011 TrainingConfig TC;
1012 PlaceholderBindings bindings;
1013
1014 auto &mod = EET_.getModule();
1015 Function *F = mod.createFunction("main");
1016 auto *P =
1017 mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 4}, "P", false);
1018 bindings.allocate(P)->zero();
1019 auto *Y =
1020 mod.createPlaceholder(ElemKind::Int64ITy, {batchSize}, "Labels", false);
1021 bindings.allocate(Y)->zero();
1022 Node *CE = F->createCrossEntropyLoss("celoss", P, Y);
1023 auto *result = F->createSave("ret", CE);
1024 auto *LTensor = bindings.allocate(result->getPlaceholder());
1025
1026 Tensor inputs(ElemKind::FloatTy, {batchSize, 4});
1027 inputs.zero();
1028 Tensor outputs(ElemKind::Int64ITy, {batchSize});
1029 outputs.zero();
1030
1031 auto inputsH = inputs.getHandle();
1032 auto outputsH = outputs.getHandle<int64_t>();
1033
1034 inputsH.randomize(0.0, 1.0, mod.getPRNG());
1035 outputsH.at({0}) = 2;
1036 outputsH.at({1}) = 0;
1037 outputsH.at({2}) = 1;
1038
1039 VariableGradientsList varGrads;
1040 glow::differentiate(F, TC, "record", &varGrads);
1041 allocateGrads(bindings, varGrads);
1042 EET_.compile(CompilationMode::Train);
1043
1044 auto *gradPlaceholder = getGrad(varGrads, P);
1045 auto gradTensorHandle = bindings.get(gradPlaceholder)->getHandle();
1046
1047 for (int i = 0; i < testSamples; ++i) {
1048 inputsH.randomize(0.0, 1.0, mod.getPRNG());
1049 for (size_t j = 0; j < inputsH.size(); ++j) {
1050 updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs});
1051 EET_.run(bindings, "record");
1052 LTensor->zero();
1053 auto x = inputsH.raw(j);
1054 auto g = gradTensorHandle.raw(j);
1055 inputsH.raw(j) = x + stepSize;
1056 updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs});
1057 EET_.run(bindings, "record");
1058 auto lp = LTensor->getHandle().raw(0);
1059 inputsH.raw(j) = x - stepSize;
1060 LTensor->zero();
1061 updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs});
1062 EET_.run(bindings, "record");
1063 auto lm = LTensor->getHandle().raw(0);
1064 auto diff = (lp - lm) / (2 * stepSize);
1065 inputsH.raw(j) = x;
1066 updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs});
1067 EET_.run(bindings, "record");
1068 EXPECT_NEAR(diff, g, delta);
1069 }
1070 }
1071}
1072
1073TEST_P(GradCheck, gradientCheckBatchedPairwiseDotProduct) {
1074 CHECK_IF_ENABLED();
1075 PlaceholderBindings bindings;
1076 constexpr dim_t kBatchSize = 1;
1077 constexpr dim_t kVectorSize = 3;
1078 Constant *B, *C;
1079 Placeholder *A, *Exp;
1080 SaveNode *result;
1081 for (auto *EE : engines_) {
1082 auto &mod = EE->getModule();
1083 Function *F = mod.createFunction("main");
1084 A = mod.createPlaceholder(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "A",
1085 false);
1086 B = mod.createConstant(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "B");
1087 B->getPayloadMutable().getHandle().randomize(-1.0, 1.0, mod.getPRNG());
1088 C = mod.createConstant(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "C");
1089 C->getPayloadMutable().getHandle().randomize(-1.0, 1.0, mod.getPRNG());
1090
1091 Exp =
1092 mod.createPlaceholder(ElemKind::FloatTy, {kBatchSize, 3}, "Exp", false);
1093
1094 Node *N = F->createBatchedPairwiseDotProduct("dot", {A, B, C});
1095 N = F->createRegression("reg", N, Exp);
1096 result = F->createSave("ret", N);
1097 }
1098
1099 Tensor inputs(ElemKind::FloatTy, {{kBatchSize, kVectorSize}});
1100 Tensor outputs(ElemKind::FloatTy, {{kBatchSize, 3}});
1101
1102 auto inputsH = inputs.getHandle<>();
1103 auto outputsH = outputs.getHandle<>();
1104 auto &mod = EET_.getModule();
1105 inputsH.initXavier(1, mod.getPRNG());
1106 outputsH.initXavier(1, mod.getPRNG());
1107
1108 performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp,
1109 &inputs, &outputs, 0.001, 0.01);
1110}
1111
1112TEST_P(GradCheck, gradientCheckFC2) {
1113 CHECK_IF_ENABLED();
1114
1115 PlaceholderBindings bindings;
1116 Module &mod = EET_.getModule();
1117 Function *F = mod.createFunction("main");
1118
1119 // Create net representing A*X+Y=B, where X and Y are trainable, while
1120 // A and B are fixed. Record gradients for X and Y after 3 steps and compare
1121 // with reference values.
1122 TrainingConfig TC;
1123
1124 // This variable records the number of the next sample to be used for
1125 // training.
1126 size_t sampleCounter = 0;
1127
1128 auto *A = mod.createPlaceholder(ElemKind::FloatTy, {2, 1}, "A", false);
1129 auto *B = mod.createPlaceholder(ElemKind::FloatTy, {2, 1}, "B", false);
1130 auto *X = mod.createPlaceholder(ElemKind::FloatTy, {1, 1}, "X", true);
1131 auto *Y = mod.createPlaceholder(ElemKind::FloatTy, {1}, "Y", true);
1132
1133 bindings.allocate(A);
1134 bindings.allocate(B);
1135 bindings.allocate(X)->init(Tensor::InitKind::Broadcast, -1.26274,
1136 mod.getPRNG());
1137 bindings.allocate(Y)->init(Tensor::InitKind::Broadcast, 0.1, mod.getPRNG());
1138
1139 auto *FC = F->createFullyConnected("fc", A, X, Y);
1140 auto *S = F->createRegression("reg", FC, B);
1141 auto *save = F->createSave("ret", S);
1142 bindings.allocate(save->getPlaceholder());
1143
1144 Tensor initA(ElemKind::FloatTy, {2, 1});
1145 Tensor initB(ElemKind::FloatTy, {2, 1});
1146 initA.getHandle() = {4.2f, 9.875f};
1147 initB.getHandle() = {-13.1f, 3.14f};
1148
1149 Function *DF = glow::differentiate(F, TC, "d_main");
1150 auto dfName = DF->getName();
1151 EET_.compile(CompilationMode::Train);
1152 runBatch(EET_, bindings, 3, sampleCounter, {A, B}, {&initA, &initB}, dfName);
1153
1154 EXPECT_NEAR(bindings.get(X)->getHandle().raw(0), -0.21294, 1E-5);
1155 EXPECT_NEAR(bindings.get(Y)->getHandle().raw(0), 0.01656, 1E-5);
1156}
1157
1158INSTANTIATE_BACKEND_TEST(GradCheck);
1159