1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #include "BackendTestUtils.h" |
17 | |
18 | #include "glow/Base/Tensor.h" |
19 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
20 | #include "glow/Graph/Graph.h" |
21 | #include "glow/Graph/Nodes.h" |
22 | #include "glow/IR/IR.h" |
23 | #include "glow/IR/IRBuilder.h" |
24 | #include "glow/IR/Instrs.h" |
25 | |
26 | #include "gtest/gtest.h" |
27 | |
28 | using namespace glow; |
29 | using llvm::cast; |
30 | |
31 | class GradCheck : public ::testing::TestWithParam<std::string> { |
32 | public: |
33 | ExecutionEngine EET_{GetParam()}; |
34 | ExecutionEngine EEI_{GetParam()}; |
35 | std::vector<ExecutionEngine *> engines_; |
36 | void SetUp() { |
37 | engines_.push_back(&EEI_); |
38 | engines_.push_back(&EET_); |
39 | } |
40 | }; |
41 | |
42 | /// \returns the regression loss for the tensor \p X with regard to \p Y. |
43 | float computeL2Loss(Tensor *X, Tensor *Y) { |
44 | assert(X->dims() == Y->dims() && "Invalid input dims" ); |
45 | auto xH = X->getHandle<>(); |
46 | auto yH = Y->getHandle<>(); |
47 | float loss = 0; |
48 | |
49 | for (size_t i = 0, e = X->size(); i < e; i++) { |
50 | float dy = (xH.raw(i) - yH.raw(i)); |
51 | loss += 0.5 * dy * dy; |
52 | } |
53 | |
54 | return loss; |
55 | } |
56 | |
57 | /// \returns the error when comparing two grads: absolute or relative. |
58 | float gradDiff(float G1, float G2) { |
59 | return std::min(std::abs(G1 - G2), std::abs(G1 - G2) / std::abs(G1 + G2 + 1)); |
60 | } |
61 | |
62 | Placeholder *getGrad(const VariableGradientsList &grads, Placeholder *V) { |
63 | for (auto &p : grads) { |
64 | if (p.first == V) { |
65 | return p.second; |
66 | } |
67 | } |
68 | return nullptr; |
69 | } |
70 | |
71 | void allocateGrads(PlaceholderBindings &bindings, |
72 | const VariableGradientsList &grads) { |
73 | for (auto &p : grads) { |
74 | auto grad = p.second; |
75 | bindings.allocate(grad); |
76 | } |
77 | } |
78 | |
79 | /// Performs gradient check by comparing analytical and numerical gradients. |
80 | /// Numeric grad is calculated based on: f(x-delta) and f(x+delta) values. |
81 | /// Analytical grad is based on the gradient output calculated during back |
82 | /// propagation. |
83 | /// |
84 | /// \param EET Execution engine to compile/run network for training. |
85 | /// \param EEI Executiuon engine to compile/run network for inference. |
86 | /// \param bindings Placeholder bindings. |
87 | /// \param result Node that contains result of f(x). |
88 | /// \param inputVar Placeholder which gradient is assessed. |
89 | /// \param expVar Placeholder with expected value, only used during the |
90 | /// training. \param inputs Tensor for \p inputVar placeholder. \param outputs |
91 | /// Tensor for \p expVar placeholder. \param allowedError allowed delta between |
92 | /// analytical and numerical gradients. |
93 | void performGradCheck(ExecutionEngine &EET, ExecutionEngine &EEI, |
94 | PlaceholderBindings &bindings, Placeholder *result, |
95 | Placeholder *inputVar, Placeholder *expVar, |
96 | Tensor *inputs, Tensor *outputs, float delta, |
97 | float allowedError) { |
98 | TrainingConfig TC; |
99 | |
100 | auto *F = EET.getModule().getFunction("main" ); |
101 | |
102 | // Allocate result, inputVar and expVar. |
103 | bindings.allocate(result); |
104 | bindings.allocate(inputVar); |
105 | bindings.allocate(expVar); |
106 | |
107 | // This variable records the number of the next sample to be used for |
108 | // training. |
109 | size_t sampleCounter = 0; |
110 | |
111 | // Create a function that trains the network. |
112 | auto *TF = glow::differentiate(F, TC); |
113 | |
114 | // Create a version of the network that records the gradients to some side |
115 | // table instead of updating them. |
116 | VariableGradientsList varGrads; |
117 | auto *RF = glow::differentiate(F, TC, "record" , &varGrads); |
118 | auto rfName = RF->getName(); |
119 | auto tfName = TF->getName(); |
120 | EET.compile(CompilationMode::Train); |
121 | bindings.allocate(EET.getModule().getPlaceholders()); |
122 | |
123 | // The network might have variables, other than inputVar and expVar. |
124 | // Train the network until other variables reach some stable local minimum. |
125 | runBatch(EET, bindings, 300, sampleCounter, {inputVar, expVar}, |
126 | {inputs, outputs}, tfName); |
127 | |
128 | // Clear the gradients of the first layer. |
129 | auto gradVar = getGrad(varGrads, inputVar); |
130 | auto gradVarTensor = bindings.get(gradVar); |
131 | gradVarTensor->zero(); |
132 | |
133 | // Train the network just once to record the values of gradient for inputVar. |
134 | runBatch(EET, bindings, 1, sampleCounter, {inputVar, expVar}, |
135 | {inputs, outputs}, rfName); |
136 | // Compile the original network in inference mode. |
137 | EEI.compile(CompilationMode::Infer); |
138 | PlaceholderBindings inferBindings; |
139 | inferBindings.allocate(EEI.getModule().getPlaceholders()); |
140 | // Copy values to inferenceBindings. This is needed for any placeholders that |
141 | // were initialized during function creation. |
142 | for (auto PH : EEI.getModule().getPlaceholders()) { |
143 | bindings.copyToTarget(PH->getName(), inferBindings); |
144 | } |
145 | auto resultTensor = inferBindings.get( |
146 | inferBindings.getPlaceholderByNameSlow(result->getName())); |
147 | |
148 | auto analyticalGradsH = gradVarTensor->getHandle(); |
149 | auto inputsH = inputs->getHandle<>(); |
150 | bool allZeroGradients = true; |
151 | auto inferInput = inferBindings.getPlaceholderByNameSlow(inputVar->getName()); |
152 | |
153 | for (size_t i = 0; i < analyticalGradsH.size(); i++) { |
154 | auto old = inputsH.raw(i); |
155 | // Calculate f(x+e): |
156 | inputsH.raw(i) = old + delta; |
157 | updateInputPlaceholders(inferBindings, {inferInput}, {inputs}); |
158 | EEI.run(inferBindings); |
159 | auto plusLoss = computeL2Loss(outputs, resultTensor); |
160 | |
161 | // Calculate f(x-e): |
162 | inputsH.raw(i) = old - delta; |
163 | updateInputPlaceholders(inferBindings, {inferInput}, {inputs}); |
164 | EEI.run(inferBindings); |
165 | |
166 | auto minusLoss = computeL2Loss(outputs, resultTensor); |
167 | |
168 | // Restore value back. |
169 | inputsH.raw(i) = old; |
170 | |
171 | auto numericGrad = (plusLoss - minusLoss) / (2 * delta); |
172 | auto analyticalGrad = analyticalGradsH.raw(i); |
173 | auto err = gradDiff(analyticalGrad, numericGrad); |
174 | |
175 | if (numericGrad != 0.0) { |
176 | allZeroGradients = false; |
177 | } |
178 | // Make sure that the analytical and numerical gradients agree. |
179 | EXPECT_LE(err, allowedError); |
180 | } |
181 | |
182 | // Make sure that some gradients are non-zero. If all gradients are zero, |
183 | // it means that the input doesn't affect the output, which also means |
184 | // that the operations being test is totally ignored. |
185 | EXPECT_FALSE(allZeroGradients); |
186 | } |
187 | |
188 | TEST_P(GradCheck, gradientCheckConcat) { |
189 | CHECK_IF_ENABLED(); |
190 | PlaceholderBindings bindings; |
191 | Placeholder *A, *Exp, *B; |
192 | SaveNode *result; |
193 | dim_t numOutputElem = 20; |
194 | for (auto *EE : engines_) { |
195 | auto &mod = EE->getModule(); |
196 | Function *F = mod.createFunction("main" ); |
197 | |
198 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, "A" , |
199 | false); |
200 | B = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem / 2}, "B" , |
201 | false); |
202 | |
203 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp" , |
204 | false); |
205 | Node *O = F->createConcat("concat" , {A, B}, 1); |
206 | O = F->createRegression("reg" , O, Exp); |
207 | result = F->createSave("ret" , O); |
208 | } |
209 | bindings.allocate(B)->zero(); |
210 | Tensor inputs(ElemKind::FloatTy, {{1, numOutputElem / 2}}); |
211 | Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}}); |
212 | |
213 | auto inputsH = inputs.getHandle<>(); |
214 | auto outputsH = outputs.getHandle<>(); |
215 | auto &trainingMod = EET_.getModule(); |
216 | inputsH.randomize(-1, 1, trainingMod.getPRNG()); |
217 | outputsH.randomize(-1, 1, trainingMod.getPRNG()); |
218 | |
219 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
220 | &inputs, &outputs, 0.001, 0.01); |
221 | } |
222 | |
223 | TEST_P(GradCheck, gradientCheckMatMul) { |
224 | CHECK_IF_ENABLED(); |
225 | PlaceholderBindings Bindings; |
226 | Placeholder *A, *Exp, *B; |
227 | SaveNode *Result; |
228 | dim_t NumDim = 10; |
229 | for (auto *EE : engines_) { |
230 | auto &Mod = EE->getModule(); |
231 | Bindings.clear(); |
232 | Function *F = Mod.createFunction("main" ); |
233 | |
234 | A = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "A" , false); |
235 | B = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "B" , false); |
236 | |
237 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumDim, NumDim}, "exp" , |
238 | false); |
239 | auto HandleB = Bindings.allocate(B)->getHandle<float>(); |
240 | HandleB.randomize(-1, 1, Mod.getPRNG()); |
241 | Node *MM = F->createMatMul("matmul" , A, B); |
242 | auto *Reg = F->createRegression("reg" , MM, Exp); |
243 | Result = F->createSave("save" , Reg); |
244 | } |
245 | |
246 | Tensor Inputs(ElemKind::FloatTy, {{NumDim, NumDim}}); |
247 | Tensor Outputs(ElemKind::FloatTy, {{NumDim, NumDim}}); |
248 | |
249 | auto InputsH = Inputs.getHandle<>(); |
250 | auto OutputsH = Outputs.getHandle<>(); |
251 | auto &Mod = EET_.getModule(); |
252 | |
253 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
254 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
255 | |
256 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
257 | &Inputs, &Outputs, 0.001, 0.01); |
258 | } |
259 | |
260 | TEST_P(GradCheck, gradientCheckBatchMatMul) { |
261 | CHECK_IF_ENABLED(); |
262 | PlaceholderBindings Bindings; |
263 | Placeholder *A, *Exp, *B; |
264 | SaveNode *Result; |
265 | |
266 | constexpr dim_t BatchSize{4}, P{5}, Q{6}, R{7}; |
267 | |
268 | for (auto *EE : engines_) { |
269 | auto &Mod = EE->getModule(); |
270 | Bindings.clear(); |
271 | Function *F = Mod.createFunction("main" ); |
272 | |
273 | A = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, P, Q}, "A" , |
274 | /*isTrainable=*/false); |
275 | B = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, Q, R}, "B" , |
276 | /*isTrainable=*/false); |
277 | |
278 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, P, R}, "exp" , |
279 | /*isTrainable=*/false); |
280 | auto HandleB = Bindings.allocate(B)->getHandle<float>(); |
281 | HandleB.randomize(-1, 1, Mod.getPRNG()); |
282 | Node *MM = F->createBatchMatMul("batchMatmul" , A, B); |
283 | auto *Reg = F->createRegression("reg" , MM, Exp); |
284 | Result = F->createSave("save" , Reg); |
285 | } |
286 | |
287 | Tensor Inputs(ElemKind::FloatTy, {{BatchSize, P, Q}}); |
288 | Tensor Outputs(ElemKind::FloatTy, {{BatchSize, P, R}}); |
289 | |
290 | auto InputsH = Inputs.getHandle<>(); |
291 | auto OutputsH = Outputs.getHandle<>(); |
292 | auto &Mod = EET_.getModule(); |
293 | |
294 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
295 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
296 | |
297 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
298 | &Inputs, &Outputs, 0.001, 0.01); |
299 | } |
300 | |
301 | TEST_P(GradCheck, gradientCheckTile) { |
302 | CHECK_IF_ENABLED(); |
303 | PlaceholderBindings Bindings; |
304 | Placeholder *A, *Exp; |
305 | SaveNode *Result; |
306 | |
307 | constexpr dim_t N{2}, C{3}, H{4}, W{5}; |
308 | constexpr dim_t NumTiles{5}; |
309 | |
310 | for (auto *EE : engines_) { |
311 | auto &Mod = EE->getModule(); |
312 | Function *F = Mod.createFunction("main" ); |
313 | |
314 | A = Mod.createPlaceholder(ElemKind::FloatTy, {N, C, H, W}, "A" , |
315 | /*isTrainable=*/false); |
316 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {N, NumTiles * C, H, W}, |
317 | "Exp" , /*isTrainable=*/false); |
318 | auto *Tile = F->createTile("tile" , A, NumTiles, /*axis=*/1, Exp->getType()); |
319 | auto *Reg = F->createRegression("reg" , Tile, Exp); |
320 | Result = F->createSave("save" , Reg); |
321 | } |
322 | |
323 | Tensor Inputs(ElemKind::FloatTy, A->getType()->dims()); |
324 | Tensor Outputs(ElemKind::FloatTy, Exp->getType()->dims()); |
325 | |
326 | auto InputsH = Inputs.getHandle<>(); |
327 | auto OutputsH = Outputs.getHandle<>(); |
328 | auto &Mod = EET_.getModule(); |
329 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
330 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
331 | |
332 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
333 | &Inputs, &Outputs, 0.001, 0.01); |
334 | } |
335 | |
336 | TEST_P(GradCheck, gradientCheckBatchedReduceAddAxis0) { |
337 | CHECK_IF_ENABLED(); |
338 | PlaceholderBindings Bindings; |
339 | Placeholder *A, *Exp; |
340 | SaveNode *Result; |
341 | dim_t BatchSize = 4; |
342 | dim_t NumRows = 3; |
343 | dim_t NumCols = 5; |
344 | for (auto *EE : engines_) { |
345 | auto &Mod = EE->getModule(); |
346 | Function *F = Mod.createFunction("main" ); |
347 | |
348 | A = Mod.createPlaceholder(ElemKind::FloatTy, {BatchSize, NumRows, NumCols}, |
349 | "A" , false); |
350 | |
351 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, NumCols}, "exp" , |
352 | false); |
353 | |
354 | TypeRef Ty = Mod.uniqueType(ElemKind::FloatTy, {NumRows, NumCols}); |
355 | Node *BRA = F->createBatchedReduceAdd("BRA" , Ty, A, 0 /*axis*/); |
356 | auto *Reg = F->createRegression("reg" , BRA, Exp); |
357 | Result = F->createSave("save" , Reg); |
358 | } |
359 | |
360 | Tensor Inputs(ElemKind::FloatTy, {{BatchSize, NumRows, NumCols}}); |
361 | Tensor Outputs(ElemKind::FloatTy, {{NumRows, NumCols}}); |
362 | |
363 | auto InputsH = Inputs.getHandle<>(); |
364 | auto OutputsH = Outputs.getHandle<>(); |
365 | auto &Mod = EET_.getModule(); |
366 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
367 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
368 | |
369 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
370 | &Inputs, &Outputs, 0.001, 0.01); |
371 | } |
372 | |
373 | TEST_P(GradCheck, gradientCheckBatchedReduceAddAxis1) { |
374 | CHECK_IF_ENABLED(); |
375 | PlaceholderBindings Bindings; |
376 | dim_t NumRows = 3; |
377 | dim_t BatchSize = 4; |
378 | dim_t NumCols = 5; |
379 | Placeholder *A, *Exp; |
380 | SaveNode *Result; |
381 | for (auto *EE : engines_) { |
382 | auto &Mod = EE->getModule(); |
383 | Function *F = Mod.createFunction("main" ); |
384 | |
385 | A = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, BatchSize, NumCols}, |
386 | "A" , false); |
387 | |
388 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {NumRows, NumCols}, "exp" , |
389 | false); |
390 | |
391 | TypeRef Ty = Mod.uniqueType(ElemKind::FloatTy, {NumRows, NumCols}); |
392 | Node *BRA = F->createBatchedReduceAdd("BRA" , Ty, A, 1 /*axis*/); |
393 | auto *Reg = F->createRegression("reg" , BRA, Exp); |
394 | Result = F->createSave("save" , Reg); |
395 | } |
396 | |
397 | Tensor Inputs(ElemKind::FloatTy, {{NumRows, BatchSize, NumCols}}); |
398 | Tensor Outputs(ElemKind::FloatTy, {{NumRows, NumCols}}); |
399 | |
400 | auto InputsH = Inputs.getHandle<>(); |
401 | auto OutputsH = Outputs.getHandle<>(); |
402 | auto &Mod = EET_.getModule(); |
403 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
404 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
405 | |
406 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
407 | &Inputs, &Outputs, 0.001, 0.01); |
408 | } |
409 | |
410 | TEST_P(GradCheck, gradientCheckGatherVec) { |
411 | CHECK_IF_ENABLED(); |
412 | PlaceholderBindings Bindings; |
413 | Placeholder *A, *Exp; |
414 | SaveNode *Result; |
415 | for (auto *EE : engines_) { |
416 | auto &Mod = EE->getModule(); |
417 | Bindings.clear(); |
418 | Function *F = Mod.createFunction("main" ); |
419 | |
420 | A = Mod.createPlaceholder(ElemKind::FloatTy, {3, 4}, "A" , false); |
421 | auto *Indices = Mod.createPlaceholder(ElemKind::Int32ITy, {2}, "I" , false); |
422 | Bindings.allocate(Indices)->getHandle<int32_t>() = {0, 2}; |
423 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {2, 4}, "exp" , false); |
424 | |
425 | Node *G = F->createGather("gather" , A, Indices, 0 /*batchDims*/); |
426 | auto *Reg = F->createRegression("reg" , G, Exp); |
427 | Result = F->createSave("save" , Reg); |
428 | } |
429 | |
430 | Tensor Inputs(ElemKind::FloatTy, {{3, 4}}); |
431 | Tensor Outputs(ElemKind::FloatTy, {{2, 4}}); |
432 | |
433 | auto InputsH = Inputs.getHandle<>(); |
434 | auto OutputsH = Outputs.getHandle<>(); |
435 | auto &Mod = EET_.getModule(); |
436 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
437 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
438 | |
439 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
440 | &Inputs, &Outputs, 0.001, 0.01); |
441 | } |
442 | |
443 | TEST_P(GradCheck, gradientCheckGatherDim) { |
444 | CHECK_IF_ENABLED(); |
445 | PlaceholderBindings Bindings; |
446 | Placeholder *A, *Exp; |
447 | SaveNode *Result; |
448 | for (auto *EE : engines_) { |
449 | auto &Mod = EE->getModule(); |
450 | Bindings.clear(); |
451 | Function *F = Mod.createFunction("main" ); |
452 | |
453 | A = Mod.createPlaceholder(ElemKind::FloatTy, {8, 4}, "A" , false); |
454 | auto *Indices = |
455 | Mod.createPlaceholder(ElemKind::Int32ITy, {2, 2}, "I" , false); |
456 | Bindings.allocate(Indices)->getHandle<int32_t>() = {0, 2, 3, 1}; |
457 | Exp = Mod.createPlaceholder(ElemKind::FloatTy, {2, 2, 4}, "exp" , false); |
458 | |
459 | Node *G = F->createGather("gather" , A, Indices, 0 /*batchDims*/); |
460 | auto *Reg = F->createRegression("reg" , G, Exp); |
461 | Result = F->createSave("save" , Reg); |
462 | } |
463 | |
464 | Tensor Inputs(ElemKind::FloatTy, {{8, 4}}); |
465 | Tensor Outputs(ElemKind::FloatTy, {{2, 2, 4}}); |
466 | |
467 | auto InputsH = Inputs.getHandle<>(); |
468 | auto OutputsH = Outputs.getHandle<>(); |
469 | auto &Mod = EET_.getModule(); |
470 | InputsH.randomize(-1, 1, Mod.getPRNG()); |
471 | OutputsH.randomize(-1, 1, Mod.getPRNG()); |
472 | |
473 | performGradCheck(EET_, EEI_, Bindings, Result->getPlaceholder(), A, Exp, |
474 | &Inputs, &Outputs, 0.001, 0.01); |
475 | } |
476 | |
477 | static void gradientCheckGroupConv(dim_t depth, dim_t group, |
478 | ExecutionEngine &EET_, |
479 | ExecutionEngine &EEI_) { |
480 | PlaceholderBindings bindings; |
481 | dim_t numDim = 10; |
482 | std::vector<ExecutionEngine *> engines; |
483 | engines.push_back(&EEI_); |
484 | engines.push_back(&EET_); |
485 | Placeholder *A, *Ex; |
486 | SaveNode *result; |
487 | for (auto *EE : engines) { |
488 | auto &mod = EE->getModule(); |
489 | bindings.clear(); |
490 | Function *F = mod.createFunction("main" ); |
491 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth}, |
492 | "A" , false); |
493 | Ex = mod.createPlaceholder( |
494 | ElemKind::FloatTy, {1, numDim + 1, numDim + 1, depth}, "exp" , false); |
495 | |
496 | Node *O = F->createConv(bindings, "conv" , A, depth, 2, 1, 1, group); |
497 | O = F->createRegression("reg" , O, Ex); |
498 | result = F->createSave("ret" , O); |
499 | } |
500 | |
501 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, depth}); |
502 | Tensor outputs(ElemKind::FloatTy, {1, numDim + 1, numDim + 1, depth}); |
503 | |
504 | auto inputsH = inputs.getHandle<>(); |
505 | auto outputsH = outputs.getHandle<>(); |
506 | auto &mod = EET_.getModule(); |
507 | inputsH.randomize(-1, 1, mod.getPRNG()); |
508 | outputsH.randomize(-1, 1, mod.getPRNG()); |
509 | |
510 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex, |
511 | &inputs, &outputs, 0.001, 0.04); |
512 | } |
513 | |
514 | TEST_P(GradCheck, gradientCheckConv) { |
515 | CHECK_IF_ENABLED(); |
516 | gradientCheckGroupConv(1, 1, EET_, EEI_); |
517 | } |
518 | |
519 | TEST_P(GradCheck, gradientCheckDepthwiseConv) { |
520 | CHECK_IF_ENABLED(); |
521 | gradientCheckGroupConv(4, 4, EET_, EEI_); |
522 | } |
523 | |
524 | TEST_P(GradCheck, gradientCheckGroupConv) { |
525 | CHECK_IF_ENABLED(); |
526 | gradientCheckGroupConv(4, 2, EET_, EEI_); |
527 | } |
528 | |
529 | static void gradientCheckDilatedConv(dim_t depth, dim_t group, |
530 | unsigned_t dilation, ExecutionEngine &EET_, |
531 | ExecutionEngine &EEI_) { |
532 | PlaceholderBindings bindings; |
533 | dim_t numDim = 10; |
534 | std::vector<ExecutionEngine *> engines; |
535 | engines.push_back(&EEI_); |
536 | engines.push_back(&EET_); |
537 | Placeholder *A, *Ex; |
538 | SaveNode *result; |
539 | for (auto *EE : engines) { |
540 | auto &mod = EE->getModule(); |
541 | bindings.clear(); |
542 | Function *F = mod.createFunction("main" ); |
543 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth}, |
544 | "A" , false); |
545 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, depth}, |
546 | "exp" , false); |
547 | |
548 | Node *O = F->createConv(bindings, "conv" , A, depth, 2, 1, 1, group, |
549 | {dilation, dilation}); |
550 | O = F->createRegression("reg" , O, Ex); |
551 | result = F->createSave("ret" , O); |
552 | } |
553 | |
554 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, depth}); |
555 | Tensor outputs(ElemKind::FloatTy, {1, numDim, numDim, depth}); |
556 | |
557 | auto inputsH = inputs.getHandle<>(); |
558 | auto outputsH = outputs.getHandle<>(); |
559 | auto &mod = EET_.getModule(); |
560 | inputsH.randomize(-1, 1, mod.getPRNG()); |
561 | outputsH.randomize(-1, 1, mod.getPRNG()); |
562 | |
563 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex, |
564 | &inputs, &outputs, 0.001, 0.04); |
565 | } |
566 | |
567 | TEST_P(GradCheck, gradientCheckDilatedConv) { |
568 | CHECK_IF_ENABLED(); |
569 | gradientCheckDilatedConv(1, 1, 2, EET_, EEI_); |
570 | } |
571 | |
572 | TEST_P(GradCheck, gradientCheckAvgPool) { |
573 | CHECK_IF_ENABLED(); |
574 | PlaceholderBindings bindings; |
575 | dim_t numDim = 10; |
576 | dim_t numOutputElem = 10; |
577 | Placeholder *A, *Exp; |
578 | SaveNode *result; |
579 | for (auto *EE : engines_) { |
580 | auto &mod = EE->getModule(); |
581 | bindings.clear(); |
582 | Function *F = mod.createFunction("main" ); |
583 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A" , |
584 | false); |
585 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
586 | false); |
587 | |
588 | Node *O = F->createAvgPool("pool" , A, 3, 3, 1); |
589 | O = F->createTanh("tanh" , O); |
590 | O = F->createFullyConnected(bindings, "fc" , O, numOutputElem); |
591 | O = F->createRegression("reg" , O, Exp); |
592 | result = F->createSave("ret" , O); |
593 | } |
594 | |
595 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2}); |
596 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
597 | |
598 | auto inputsH = inputs.getHandle<>(); |
599 | auto outputsH = outputs.getHandle<>(); |
600 | auto &mod = EET_.getModule(); |
601 | inputsH.initXavier(1, mod.getPRNG()); |
602 | outputsH.initXavier(1, mod.getPRNG()); |
603 | |
604 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
605 | &inputs, &outputs, 0.001, 0.004); |
606 | } |
607 | |
608 | TEST_P(GradCheck, gradientCheckAvgPoolCountExcludePads) { |
609 | CHECK_IF_ENABLED(); |
610 | PlaceholderBindings bindings; |
611 | dim_t numDim = 10; |
612 | dim_t numOutputElem = 10; |
613 | Placeholder *A, *Exp; |
614 | SaveNode *result; |
615 | for (auto *EE : engines_) { |
616 | auto &mod = EE->getModule(); |
617 | bindings.clear(); |
618 | Function *F = mod.createFunction("main" ); |
619 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A" , |
620 | false); |
621 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
622 | false); |
623 | |
624 | Node *O = F->createAvgPool("pool" , A, 3, 3, 1, NHWC, |
625 | /* countIncludePads */ false); |
626 | O = F->createTanh("tanh" , O); |
627 | O = F->createFullyConnected(bindings, "fc" , O, numOutputElem); |
628 | O = F->createRegression("reg" , O, Exp); |
629 | result = F->createSave("ret" , O); |
630 | } |
631 | |
632 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2}); |
633 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
634 | |
635 | auto inputsH = inputs.getHandle<>(); |
636 | auto outputsH = outputs.getHandle<>(); |
637 | auto &mod = EET_.getModule(); |
638 | inputsH.initXavier(1, mod.getPRNG()); |
639 | outputsH.initXavier(1, mod.getPRNG()); |
640 | |
641 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
642 | &inputs, &outputs, 0.001, 0.004); |
643 | } |
644 | |
645 | TEST_P(GradCheck, gradientCheckMaxPool) { |
646 | CHECK_IF_ENABLED(); |
647 | PlaceholderBindings bindings; |
648 | dim_t numDim = 10; |
649 | dim_t numOutputElem = 10; |
650 | Placeholder *A, *Exp; |
651 | SaveNode *result; |
652 | for (auto *EE : engines_) { |
653 | auto &mod = EE->getModule(); |
654 | bindings.clear(); |
655 | Function *F = mod.createFunction("main" ); |
656 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 2}, "A" , |
657 | false); |
658 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
659 | false); |
660 | |
661 | MaxPoolNode *P = F->createMaxPool("pool" , A, 3, 3, 1); |
662 | Node *O = F->createTanh("tanh" , P->getResult()); |
663 | O = F->createFullyConnected(bindings, "fc" , O, numOutputElem); |
664 | O = F->createRegression("reg" , O, Exp); |
665 | result = F->createSave("ret" , O); |
666 | } |
667 | |
668 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 2}); |
669 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
670 | |
671 | auto inputsH = inputs.getHandle<>(); |
672 | auto outputsH = outputs.getHandle<>(); |
673 | auto &mod = EET_.getModule(); |
674 | inputsH.initXavier(1, mod.getPRNG()); |
675 | outputsH.initXavier(1, mod.getPRNG()); |
676 | |
677 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
678 | &inputs, &outputs, 0.001, 0.004); |
679 | } |
680 | |
681 | TEST_P(GradCheck, gradientCheckAdaptiveAvgPool) { |
682 | CHECK_IF_ENABLED(); |
683 | PlaceholderBindings bindings; |
684 | dim_t numDim = 10; |
685 | dim_t numOutputElem = 10; |
686 | Placeholder *A, *Exp; |
687 | SaveNode *result; |
688 | for (auto *EE : engines_) { |
689 | auto &mod = EE->getModule(); |
690 | bindings.clear(); |
691 | Function *F = mod.createFunction("main" ); |
692 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 1}, "A" , |
693 | false); |
694 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
695 | false); |
696 | |
697 | auto outTy = mod.uniqueType(ElemKind::FloatTy, {1, 3, 3, 1}); |
698 | Node *O = F->createAdaptiveAvgPool("pool" , A, outTy); |
699 | O = F->createTanh("tanh" , O); |
700 | O = F->createFullyConnected(bindings, "fc" , O, numOutputElem); |
701 | O = F->createRegression("reg" , O, Exp); |
702 | result = F->createSave("ret" , O); |
703 | } |
704 | |
705 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 1}); |
706 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
707 | |
708 | auto inputsH = inputs.getHandle<>(); |
709 | auto outputsH = outputs.getHandle<>(); |
710 | auto &mod = EET_.getModule(); |
711 | inputsH.initXavier(1, mod.getPRNG()); |
712 | outputsH.initXavier(1, mod.getPRNG()); |
713 | |
714 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
715 | &inputs, &outputs, 0.001, 0.005); |
716 | } |
717 | |
718 | TEST_P(GradCheck, gradientCheckBatchNorm) { |
719 | CHECK_IF_ENABLED(); |
720 | PlaceholderBindings bindings; |
721 | dim_t numDim = 5; |
722 | dim_t numOutputElem = numDim * numDim * 3; |
723 | Placeholder *A, *Ex; |
724 | SaveNode *result; |
725 | for (auto *EE : engines_) { |
726 | auto &mod = EE->getModule(); |
727 | bindings.clear(); |
728 | Function *F = mod.createFunction("main" ); |
729 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim, numDim, 3}, "A" , |
730 | false); |
731 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp" , |
732 | false); |
733 | |
734 | Node *O = F->createBatchNormalization(bindings, "batch" , A, 3, 0.0001, 0.9); |
735 | O = F->createReshape("reshape" , O, {1, numDim * numDim * 3}); |
736 | O = F->createRegression("reg" , O, Ex); |
737 | result = F->createSave("ret" , O); |
738 | } |
739 | |
740 | Tensor inputs(ElemKind::FloatTy, {1, numDim, numDim, 3}); |
741 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
742 | |
743 | auto inputsH = inputs.getHandle<>(); |
744 | auto outputsH = outputs.getHandle<>(); |
745 | auto &mod = EET_.getModule(); |
746 | inputsH.initXavier(1, mod.getPRNG()); |
747 | outputsH.initXavier(1, mod.getPRNG()); |
748 | |
749 | for (auto &elem : inputsH) { |
750 | elem = elem * 6 + 4; |
751 | } |
752 | |
753 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Ex, |
754 | &inputs, &outputs, 0.001, 0.004); |
755 | } |
756 | |
757 | TEST_P(GradCheck, gradientCheckArithmeticDiv) { |
758 | CHECK_IF_ENABLED(); |
759 | // The test creates a net: A / B = Exp. Where A is trainable weight, |
760 | // B and Exp are external data (initialized randomly once). SGD will find |
761 | // correct value for A, and then gradient check will be performed. |
762 | PlaceholderBindings bindings; |
763 | dim_t numDim = 10; |
764 | Placeholder *B, *Exp; |
765 | SaveNode *result; |
766 | for (auto *EE : engines_) { |
767 | auto &mod = EE->getModule(); |
768 | bindings.clear(); |
769 | Function *F = mod.createFunction("main" ); |
770 | auto *A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "A" , true); |
771 | B = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "B" , false); |
772 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "exp" , false); |
773 | |
774 | auto *ATensor = bindings.allocate(A); |
775 | ATensor->init(Tensor::InitKind::Xavier, 1.0, mod.getPRNG()); |
776 | |
777 | Node *O = F->createDiv("div" , A, B); |
778 | O = F->createRegression("reg" , O, Exp); |
779 | result = F->createSave("ret" , O); |
780 | } |
781 | |
782 | Tensor BValues(ElemKind::FloatTy, {1, numDim}); |
783 | Tensor ExpValues(ElemKind::FloatTy, {1, numDim}); |
784 | // Random values are in the range, so that all intermediate computations are |
785 | // not too small and not too large. |
786 | auto &mod = EET_.getModule(); |
787 | BValues.getHandle().randomize(0.1, 1, mod.getPRNG()); |
788 | ExpValues.getHandle().randomize(0.1, 1, mod.getPRNG()); |
789 | |
790 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), B, Exp, |
791 | &BValues, &ExpValues, 0.0001, 0.001); |
792 | } |
793 | |
794 | TEST_P(GradCheck, gradientCheckArithmetic) { |
795 | CHECK_IF_ENABLED(); |
796 | PlaceholderBindings bindings; |
797 | dim_t numDim = 20; |
798 | Placeholder *A, *Exp; |
799 | SaveNode *result; |
800 | for (auto *EE : engines_) { |
801 | auto &mod = EE->getModule(); |
802 | bindings.clear(); |
803 | Function *F = mod.createFunction("main" ); |
804 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "A" , false); |
805 | auto *B = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "B" , true); |
806 | auto *C = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "C" , false); |
807 | auto *D = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "D" , false); |
808 | auto *E = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "E" , false); |
809 | bindings.allocate(B)->zero(); |
810 | bindings.allocate(C)->zero(); |
811 | bindings.allocate(D)->zero(); |
812 | |
813 | // Randomize E to avoid div by zero. |
814 | auto *ETensor = bindings.allocate(E); |
815 | ETensor->getHandle().randomize(-1, 1, mod.getPRNG()); |
816 | |
817 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numDim}, "exp" , false); |
818 | |
819 | Node *O = F->createMul("mul" , A, B); |
820 | O = F->createAdd("add" , O, C); |
821 | O = F->createSub("sub" , D, O); |
822 | O = F->createDiv("div" , O, E); |
823 | O = F->createRegression("reg" , O, Exp); |
824 | result = F->createSave("ret" , O); |
825 | } |
826 | |
827 | Tensor inputs(ElemKind::FloatTy, {1, numDim}); |
828 | Tensor outputs(ElemKind::FloatTy, {1, numDim}); |
829 | |
830 | auto inputsH = inputs.getHandle<>(); |
831 | auto outputsH = outputs.getHandle<>(); |
832 | auto &mod = EET_.getModule(); |
833 | inputsH.randomize(-1, 1, mod.getPRNG()); |
834 | outputsH.randomize(-1, 1, mod.getPRNG()); |
835 | |
836 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
837 | &inputs, &outputs, 0.01, 0.004); |
838 | } |
839 | |
840 | TEST_P(GradCheck, gradientCheckFCConcatTanh) { |
841 | CHECK_IF_ENABLED(); |
842 | // Using the same gradient check test setup as gradientCheck_FC_Concat_RELU |
843 | PlaceholderBindings bindings; |
844 | dim_t numInputElem = 20; |
845 | dim_t numOutputElem = 10; |
846 | Placeholder *A, *Exp; |
847 | SaveNode *result; |
848 | for (auto *EE : engines_) { |
849 | auto &mod = EE->getModule(); |
850 | bindings.clear(); |
851 | Function *F = mod.createFunction("main" ); |
852 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A" , false); |
853 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
854 | false); |
855 | |
856 | Node *FA = F->createFullyConnected(bindings, "fc" , A, numOutputElem); |
857 | FA = F->createTanh("tanh" , FA); |
858 | FA = F->createRegression("reg" , FA, Exp); |
859 | result = F->createSave("ret" , FA); |
860 | } |
861 | Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}}); |
862 | Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}}); |
863 | |
864 | auto inputsH = inputs.getHandle<>(); |
865 | auto outputsH = outputs.getHandle<>(); |
866 | auto &mod = EET_.getModule(); |
867 | inputsH.initXavier(1, mod.getPRNG()); |
868 | outputsH.initXavier(1, mod.getPRNG()); |
869 | |
870 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
871 | &inputs, &outputs, 0.0001, 0.001); |
872 | } |
873 | |
874 | TEST_P(GradCheck, gradientCheckFC) { |
875 | CHECK_IF_ENABLED(); |
876 | PlaceholderBindings bindings; |
877 | dim_t numInputElem = 20; |
878 | dim_t numOutputElem = 10; |
879 | Placeholder *A, *Exp; |
880 | SaveNode *result; |
881 | for (auto *EE : engines_) { |
882 | auto &mod = EE->getModule(); |
883 | bindings.clear(); |
884 | Function *F = mod.createFunction("main" ); |
885 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A" , false); |
886 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
887 | false); |
888 | |
889 | Node *FA = F->createFullyConnected(bindings, "fc" , A, numOutputElem); |
890 | FA = F->createRegression("reg" , FA, Exp); |
891 | result = F->createSave("ret" , FA); |
892 | } |
893 | |
894 | Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}}); |
895 | Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}}); |
896 | |
897 | auto inputsH = inputs.getHandle<>(); |
898 | auto outputsH = outputs.getHandle<>(); |
899 | auto &mod = EET_.getModule(); |
900 | inputsH.initXavier(1, mod.getPRNG()); |
901 | outputsH.initXavier(1, mod.getPRNG()); |
902 | |
903 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
904 | &inputs, &outputs, 0.0001, 0.0001); |
905 | } |
906 | |
907 | TEST_P(GradCheck, gradientCheckSigmoid) { |
908 | CHECK_IF_ENABLED(); |
909 | PlaceholderBindings bindings; |
910 | dim_t numInputElem = 20; |
911 | dim_t numOutputElem = 20; |
912 | Placeholder *A, *Exp; |
913 | SaveNode *result; |
914 | for (auto *EE : engines_) { |
915 | auto &mod = EE->getModule(); |
916 | Function *F = mod.createFunction("main" ); |
917 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A" , false); |
918 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
919 | false); |
920 | |
921 | Node *FA = F->createSigmoid("sig" , A); |
922 | FA = F->createRegression("reg" , FA, Exp); |
923 | result = F->createSave("ret" , FA); |
924 | } |
925 | |
926 | Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}}); |
927 | Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}}); |
928 | |
929 | auto inputsH = inputs.getHandle<>(); |
930 | auto outputsH = outputs.getHandle<>(); |
931 | auto &mod = EET_.getModule(); |
932 | inputsH.initXavier(1, mod.getPRNG()); |
933 | outputsH.initXavier(1, mod.getPRNG()); |
934 | |
935 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
936 | &inputs, &outputs, 0.001, 0.01); |
937 | } |
938 | |
939 | TEST_P(GradCheck, gradientCheckRelu) { |
940 | CHECK_IF_ENABLED(); |
941 | PlaceholderBindings bindings; |
942 | dim_t numInputElem = 20; |
943 | dim_t numOutputElem = 20; |
944 | Placeholder *A, *Exp; |
945 | SaveNode *result; |
946 | for (auto *EE : engines_) { |
947 | auto &mod = EE->getModule(); |
948 | Function *F = mod.createFunction("main" ); |
949 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputElem}, "A" , false); |
950 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "Exp" , |
951 | false); |
952 | |
953 | Node *FA = F->createRELU("relu" , A); |
954 | FA = F->createRegression("reg" , FA, Exp); |
955 | result = F->createSave("ret" , FA); |
956 | } |
957 | |
958 | Tensor inputs(ElemKind::FloatTy, {{1, numInputElem}}); |
959 | Tensor outputs(ElemKind::FloatTy, {{1, numOutputElem}}); |
960 | |
961 | auto inputsH = inputs.getHandle<>(); |
962 | auto outputsH = outputs.getHandle<>(); |
963 | auto &mod = EET_.getModule(); |
964 | inputsH.initXavier(1, mod.getPRNG()); |
965 | outputsH.initXavier(1, mod.getPRNG()); |
966 | |
967 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
968 | &inputs, &outputs, 0.001, 0.01); |
969 | } |
970 | |
971 | TEST_P(GradCheck, gradientCheckTranspose) { |
972 | CHECK_IF_ENABLED(); |
973 | // Using the same gradient check test setup as gradientCheck_FC_Concat_RELU |
974 | PlaceholderBindings bindings; |
975 | dim_t numOutputElem = 10; |
976 | Placeholder *A, *Exp; |
977 | SaveNode *result; |
978 | for (auto *EE : engines_) { |
979 | auto &mod = EE->getModule(); |
980 | bindings.clear(); |
981 | Function *F = mod.createFunction("main" ); |
982 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, 5, 10, 5}, "input" , false, |
983 | "NHWC" ); |
984 | Exp = mod.createPlaceholder(ElemKind::FloatTy, {1, numOutputElem}, "exp" , |
985 | false); |
986 | Node *TA = F->createTranspose("transpose" , A, NHWC2NCHW); |
987 | TA = F->createFullyConnected(bindings, "fc" , TA, numOutputElem); |
988 | TA = F->createRegression("regress" , TA, Exp); |
989 | result = F->createSave("ret" , TA); |
990 | } |
991 | |
992 | Tensor inputs(ElemKind::FloatTy, {1, 5, 10, 5}); |
993 | Tensor outputs(ElemKind::FloatTy, {1, numOutputElem}); |
994 | |
995 | auto inputsH = inputs.getHandle<>(); |
996 | auto outputsH = outputs.getHandle<>(); |
997 | auto &mod = EET_.getModule(); |
998 | inputsH.randomize(-1, 1, mod.getPRNG()); |
999 | outputsH.randomize(-1, 1, mod.getPRNG()); |
1000 | |
1001 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
1002 | &inputs, &outputs, 0.0001, 0.001); |
1003 | } |
1004 | |
1005 | TEST_P(GradCheck, gradientCheckCrossEntropyLoss) { |
1006 | CHECK_IF_ENABLED(); |
1007 | const dim_t batchSize = 6; |
1008 | const int testSamples = 5; |
1009 | const float stepSize = 1e-4; |
1010 | const float delta = 0.015; |
1011 | TrainingConfig TC; |
1012 | PlaceholderBindings bindings; |
1013 | |
1014 | auto &mod = EET_.getModule(); |
1015 | Function *F = mod.createFunction("main" ); |
1016 | auto *P = |
1017 | mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 4}, "P" , false); |
1018 | bindings.allocate(P)->zero(); |
1019 | auto *Y = |
1020 | mod.createPlaceholder(ElemKind::Int64ITy, {batchSize}, "Labels" , false); |
1021 | bindings.allocate(Y)->zero(); |
1022 | Node *CE = F->createCrossEntropyLoss("celoss" , P, Y); |
1023 | auto *result = F->createSave("ret" , CE); |
1024 | auto *LTensor = bindings.allocate(result->getPlaceholder()); |
1025 | |
1026 | Tensor inputs(ElemKind::FloatTy, {batchSize, 4}); |
1027 | inputs.zero(); |
1028 | Tensor outputs(ElemKind::Int64ITy, {batchSize}); |
1029 | outputs.zero(); |
1030 | |
1031 | auto inputsH = inputs.getHandle(); |
1032 | auto outputsH = outputs.getHandle<int64_t>(); |
1033 | |
1034 | inputsH.randomize(0.0, 1.0, mod.getPRNG()); |
1035 | outputsH.at({0}) = 2; |
1036 | outputsH.at({1}) = 0; |
1037 | outputsH.at({2}) = 1; |
1038 | |
1039 | VariableGradientsList varGrads; |
1040 | glow::differentiate(F, TC, "record" , &varGrads); |
1041 | allocateGrads(bindings, varGrads); |
1042 | EET_.compile(CompilationMode::Train); |
1043 | |
1044 | auto *gradPlaceholder = getGrad(varGrads, P); |
1045 | auto gradTensorHandle = bindings.get(gradPlaceholder)->getHandle(); |
1046 | |
1047 | for (int i = 0; i < testSamples; ++i) { |
1048 | inputsH.randomize(0.0, 1.0, mod.getPRNG()); |
1049 | for (size_t j = 0; j < inputsH.size(); ++j) { |
1050 | updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs}); |
1051 | EET_.run(bindings, "record" ); |
1052 | LTensor->zero(); |
1053 | auto x = inputsH.raw(j); |
1054 | auto g = gradTensorHandle.raw(j); |
1055 | inputsH.raw(j) = x + stepSize; |
1056 | updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs}); |
1057 | EET_.run(bindings, "record" ); |
1058 | auto lp = LTensor->getHandle().raw(0); |
1059 | inputsH.raw(j) = x - stepSize; |
1060 | LTensor->zero(); |
1061 | updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs}); |
1062 | EET_.run(bindings, "record" ); |
1063 | auto lm = LTensor->getHandle().raw(0); |
1064 | auto diff = (lp - lm) / (2 * stepSize); |
1065 | inputsH.raw(j) = x; |
1066 | updateInputPlaceholders(bindings, {P, Y}, {&inputs, &outputs}); |
1067 | EET_.run(bindings, "record" ); |
1068 | EXPECT_NEAR(diff, g, delta); |
1069 | } |
1070 | } |
1071 | } |
1072 | |
1073 | TEST_P(GradCheck, gradientCheckBatchedPairwiseDotProduct) { |
1074 | CHECK_IF_ENABLED(); |
1075 | PlaceholderBindings bindings; |
1076 | constexpr dim_t kBatchSize = 1; |
1077 | constexpr dim_t kVectorSize = 3; |
1078 | Constant *B, *C; |
1079 | Placeholder *A, *Exp; |
1080 | SaveNode *result; |
1081 | for (auto *EE : engines_) { |
1082 | auto &mod = EE->getModule(); |
1083 | Function *F = mod.createFunction("main" ); |
1084 | A = mod.createPlaceholder(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "A" , |
1085 | false); |
1086 | B = mod.createConstant(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "B" ); |
1087 | B->getPayloadMutable().getHandle().randomize(-1.0, 1.0, mod.getPRNG()); |
1088 | C = mod.createConstant(ElemKind::FloatTy, {kBatchSize, kVectorSize}, "C" ); |
1089 | C->getPayloadMutable().getHandle().randomize(-1.0, 1.0, mod.getPRNG()); |
1090 | |
1091 | Exp = |
1092 | mod.createPlaceholder(ElemKind::FloatTy, {kBatchSize, 3}, "Exp" , false); |
1093 | |
1094 | Node *N = F->createBatchedPairwiseDotProduct("dot" , {A, B, C}); |
1095 | N = F->createRegression("reg" , N, Exp); |
1096 | result = F->createSave("ret" , N); |
1097 | } |
1098 | |
1099 | Tensor inputs(ElemKind::FloatTy, {{kBatchSize, kVectorSize}}); |
1100 | Tensor outputs(ElemKind::FloatTy, {{kBatchSize, 3}}); |
1101 | |
1102 | auto inputsH = inputs.getHandle<>(); |
1103 | auto outputsH = outputs.getHandle<>(); |
1104 | auto &mod = EET_.getModule(); |
1105 | inputsH.initXavier(1, mod.getPRNG()); |
1106 | outputsH.initXavier(1, mod.getPRNG()); |
1107 | |
1108 | performGradCheck(EET_, EEI_, bindings, result->getPlaceholder(), A, Exp, |
1109 | &inputs, &outputs, 0.001, 0.01); |
1110 | } |
1111 | |
1112 | TEST_P(GradCheck, gradientCheckFC2) { |
1113 | CHECK_IF_ENABLED(); |
1114 | |
1115 | PlaceholderBindings bindings; |
1116 | Module &mod = EET_.getModule(); |
1117 | Function *F = mod.createFunction("main" ); |
1118 | |
1119 | // Create net representing A*X+Y=B, where X and Y are trainable, while |
1120 | // A and B are fixed. Record gradients for X and Y after 3 steps and compare |
1121 | // with reference values. |
1122 | TrainingConfig TC; |
1123 | |
1124 | // This variable records the number of the next sample to be used for |
1125 | // training. |
1126 | size_t sampleCounter = 0; |
1127 | |
1128 | auto *A = mod.createPlaceholder(ElemKind::FloatTy, {2, 1}, "A" , false); |
1129 | auto *B = mod.createPlaceholder(ElemKind::FloatTy, {2, 1}, "B" , false); |
1130 | auto *X = mod.createPlaceholder(ElemKind::FloatTy, {1, 1}, "X" , true); |
1131 | auto *Y = mod.createPlaceholder(ElemKind::FloatTy, {1}, "Y" , true); |
1132 | |
1133 | bindings.allocate(A); |
1134 | bindings.allocate(B); |
1135 | bindings.allocate(X)->init(Tensor::InitKind::Broadcast, -1.26274, |
1136 | mod.getPRNG()); |
1137 | bindings.allocate(Y)->init(Tensor::InitKind::Broadcast, 0.1, mod.getPRNG()); |
1138 | |
1139 | auto *FC = F->createFullyConnected("fc" , A, X, Y); |
1140 | auto *S = F->createRegression("reg" , FC, B); |
1141 | auto *save = F->createSave("ret" , S); |
1142 | bindings.allocate(save->getPlaceholder()); |
1143 | |
1144 | Tensor initA(ElemKind::FloatTy, {2, 1}); |
1145 | Tensor initB(ElemKind::FloatTy, {2, 1}); |
1146 | initA.getHandle() = {4.2f, 9.875f}; |
1147 | initB.getHandle() = {-13.1f, 3.14f}; |
1148 | |
1149 | Function *DF = glow::differentiate(F, TC, "d_main" ); |
1150 | auto dfName = DF->getName(); |
1151 | EET_.compile(CompilationMode::Train); |
1152 | runBatch(EET_, bindings, 3, sampleCounter, {A, B}, {&initA, &initB}, dfName); |
1153 | |
1154 | EXPECT_NEAR(bindings.get(X)->getHandle().raw(0), -0.21294, 1E-5); |
1155 | EXPECT_NEAR(bindings.get(Y)->getHandle().raw(0), 0.01656, 1E-5); |
1156 | } |
1157 | |
1158 | INSTANTIATE_BACKEND_TEST(GradCheck); |
1159 | |