1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#include "BackendTestUtils.h"
17
18#include "glow/ExecutionEngine/ExecutionEngine.h"
19#include "glow/Graph/Graph.h"
20#include "glow/Quantization/Quantization.h"
21
22#include "gtest/gtest.h"
23
24#include "llvm/Support/Casting.h"
25#include "llvm/Support/raw_ostream.h"
26
27#include <string>
28
29using namespace glow;
30using llvm::isa;
31
32class TestRunnerBase : public ::testing::TestWithParam<std::string> {
33public:
34 ExecutionEngine EEI_{GetParam()};
35 ExecutionEngine EET_{GetParam()};
36 std::vector<ExecutionEngine *> engines_;
37 void SetUp() override {
38 // The order here is intentional, the tests assume that EET is the last in
39 // the list.
40 engines_.push_back(&EEI_);
41 engines_.push_back(&EET_);
42 }
43};
44
45class MLTest : public TestRunnerBase {};
46
47/// Use placeholders (and not variables) to learn the square root of two.
48TEST_P(MLTest, learnSqrt2Placeholder) {
49 CHECK_IF_ENABLED();
50 TrainingConfig TC;
51 PlaceholderBindings bindings;
52
53 TC.learningRate = 0.03;
54
55 auto &mod = EET_.getModule();
56 Function *F = mod.createFunction("Square root of 2");
57
58 auto *A = mod.createPlaceholder(ElemKind::FloatTy, {1}, "A", true);
59 auto *inputTensor = bindings.allocate(A);
60 inputTensor->init(Tensor::InitKind::Broadcast, 1, mod.getPRNG());
61
62 auto *E = mod.createPlaceholder(ElemKind::FloatTy, {1}, "Ex", false);
63 bindings.allocate(E)->getHandle() = {2};
64
65 auto *O = mod.createPlaceholder(ElemKind::FloatTy, {1}, "output", false);
66 bindings.allocate(O);
67
68 Node *M = F->createMul("Mult", A, A);
69 M = F->createRegression("reg", M, E);
70 SaveNode *SN = F->createSave("ret", M);
71
72 bindings.allocate(SN->getPlaceholder());
73
74 auto *TF = glow::differentiate(F, TC);
75 auto fName = TF->getName();
76 EET_.compile(CompilationMode::Train);
77
78 // Train the network:
79 for (int i = 0; i < 100; i++) {
80 EET_.run(bindings, fName);
81 }
82
83 float res = inputTensor->getHandle().at({0});
84 EXPECT_NEAR(res, 1.4142, 0.01);
85}
86
87TEST_P(MLTest, trainASimpleNetwork) {
88 CHECK_IF_ENABLED();
89 TrainingConfig TC;
90 PlaceholderBindings bindings;
91
92 // This variable records the number of the next sample to be used for
93 // training.
94 size_t sampleCounter = 0;
95
96 // Learning a single input vector.
97 TC.learningRate = 0.05;
98 Function *F;
99 Placeholder *A, *E;
100 for (auto *EE : engines_) {
101 auto &mod = EE->getModule();
102 F = mod.createFunction("trainASimpleNetwork");
103
104 // Create a variable with 1 input, which is a vector of 4 elements.
105 A = mod.createPlaceholder(ElemKind::FloatTy, {1, 4}, "A", false);
106 E = mod.createPlaceholder(ElemKind::FloatTy, {1, 4}, "E", false);
107 Node *O = F->createFullyConnected(bindings, "fc1", A, 10);
108 O = F->createSigmoid("sig1", O);
109 O = F->createFullyConnected(bindings, "fc2", O, 4);
110 O = F->createRegression("reg", O, E);
111 F->createSave("return", O);
112 }
113 // TODO if PHs aren't zeroed this will not always pass in release. Should
114 // check which operations are sensitive and update them to set AllocZero
115 // properly.
116 for (auto *PH : EET_.getModule().getPlaceholders()) {
117 PH->setAllocZero();
118 }
119 PlaceholderBindings trainingBindings;
120 trainingBindings.allocate(EET_.getModule().getPlaceholders());
121 auto *resPH = EEI_.getModule().getPlaceholderByNameSlow("return");
122
123 // Values for the input and output variables.
124 Tensor inputs(ElemKind::FloatTy, {1, 4});
125 Tensor expected(ElemKind::FloatTy, {1, 4});
126 inputs.getHandle<>() = {0.15f, 0.15f, 0.15f, 0.15f};
127 expected.getHandle<>() = {0.9f, 0.9f, 0.9f, 0.9f};
128
129 auto *TF = glow::differentiate(F, TC);
130 auto tfName = TF->getName();
131 auto fname = F->getName();
132 EET_.compile(CompilationMode::Train);
133
134 // Train the network. Learn 1000 batches.
135 runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, E},
136 {&inputs, &expected}, tfName);
137
138 // Testing the output vector.
139 PlaceholderBindings inferBindings;
140 inferBindings.allocate(EEI_.getModule().getPlaceholders());
141 A = EEI_.getModule().getPlaceholderByNameSlow("A");
142 EEI_.compile(CompilationMode::Infer);
143 trainingBindings.copyTrainableWeightsTo(inferBindings);
144 updateInputPlaceholders(inferBindings, {A}, {&inputs});
145
146 EEI_.run(inferBindings, fname);
147
148 auto RNWH = inferBindings.get(resPH)->getHandle();
149 (void)RNWH;
150
151 // Test the output:
152 EXPECT_NEAR(RNWH.at({0, 0}), 0.9, 0.05);
153}
154
155TEST_P(MLTest, simpleRegression) {
156 CHECK_IF_ENABLED();
157 TrainingConfig TC;
158 PlaceholderBindings trainingBindings, inferBindings;
159
160 // This variable records the number of the next sample to be used for
161 // training.
162 size_t sampleCounter = 0;
163
164 // Testing the regression layer. This test takes the first element from the
165 // input vector, adds one to it and places the result in the second element of
166 // the output vector.
167 const dim_t numInputs = 4;
168
169 // Learning a single input vector.
170 TC.learningRate = 0.05;
171
172 Tensor inputs(ElemKind::FloatTy, {1, numInputs});
173 Tensor expected(ElemKind::FloatTy, {1, numInputs});
174 Placeholder *A, *Ex;
175 Function *F;
176 for (auto *EE : engines_) {
177 auto &mod = EE->getModule();
178 F = mod.createFunction("simpleRegression");
179 A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputs}, "A", false);
180 Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputs}, "E", false);
181 Node *O = F->createFullyConnected(inferBindings, "fc", A, 4);
182 O = F->createRELU("relu", O);
183 O = F->createRegression("reg", O, Ex);
184 F->createSave("result", O);
185 }
186 auto resPH = EEI_.getModule().getPlaceholderByNameSlow("result");
187 trainingBindings.allocate(EET_.getModule().getPlaceholders());
188 inferBindings.copyTrainableWeightsTo(trainingBindings);
189 inferBindings.clear();
190
191 auto I = inputs.getHandle<>();
192 auto E = expected.getHandle<>();
193
194 auto *TF = glow::differentiate(F, TC);
195 auto tfName = TF->getName();
196 auto fName = F->getName();
197 EET_.compile(CompilationMode::Train);
198
199 // Train the network:
200 for (int iter = 0; iter < 1000; iter++) {
201 float target = float(iter % 9);
202 I = {target, 0., 0., 0.};
203 E = {0., target + 1, 0., 0.};
204 runBatch(EET_, trainingBindings, 1, sampleCounter, {A, Ex},
205 {&inputs, &expected}, tfName);
206 }
207
208 // Verify the result of the regression layer.
209 inferBindings.allocate(EEI_.getModule().getPlaceholders());
210 trainingBindings.copyTrainableWeightsTo(inferBindings);
211 A = inferBindings.getPlaceholderByNameSlow("A");
212 EEI_.compile(CompilationMode::Infer);
213
214 // Test the output:
215 for (int iter = 0; iter < 5; iter++) {
216 float target = iter % 9 + 1;
217 I = {target, 0., 0., 0.};
218 updateInputPlaceholders(inferBindings, {A}, {&inputs});
219 EEI_.run(inferBindings, fName);
220 auto *res = inferBindings.get(resPH);
221 auto resH = res->getHandle<>();
222 (void)resH;
223
224 EXPECT_NEAR(I.at({0, 0}) + 1, resH.at({0, 1}), 0.1);
225 }
226}
227
228TEST_P(MLTest, learnXor) {
229 CHECK_IF_ENABLED();
230 TrainingConfig TC;
231 PlaceholderBindings trainingBindings, inferBindings;
232
233 unsigned numInputs = 10;
234
235 // This variable records the number of the next sample to be used for
236 // training.
237 size_t sampleCounter = 0;
238
239 // Learning a single input vector.
240 TC.learningRate = 0.05;
241 TC.batchSize = numInputs;
242 Placeholder *A, *Ex;
243 Function *F;
244 for (auto *EE : engines_) {
245 auto &mod = EE->getModule();
246 F = mod.createFunction("learnXor");
247
248 A = mod.createPlaceholder(ElemKind::FloatTy, {numInputs, 2}, "A", false);
249 Ex = mod.createPlaceholder(ElemKind::FloatTy, {numInputs, 1}, "Ex", false);
250
251 Node *O = F->createFullyConnected(inferBindings, "fc1", A, 6);
252 O = F->createTanh("tanh1", O);
253 O = F->createFullyConnected(inferBindings, "fc2", O, 1);
254 O = F->createRegression("reg", O, Ex);
255 F->createSave("ret", O);
256 }
257 trainingBindings.allocate(EET_.getModule().getPlaceholders());
258 inferBindings.copyTrainableWeightsTo(trainingBindings);
259 inferBindings.clear();
260 inferBindings.allocate(EEI_.getModule().getPlaceholders());
261
262 auto *res =
263 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
264
265 // Prepare the training set and the testing set.
266 Tensor trainingSet(ElemKind::FloatTy, {numInputs, 2});
267 Tensor trainingLabels(ElemKind::FloatTy, {numInputs, 1});
268
269 auto TS = trainingSet.getHandle<>();
270 auto TL = trainingLabels.getHandle<>();
271
272 // Prepare the training data:
273 for (unsigned i = 0; i < numInputs; i++) {
274 int a = i % 2;
275 int b = (i >> 1) % 2;
276 TS.at({i, 0}) = a;
277 TS.at({i, 1}) = b;
278 TL.at({i, 0}) = a ^ b;
279 }
280
281 auto *TF = glow::differentiate(F, TC);
282 auto tfName = TF->getName();
283 auto fname = F->getName();
284 EET_.compile(CompilationMode::Train);
285
286 // Train the network:
287 runBatch(EET_, trainingBindings, 2500, sampleCounter, {A, Ex},
288 {&trainingSet, &trainingLabels}, tfName);
289 trainingBindings.copyTrainableWeightsTo(inferBindings);
290 EEI_.compile(CompilationMode::Infer);
291
292 // Prepare the testing tensor:
293 for (unsigned i = 0; i < numInputs; i++) {
294 int a = (numInputs - i) % 2;
295 int b = ((numInputs - i) >> 1) % 2;
296 TS.at({i, 0}) = a;
297 TS.at({i, 1}) = b;
298 }
299 A = inferBindings.getPlaceholderByNameSlow("A");
300 updateInputPlaceholders(inferBindings, {A}, {&trainingSet});
301 EEI_.run(inferBindings, fname);
302
303 auto resH = res->getHandle<>();
304
305 // Test the output:
306 for (dim_t i = 0; i < numInputs; i++) {
307 int a = TS.at({i, 0});
308 int b = TS.at({i, 1});
309 EXPECT_NEAR(resH.at({i, 0}), (a ^ b), 0.1);
310 }
311}
312
313/// Learn the logarithmic function.
314TEST_P(MLTest, learnLog) {
315 CHECK_IF_ENABLED();
316 TrainingConfig TC;
317 PlaceholderBindings inferBindings, trainingBindings;
318
319 // This variable records the number of the next sample to be used for
320 // training.
321 size_t sampleCounter = 0;
322
323 unsigned numInputs = 50;
324 unsigned batchSize = 5;
325 TC.learningRate = 0.07;
326 TC.batchSize = batchSize;
327 Function *F;
328 Placeholder *A, *Ex;
329 for (auto *EE : engines_) {
330 auto &mod = EE->getModule();
331 F = mod.createFunction("learnLog");
332
333 A = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 1}, "A", false);
334 Ex = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 1}, "Ex", false);
335
336 Node *O = F->createFullyConnected(inferBindings, "fc1", A, 4);
337 O = F->createTanh("tanh1", O);
338 O = F->createFullyConnected(inferBindings, "fc2", O, 1);
339 O = F->createRegression("reg", O, Ex);
340 F->createSave("ret", O);
341 }
342 trainingBindings.allocate(EET_.getModule().getPlaceholders());
343 inferBindings.copyTrainableWeightsTo(trainingBindings);
344 inferBindings.clear();
345 inferBindings.allocate(EEI_.getModule().getPlaceholders());
346
347 auto *res =
348 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
349
350 // Set the training data.
351 Tensor trainingSet(ElemKind::FloatTy, {numInputs, 1});
352 Tensor trainingLabels(ElemKind::FloatTy, {numInputs, 1});
353
354 auto TS = trainingSet.getHandle<>();
355 auto TL = trainingLabels.getHandle<>();
356
357 // Set the training data as floating number from range [0.75, 1.5).
358 const float LO = 0.75; // Lower bound of training data.
359 const float HI = 1.5; // Upper bound of training data.
360 for (dim_t i = 0; i < numInputs; i++) {
361 // Generate a floating number in the range of [LO,HI).
362 float a = LO + i * (HI - LO) / numInputs;
363 TS.at({i, 0}) = a;
364 TL.at({i, 0}) = std::log(a);
365 }
366
367 auto *TF = glow::differentiate(F, TC);
368 auto tfName = TF->getName();
369 auto fname = F->getName();
370 EET_.compile(CompilationMode::Train);
371
372 // Train the network:
373 runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, Ex},
374 {&trainingSet, &trainingLabels}, tfName);
375 trainingBindings.copyTrainableWeightsTo(inferBindings);
376 EEI_.compile(CompilationMode::Infer);
377
378 // Set the testing data.
379 Tensor testSet(ElemKind::FloatTy, {batchSize, 1});
380
381 auto TES = testSet.getHandle<>();
382
383 const float LO_T = 0.85; // Lower bound of testing data.
384 const float HI_T = 1.45; // Upper bound of testing data.
385
386 for (dim_t i = 0; i < batchSize; i++) {
387 // Generate a floating number in the range of [LO_T,HI_T).
388 float a = EEI_.getModule().getPRNG().nextRandReal(LO_T, HI_T);
389 TES.at({i, 0}) = a;
390 }
391 A = inferBindings.getPlaceholderByNameSlow("A");
392 updateInputPlaceholders(inferBindings, {A}, {&testSet});
393 EEI_.run(inferBindings, fname);
394
395 auto resH = res->getHandle<>();
396
397 // Test the output:
398 for (dim_t i = 0; i < batchSize; i++) {
399 float a = TES.at({i, 0});
400 EXPECT_NEAR(resH.at({i, 0}), (std::log(a)), 0.02);
401 }
402}
403
404unsigned numSamples = 230;
405
406/// Generate data in two classes. The circle of dots that's close to the axis is
407/// L0, and the rest of the dots, away from the axis are L1.
408void generateCircleData(Tensor &coordinates, Tensor &labels, PseudoRNG &PRNG) {
409 auto C = coordinates.getHandle<>();
410 auto L = labels.getHandle<int64_t>();
411
412 for (dim_t i = 0; i < numSamples / 2; i++) {
413 float r = PRNG.nextRand() * 0.4;
414 float a = PRNG.nextRand() * 3.141592 * 2;
415 float y = r * sin(a);
416 float x = r * cos(a);
417
418 C.at({i * 2, 0u}) = x;
419 C.at({i * 2, 1u}) = y;
420 L.at({i * 2, 0}) = 1;
421
422 r = PRNG.nextRand() * 0.4 + 0.8;
423 a = PRNG.nextRand() * 3.141592 * 2;
424 y = r * sin(a);
425 x = r * cos(a);
426
427 C.at({i * 2 + 1, 0u}) = x;
428 C.at({i * 2 + 1, 1u}) = y;
429 L.at({i * 2 + 1, 0}) = 0;
430 }
431}
432
433/// Test the fully connected layer and the softmax function.
434/// Example from:
435/// http://cs.stanford.edu/people/karpathy/convnetjs/demo/classify2d.html
436TEST_P(MLTest, circle) {
437 CHECK_IF_ENABLED();
438 TrainingConfig TC;
439 PlaceholderBindings trainingBindings, inferBindings;
440
441 // This variable records the number of the next sample to be used for
442 // training.
443 size_t sampleCounter = 0;
444
445 unsigned minibatchSize = 11;
446
447 // Testing the softmax layer.
448 // Learning a single input vector.
449 TC.momentum = 0.9;
450 TC.learningRate = 0.01;
451 TC.batchSize = minibatchSize;
452 Function *F;
453 Placeholder *A, *S;
454 for (auto *EE : engines_) {
455 auto &mod = EE->getModule();
456 F = mod.createFunction("circle");
457 A = mod.createPlaceholder(ElemKind::FloatTy, {minibatchSize, 2}, "A",
458 false);
459 S = mod.createPlaceholder(ElemKind::Int64ITy, {minibatchSize, 1}, "S",
460 false);
461
462 auto *FCL0 = F->createFullyConnected(inferBindings, "fc1", A, 8);
463 auto *T0 = F->createTanh("tanh1", FCL0);
464 auto *FCL1 = F->createFullyConnected(inferBindings, "fc2", T0, 2);
465 auto *SM = F->createSoftMax("soft", FCL1, S);
466 F->createSave("ret", SM);
467 }
468 trainingBindings.allocate(EET_.getModule().getPlaceholders());
469 inferBindings.copyTrainableWeightsTo(trainingBindings);
470 inferBindings.clear();
471 inferBindings.allocate(EEI_.getModule().getPlaceholders());
472 auto *res =
473 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
474
475 auto *TF = glow::differentiate(F, TC);
476 auto tfName = TF->getName();
477 auto fname = F->getName();
478 EET_.compile(CompilationMode::Train);
479 trainingBindings.allocate(EET_.getModule().getPlaceholders());
480
481 Tensor coordinates(ElemKind::FloatTy, {numSamples, 2});
482 Tensor labels(ElemKind::Int64ITy, {numSamples, 1});
483 generateCircleData(coordinates, labels, EET_.getModule().getPRNG());
484
485 // Training:
486 runBatch(EET_, trainingBindings, 4000, sampleCounter, {A, S},
487 {&coordinates, &labels}, tfName);
488 trainingBindings.copyTrainableWeightsTo(inferBindings);
489 EEI_.compile(CompilationMode::Infer);
490 A = inferBindings.getPlaceholderByNameSlow("A");
491 // Print a diagram that depicts the network decision on a grid.
492 Tensor sample(ElemKind::FloatTy, {minibatchSize, 2});
493 sample.zero();
494 for (int x = -10; x < 10; x++) {
495 for (int y = -10; y < 10; y++) {
496 // Load the inputs:
497 sample.getHandle<>().at({0, 0}) = float(x) / 10;
498 sample.getHandle<>().at({0, 1}) = float(y) / 10;
499
500 updateInputPlaceholders(inferBindings, {A}, {&sample});
501 EEI_.run(inferBindings, fname);
502
503 auto SMH = res->getHandle<>();
504 auto A = SMH.at({0, 0});
505 auto B = SMH.at({0, 1});
506
507 char ch = '=';
508 if (A > (B + 0.2)) {
509 ch = '+';
510 } else if (B > (A + 0.2)) {
511 ch = '-';
512 }
513
514 llvm::outs() << ch;
515 }
516 llvm::outs() << "\n";
517 }
518 llvm::outs() << "\n";
519
520 {
521 // The dot in the middle must be one.
522 sample.getHandle<>().at({0, 0}) = 0;
523 sample.getHandle<>().at({0, 1}) = 0;
524 updateInputPlaceholders(inferBindings, {A}, {&sample});
525 EEI_.run(inferBindings, fname);
526
527 auto SMH = res->getHandle<>();
528 auto A = SMH.at({0, 0});
529 auto B = SMH.at({0, 1});
530 EXPECT_TRUE(B > (A + 0.2));
531 }
532
533 {
534 // Far away dot must be zero.
535 sample.getHandle<>().at({0, 0}) = 1;
536 sample.getHandle<>().at({0, 1}) = 1;
537 updateInputPlaceholders(inferBindings, {A}, {&sample});
538 EEI_.run(inferBindings, fname);
539 auto SMH = res->getHandle<>();
540 auto A = SMH.at({0, 0});
541 auto B = SMH.at({0, 1});
542 EXPECT_TRUE(A > (B + 0.2));
543 }
544}
545
546TEST_P(MLTest, learnSingleValueConcat) {
547 CHECK_IF_ENABLED();
548 unsigned width = 6;
549 PlaceholderBindings inferBindings, trainingBindings;
550
551 // This variable records the number of the next sample to be used for
552 // training.
553 size_t sampleCounter = 0;
554
555 // Learning a single input vector.
556 TrainingConfig TC;
557 TC.momentum = 0.9;
558 TC.learningRate = 0.01;
559 Function *F;
560 Placeholder *A, *Ex, *B;
561 for (auto *EE : engines_) {
562 auto &mod = EE->getModule();
563 F = mod.createFunction("learnSingleValueConcat");
564
565 Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, width * 2}, "Ex", false);
566
567 // Left side of the network:
568 A = mod.createPlaceholder(ElemKind::FloatTy, {1, width}, "A", false);
569 Node *L = F->createFullyConnected(inferBindings, "fc1", A, width);
570 L = F->createSigmoid("", L);
571
572 // Right side of the network:
573 B = mod.createPlaceholder(ElemKind::FloatTy, {1, width}, "B", false);
574 Node *R = F->createFullyConnected(inferBindings, "fc2", B, width);
575 R = F->createSigmoid("sig", R);
576
577 // Concat:
578 auto *C = F->createConcat("con", {L, R}, 1);
579 auto *RN = F->createRegression("reg", C, Ex);
580 F->createSave("ret", RN);
581 }
582
583 trainingBindings.allocate(EET_.getModule().getPlaceholders());
584 inferBindings.copyTrainableWeightsTo(trainingBindings);
585 inferBindings.clear();
586 inferBindings.allocate(EEI_.getModule().getPlaceholders());
587 auto *res =
588 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
589
590 Tensor inputs(ElemKind::FloatTy, {1, width});
591 Tensor expected(ElemKind::FloatTy, {1, width * 2});
592 inputs.getHandle<>().clear(0.15);
593 expected.getHandle<>().clear(0.9);
594 auto *TF = glow::differentiate(F, TC);
595 auto tfName = TF->getName();
596 EET_.compile(CompilationMode::Train);
597 trainingBindings.allocate(EET_.getModule().getPlaceholders());
598
599 // Train the network:
600 runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, B, Ex},
601 {&inputs, &inputs, &expected}, tfName);
602 trainingBindings.copyTrainableWeightsTo(inferBindings);
603 EEI_.compile(CompilationMode::Infer);
604 A = inferBindings.getPlaceholderByNameSlow("A");
605 // Testing the output vector.
606 updateInputPlaceholders(inferBindings, {A}, {&inputs});
607 EEI_.run(inferBindings);
608 auto RNWH = res->getHandle<>();
609 (void)RNWH;
610
611 // Test the output:
612 EXPECT_NEAR(RNWH.at({0, 0}), 0.9, 0.1);
613}
614
615void buildGRU(PlaceholderBindings &bindings, Function *F,
616 const std::vector<NodeValue> &slicesX, unsigned hiddenSize,
617 unsigned outputSize, std::vector<NodeValue> &outputs) {
618 return F->createGRU(bindings, "GRU", slicesX, 1, hiddenSize, outputSize,
619 outputs);
620};
621
622void buildRNN(PlaceholderBindings &bindings, Function *F,
623 const std::vector<NodeValue> &slicesX, unsigned hiddenSize,
624 unsigned outputSize, std::vector<NodeValue> &outputs) {
625 return F->createSimpleRNN(bindings, "SimpleRNN", slicesX, 1, hiddenSize,
626 outputSize, outputs);
627};
628
629void buildLSTM(PlaceholderBindings &bindings, Function *F,
630 const std::vector<NodeValue> &slicesX, unsigned hiddenSize,
631 unsigned outputSize, std::vector<NodeValue> &outputs) {
632 return F->createLSTM(bindings, "LSTM", slicesX, 1, hiddenSize, outputSize,
633 outputs);
634};
635
636using TCellGenerator = void (*)(PlaceholderBindings &, Function *,
637 const std::vector<NodeValue> &, unsigned,
638 unsigned, std::vector<NodeValue> &);
639
640void testRNNCell(TCellGenerator cell) {
641 TrainingConfig TC;
642
643 // This variable records the number of the next sample to be used for
644 // training.
645 size_t sampleCounter = 0;
646
647 PlaceholderBindings inferBindings, trainingBindings;
648 ExecutionEngine EEI, EET;
649 std::vector<ExecutionEngine *> engines;
650 engines.push_back(&EEI);
651 engines.push_back(&EET);
652 const unsigned NumVectors = 3;
653 const unsigned NumElements = 4;
654 // Learning a single input vector.
655 TC.learningRate = 0.05;
656 Function *F;
657 Placeholder *X, *Y;
658 for (auto *EE : engines) {
659 auto &mod = EE->getModule();
660 F = mod.createFunction("testRNNCell");
661
662 // Create a variable with 1 input, which is 3 consecutive vectors
663 // of 4 elements each.
664 X = mod.createPlaceholder(ElemKind::FloatTy, {1, NumVectors, NumElements},
665 "X", false);
666 Y = mod.createPlaceholder(ElemKind::FloatTy, {1, NumVectors}, "Y", false);
667 inferBindings.allocate(X);
668 inferBindings.allocate(Y);
669
670 // Extract a slice for each input.
671 std::vector<NodeValue> XSliced;
672
673 for (unsigned i = 0; i < NumVectors; ++i) {
674 std::string Name{"X"};
675 Name.append(std::to_string(i + 1));
676 XSliced.push_back(F->createSlice(Name, X, {0, i, 0}, {1, i + 1, 4}));
677 }
678
679 // Extract a slice for each output.
680 std::vector<Node *> YSliced;
681
682 for (unsigned i = 0; i < NumVectors; ++i) {
683 std::string Name{"Y"};
684 Name.append(std::to_string(i + 1));
685 YSliced.push_back(F->createSlice(Name, Y, {0, i}, {1, i + 1}));
686 }
687
688 const unsigned hiddenSize = 5;
689 const unsigned outputSize = 1;
690
691 std::vector<NodeValue> outputNodes;
692 cell(inferBindings, F, XSliced, hiddenSize, outputSize, outputNodes);
693
694 std::vector<NodeValue> regressionNodes;
695 for (unsigned t = 0; t < NumVectors; t++) {
696 regressionNodes.push_back(
697 F->createRegression("", outputNodes[t], YSliced[t]));
698 };
699
700 auto *R = F->createConcat("O", regressionNodes, 1);
701 F->createSave("result", R);
702 }
703 // TODO if PHs aren't zeroed this will not always pass in release. Should
704 // check which operations are sensitive and update them to set AllocZero
705 // properly.
706 for (auto *PH : EEI.getModule().getPlaceholders()) {
707 PH->setAllocZero();
708 }
709 for (auto *PH : EET.getModule().getPlaceholders()) {
710 PH->setAllocZero();
711 }
712 trainingBindings.allocate(EET.getModule().getPlaceholders());
713 inferBindings.copyTrainableWeightsTo(trainingBindings);
714 inferBindings.clear();
715 inferBindings.allocate(EEI.getModule().getPlaceholders());
716 auto *res =
717 inferBindings.get(EEI.getModule().getPlaceholderByNameSlow("result"));
718
719 auto *TF = glow::differentiate(F, TC);
720 auto tfName = TF->getName();
721 auto fname = F->getName();
722 EET.compile(CompilationMode::Train);
723
724 // Values for the input and output variables.
725 Tensor inputs(ElemKind::FloatTy, {1, NumVectors, NumElements});
726 Tensor expected(ElemKind::FloatTy, {1, NumVectors});
727 inputs.zero();
728 expected.zero();
729 for (dim_t i = 0; i < NumVectors; i++) {
730 inputs.getHandle<float_t>().at({0, i, 1}) = i;
731 expected.getHandle<float_t>().at({0, i}) = i;
732 }
733
734 // Train the network. Learn 1000 batches.
735 runBatch(EET, trainingBindings, 1000, sampleCounter, {X, Y},
736 {&inputs, &expected}, tfName);
737 trainingBindings.copyTrainableWeightsTo(inferBindings);
738 // Testing the output vector.
739 EEI.compile(CompilationMode::Infer);
740 X = inferBindings.getPlaceholderByNameSlow("X");
741 updateInputPlaceholders(inferBindings, {X}, {&inputs});
742 EEI.run(inferBindings, fname);
743
744 auto RNWH = res->getHandle<>();
745 (void)RNWH;
746
747 // Test the output:
748 for (dim_t t = 0; t < NumVectors; ++t) {
749 EXPECT_NEAR(RNWH.at({0, t}), t, 0.05);
750 }
751};
752
753TEST_P(MLTest, trainASimpleRNN) {
754 CHECK_IF_ENABLED();
755 testRNNCell(buildRNN);
756};
757
758TEST_P(MLTest, trainGRU) {
759 CHECK_IF_ENABLED();
760 testRNNCell(buildGRU);
761};
762
763TEST_P(MLTest, trainLSTM) {
764 CHECK_IF_ENABLED();
765 testRNNCell(buildLSTM);
766};
767
768TEST_P(MLTest, trainSimpleLinearRegression) {
769 CHECK_IF_ENABLED();
770 TrainingConfig TC;
771 PlaceholderBindings bindings;
772
773 // Given 1-D vectors x and y, find real numbers m and b such that
774 // m * x + b is approximately equal to y.
775 unsigned numSamples = 500;
776
777 // This variable records the number of the next sample to be used for
778 // training.
779 size_t sampleCounter = 0;
780
781 TC.learningRate = 0.1;
782 TC.batchSize = numSamples;
783
784 auto &mod = EET_.getModule();
785 Function *F = mod.createFunction(
786 "Gradient descent solution for simple linear regression");
787
788 // These m and b are only used to generate training data.
789 float referenceM = 3.0;
790 float referenceB = 6.0;
791
792 Tensor tensorX(ElemKind::FloatTy, {numSamples, 1});
793 Tensor tensorY(ElemKind::FloatTy, {numSamples, 1});
794 for (unsigned i = 0; i < numSamples; i++) {
795 float x_i = -2.0 + 4.0 * i / numSamples;
796 float y_i = referenceM * x_i + referenceB + mod.getPRNG().nextRand() / 10.0;
797 tensorX.getHandle<>().at({i, 0}) = x_i;
798 tensorY.getHandle<>().at({i, 0}) = y_i;
799 }
800
801 // Create a variable with 1 input, which is a real number.
802 Placeholder *inputX =
803 mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1}, "input", false);
804 Placeholder *expectedY = mod.createPlaceholder(
805 ElemKind::FloatTy, {numSamples, 1}, "expected", false);
806
807 FullyConnectedNode *FC = F->createFullyConnected(bindings, "fc", inputX, 1);
808 Node *R = F->createRegression("reg", FC, expectedY);
809 SaveNode *SN = F->createSave("return", R);
810
811 bindings.allocate(inputX);
812 bindings.allocate(expectedY);
813 bindings.allocate(SN->getPlaceholder());
814
815 Placeholder *M = llvm::cast<Placeholder>(FC->getWeights());
816 Placeholder *B = llvm::cast<Placeholder>(FC->getBias());
817
818 auto *TF = glow::differentiate(F, TC);
819 auto tfName = TF->getName();
820 EET_.compile(CompilationMode::Train);
821
822 // Train the network doing 100 steps. Learn on 500 samples.
823 runBatch(EET_, bindings, 100, sampleCounter, {inputX, expectedY},
824 {&tensorX, &tensorY}, tfName);
825
826 // Testing trained m and b:
827 EXPECT_NEAR(bindings.get(M)->getHandle<>().at({0, 0}), referenceM, 0.01);
828 EXPECT_NEAR(bindings.get(B)->getHandle<>().at({0}), referenceB, 0.01);
829}
830
831enum class Sport : size_t { BASKETBALL = 0, SOCCER = 1 };
832
833void generatePlayerData(Tensor &players, Tensor &labels,
834 unsigned numTrainPlayers, PseudoRNG &PRNG) {
835 auto P = players.getHandle<>();
836 auto L = labels.getHandle<int64_t>();
837
838 // Auto generate height/weights for basketball players.
839 for (dim_t i = 0; i < numTrainPlayers / 2; i++) {
840 auto heightInches = PRNG.nextRandInt(70, 88);
841 auto weightLbs =
842 4 * heightInches + PRNG.nextRandInt(-85, -55); // [195, 297]
843 P.at({i, 0}) = heightInches;
844 P.at({i, 1}) = weightLbs;
845 L.at({i, 0}) = static_cast<size_t>(Sport::BASKETBALL);
846 }
847
848 // Auto generate height/weights for soccer players.
849 for (dim_t i = numTrainPlayers / 2; i < numTrainPlayers; i++) {
850 auto heightInches = PRNG.nextRandInt(60, 76);
851 auto weightLbs = static_cast<unsigned>(2 * heightInches) +
852 PRNG.nextRandInt(20, 50); // [140, 202]
853 P.at({i, 0}) = heightInches;
854 P.at({i, 1}) = weightLbs;
855 L.at({i, 0}) = static_cast<size_t>(Sport::SOCCER);
856 }
857}
858
859// Given a player's height and weight, classify them as a basketball or
860// soccer player.
861TEST_P(MLTest, classifyPlayerSport) {
862 CHECK_IF_ENABLED();
863 const unsigned numTrainPlayers = 200;
864 const dim_t numFeatures = 2;
865 const dim_t numClasses = 2;
866
867 TrainingConfig TC;
868 PlaceholderBindings inferBindings, trainingBindings;
869
870 // This variable records the number of the next sample to be used for
871 // training.
872 size_t sampleCounter = 0;
873
874 TC.learningRate = 0.05;
875 TC.batchSize = numTrainPlayers;
876 Function *F;
877 Placeholder *A, *S;
878 for (auto *EE : engines_) {
879 auto &mod = EE->getModule();
880 F = mod.createFunction("classifyPlayers");
881
882 A = mod.createPlaceholder(ElemKind::FloatTy, {numTrainPlayers, numFeatures},
883 "A", false);
884 S = mod.createPlaceholder(ElemKind::Int64ITy, {numTrainPlayers, 1}, "S",
885 false);
886
887 auto *FC = F->createFullyConnected(inferBindings, "fc", A, numClasses);
888 auto *SM = F->createSoftMax("softmax", FC, S);
889 F->createSave("result", SM);
890 }
891 trainingBindings.allocate(EET_.getModule().getPlaceholders());
892 inferBindings.copyTrainableWeightsTo(trainingBindings);
893 inferBindings.clear();
894 inferBindings.allocate(EEI_.getModule().getPlaceholders());
895
896 auto *TF = glow::differentiate(F, TC);
897 auto tfName = TF->getName();
898 auto fname = F->getName();
899 EET_.compile(CompilationMode::Train);
900
901 Tensor players(ElemKind::FloatTy, {numTrainPlayers, numFeatures});
902 Tensor labels(ElemKind::Int64ITy, {numTrainPlayers, 1});
903 generatePlayerData(players, labels, numTrainPlayers,
904 EET_.getModule().getPRNG());
905
906 // Training:
907 runBatch(EET_, trainingBindings, 2000, sampleCounter, {A, S},
908 {&players, &labels}, tfName);
909 trainingBindings.copyTrainableWeightsTo(inferBindings);
910 EEI_.compile(CompilationMode::Infer);
911 A = inferBindings.getPlaceholderByNameSlow("A");
912 std::vector<std::tuple<unsigned, unsigned, Sport>> testPlayers;
913 testPlayers.emplace_back(82, 240, Sport::BASKETBALL);
914 testPlayers.emplace_back(86, 260, Sport::BASKETBALL);
915 testPlayers.emplace_back(90, 270, Sport::BASKETBALL);
916 testPlayers.emplace_back(60, 160, Sport::SOCCER);
917 testPlayers.emplace_back(63, 155, Sport::SOCCER);
918 testPlayers.emplace_back(66, 170, Sport::SOCCER);
919
920 Tensor testPlayersTensor(ElemKind::FloatTy, {numTrainPlayers, numFeatures});
921 for (dim_t i = 0; i < testPlayers.size(); i++) {
922 testPlayersTensor.getHandle<>().at({i, 0}) = std::get<0>(testPlayers[i]);
923 testPlayersTensor.getHandle<>().at({i, 1}) = std::get<1>(testPlayers[i]);
924 }
925
926 updateInputPlaceholders(inferBindings, {A}, {&testPlayersTensor});
927 EEI_.run(inferBindings, fname);
928
929 auto SMH =
930 inferBindings.get(inferBindings.getPlaceholderByNameSlow("result"))
931 ->getHandle<>();
932 for (dim_t i = 0; i < testPlayers.size(); i++) {
933 const dim_t sport = static_cast<dim_t>(std::get<2>(testPlayers[i]));
934 EXPECT_NEAR(SMH.at({i, sport}), 1.0, 0.1);
935 }
936}
937
938TEST_P(MLTest, learnSinus) {
939 CHECK_IF_ENABLED();
940 TrainingConfig TC;
941 PlaceholderBindings trainingBindings, inferBindings;
942
943 // This variable records the number of the next sample to be used for
944 // training.
945 size_t sampleCounter = 0;
946
947 // Try to learn the sin(x) function.
948 float epsilon = 0.1;
949 unsigned numSamples = 50;
950 Tensor tensorX(ElemKind::FloatTy, {numSamples, 1});
951 Tensor tensorY(ElemKind::FloatTy, {numSamples, 1});
952
953 TC.learningRate = 0.2;
954 TC.batchSize = numSamples;
955
956 // Function that should be learned by the NN
957 auto FF = [](float x) -> float {
958 // Return a shifted sin(x) value, so that it is in the range [0, 1].
959 return (sin(x) + 1) / 2;
960 };
961 Function *F;
962 Placeholder *inputX, *expectedY;
963 for (auto *EE : engines_) {
964 auto &mod = EE->getModule();
965 F = mod.createFunction("Gradient descent solution for sin(x)");
966
967 for (unsigned i = 0; i < numSamples; i++) {
968 // Scale x to cover the range [0, 4] as this leads to a good convergence
969 // during training.
970 float x = i / (numSamples / 4.0);
971 float y = FF(x);
972 tensorX.getHandle<>().at({i, 0}) = x;
973 tensorY.getHandle<>().at({i, 0}) = y;
974 }
975
976 inputX = mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1}, "input",
977 false);
978
979 expectedY = mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1},
980 "expected", false);
981
982 FullyConnectedNode *FC1 =
983 F->createFullyConnected(inferBindings, "fc1", inputX, 10);
984 Node *O = F->createSigmoid("sigmoid1", FC1);
985 FullyConnectedNode *FC2 =
986 F->createFullyConnected(inferBindings, "fc2", O, 1);
987 Node *R = F->createRegression("reg", FC2, expectedY);
988 F->createSave("return", R);
989 }
990 trainingBindings.allocate(EET_.getModule().getPlaceholders());
991 inferBindings.copyTrainableWeightsTo(trainingBindings);
992 inferBindings.clear();
993 inferBindings.allocate(EEI_.getModule().getPlaceholders());
994 auto *res =
995 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("return"));
996
997 auto *TF = glow::differentiate(F, TC);
998 auto tfName = TF->getName();
999 auto fname = F->getName();
1000 EET_.compile(CompilationMode::Train);
1001
1002 // Learn on numSamples samples.
1003 runBatch(EET_, trainingBindings, 2700, sampleCounter, {inputX, expectedY},
1004 {&tensorX, &tensorY}, tfName);
1005 trainingBindings.copyTrainableWeightsTo(inferBindings);
1006 // Create a test set, which is similar, but different from the training set.
1007 for (unsigned i = 0; i < numSamples; i++) {
1008 // Scale x to cover the range [0, 4.2] as this leads to a good convergence
1009 // during training.
1010 float x = i / (numSamples / 4.2) + 0.123456;
1011 float y = FF(x);
1012 tensorX.getHandle<>().at({i, 0}) = x;
1013 tensorY.getHandle<>().at({i, 0}) = y;
1014 }
1015 inputX = inferBindings.getPlaceholderByNameSlow("input");
1016 EEI_.compile(CompilationMode::Infer);
1017 updateInputPlaceholders(inferBindings, {inputX}, {&tensorX});
1018 EEI_.run(inferBindings, fname);
1019 auto resH = res->getHandle<>();
1020
1021 for (dim_t i = 0; i < numSamples; i++) {
1022 float x = tensorX.getHandle().at({i, 0});
1023 EXPECT_NEAR(resH.at({i, 0}), FF(x), epsilon);
1024 }
1025}
1026
1027TEST_P(MLTest, nonLinearClassifier) {
1028 CHECK_IF_ENABLED();
1029 // Test non-linear classification on a set of 2d points. Generate x and y in
1030 // (-1, 1) and classify according to XOR of the sign bit.
1031 unsigned batchSize = 46;
1032 unsigned numSamples = 230;
1033
1034 // This variable records the number of the next sample to be used for
1035 // training.
1036 size_t sampleCounter = 0;
1037
1038 PlaceholderBindings inferBindings, trainingBindings;
1039 TrainingConfig TC;
1040 TC.learningRate = 0.01;
1041 TC.momentum = 0.9;
1042 TC.batchSize = batchSize;
1043 Function *F;
1044 Placeholder *A, *S;
1045 for (auto *EE : engines_) {
1046 auto &mod = EE->getModule();
1047 F = mod.createFunction("nonLinearClassifier");
1048
1049 A = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 2}, "A", false);
1050 S = mod.createPlaceholder(ElemKind::Int64ITy, {batchSize, 1}, "S", false);
1051
1052 auto *FCL0 = F->createFullyConnected(inferBindings, "fc1", A, 8);
1053 auto *T0 = F->createTanh("tanh1", FCL0);
1054 auto *FCL1 = F->createFullyConnected(inferBindings, "fc2", T0, 8);
1055 auto *T1 = F->createTanh("tanh2", FCL1);
1056 auto *FCL2 = F->createFullyConnected(inferBindings, "fc2", T1, 2);
1057 auto *SM = F->createSoftMax("soft", FCL2, S);
1058 F->createSave("ret", SM);
1059 }
1060 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1061 inferBindings.copyTrainableWeightsTo(trainingBindings);
1062 inferBindings.clear();
1063 inferBindings.allocate(EEI_.getModule().getPlaceholders());
1064 auto *res =
1065 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
1066
1067 auto *TF = glow::differentiate(F, TC);
1068 auto tfName = TF->getName();
1069 auto fname = F->getName();
1070 EET_.compile(CompilationMode::Train);
1071 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1072
1073 Tensor samples(ElemKind::FloatTy, {numSamples, 2});
1074 Tensor labels(ElemKind::Int64ITy, {numSamples, 1});
1075
1076 for (dim_t i = 0; i < numSamples; i++) {
1077 float x = EET_.getModule().getPRNG().nextRand();
1078 float y = EET_.getModule().getPRNG().nextRand();
1079 dim_t label = (x < 0.0) ^ (y < 0.0);
1080 samples.getHandle<>().at({i, 0}) = x;
1081 samples.getHandle<>().at({i, 1}) = y;
1082 labels.getHandle<int64_t>().at({i, 0}) = label;
1083 }
1084
1085 runBatch(EET_, trainingBindings, 500, sampleCounter, {A, S},
1086 {&samples, &labels}, tfName);
1087 trainingBindings.copyTrainableWeightsTo(inferBindings);
1088 EEI_.compile(CompilationMode::Infer);
1089 A = inferBindings.getPlaceholderByNameSlow("A");
1090 std::vector<std::tuple<float, float, dim_t>> tests;
1091 tests.emplace_back(-0.8, -0.8, 0);
1092 tests.emplace_back(0.8, -0.8, 1);
1093 tests.emplace_back(-0.8, 0.8, 1);
1094 tests.emplace_back(0.8, 0.8, 0);
1095 auto RH = res->getHandle<>();
1096 for (dim_t i = 0; i < tests.size(); i++) {
1097 Tensor T(ElemKind::FloatTy, {batchSize, 2});
1098 T.getHandle<>().at({0, 0}) = std::get<0>(tests[i]);
1099 T.getHandle<>().at({0, 1}) = std::get<1>(tests[i]);
1100 updateInputPlaceholders(inferBindings, {A}, {&T});
1101 EEI_.run(inferBindings, fname);
1102 EXPECT_NEAR(RH.at({0, std::get<2>(tests[i])}), 1.0, 0.2);
1103 }
1104}
1105
1106/// Generate images in two classes.
1107/// A "line" is labeled as 0 and a "cross" is labeled as 1.
1108static void generateImageData(Tensor &images, Tensor &labels, PseudoRNG &PRNG) {
1109 auto L = labels.getHandle<int64_t>();
1110 auto image = images.getHandle<>();
1111 unsigned numSamples = images.dims()[0];
1112 images.zero();
1113
1114 for (dim_t i = 0; i < numSamples; i++) {
1115 bool isLine = i % 2 == 0;
1116 L.at({i, 0}) = isLine ? 0 : 1;
1117 dim_t target = PRNG.nextRandInt(1, 6);
1118 if (isLine) {
1119 for (dim_t y = 0; y < 8; y++)
1120 image.at({i, target, y, 0u}) = 1;
1121 } else {
1122 for (dim_t pos = 0; pos < 8; pos++) {
1123 image.at({i, pos, target, 0u}) = 1;
1124 image.at({i, target, pos, 0u}) = 1;
1125 }
1126 }
1127 }
1128}
1129
1130/// Test the convolutional layer.
1131/// Use a simple convnet to learn two classes of images: Line and Cross.
1132/// This test checks the results of the quantized network.
1133TEST_P(MLTest, convNetForImageRecognition) {
1134 CHECK_IF_ENABLED();
1135 EET_.setBackendName("Interpreter");
1136 ExecutionEngine EEP{"Interpreter"};
1137 engines_.emplace(engines_.begin(), &EEP);
1138 const unsigned numSamples = 500;
1139 const unsigned batchSize = 7;
1140 PlaceholderBindings inferBindings, trainingBindings, profileBindings;
1141
1142 // This variable records the number of the next sample to be used for
1143 // training.
1144 size_t sampleCounter = 0;
1145
1146 TrainingConfig TC;
1147 TC.learningRate = 0.01;
1148 TC.batchSize = batchSize;
1149 TC.momentum = 0.9;
1150 std::string fName;
1151 for (auto *EE : engines_) {
1152 auto &mod = EE->getModule();
1153 Function *F = mod.createFunction("convNetForImageRecognition");
1154
1155 Placeholder *input = mod.createPlaceholder(
1156 ElemKind::FloatTy, {batchSize, 8, 8, 1}, "input", false);
1157
1158 Placeholder *ex =
1159 mod.createPlaceholder(ElemKind::Int64ITy, {batchSize, 1}, "exp", false);
1160
1161 auto *CV = F->createConv(inferBindings, "conv", input, 1, 3, 1, 0, 1);
1162 auto *TANH = F->createTanh("tanh", CV);
1163 auto *FCL = F->createFullyConnected(inferBindings, "fc", TANH, 2);
1164 auto *SM = F->createSoftMax("sm", FCL, ex);
1165 F->createSave("ret", SM);
1166 fName = F->getName().str();
1167 }
1168
1169 auto *mod = &EET_.getModule();
1170 auto input = mod->getPlaceholderByNameSlow("input");
1171 auto ex = mod->getPlaceholderByNameSlow("exp");
1172
1173 auto *TF = glow::differentiate(mod->getFunction(fName), TC);
1174 auto tfName = TF->getName();
1175 EET_.compile(CompilationMode::Train);
1176 trainingBindings.allocate(mod->getPlaceholders());
1177 inferBindings.copyTrainableWeightsTo(trainingBindings);
1178
1179 Tensor images(ElemKind::FloatTy, {numSamples, 8, 8, 1});
1180 Tensor labels(ElemKind::Int64ITy, {numSamples, 1});
1181 generateImageData(images, labels, mod->getPRNG());
1182
1183 // Training:
1184 runBatch(EET_, trainingBindings, 500, sampleCounter, {input, ex},
1185 {&images, &labels}, tfName);
1186
1187 mod = &EEP.getModule();
1188 profileBindings.allocate(mod->getPlaceholders());
1189 LoweredInfoMap loweredMapForProf;
1190 CompilationContext cctxProf{&profileBindings, &loweredMapForProf};
1191 cctxProf.precisionConfig.quantMode = QuantizationMode::Profile;
1192
1193 auto F = mod->getFunction(fName);
1194 input = mod->getPlaceholderByNameSlow("input");
1195 trainingBindings.copyTrainableWeightsTo(profileBindings);
1196 EEP.compile(cctxProf);
1197 // Since we are compiling in profiling mode the partitioner will create a new
1198 // function from the original. Get the new function.
1199 F = mod->getFunctions().front();
1200
1201 runBatch(EEP, profileBindings, 100, sampleCounter, {input}, {&images}, fName);
1202
1203 // Evaluate on the quantized function:
1204 // Set the execution backend to the backend that we test.
1205 mod = &EEI_.getModule();
1206 inferBindings.clear();
1207 inferBindings.allocate(mod->getPlaceholders());
1208 trainingBindings.copyTrainableWeightsTo(inferBindings);
1209
1210 LoweredInfoMap loweredMapForQuant;
1211 CompilationContext cctxQuant{&inferBindings, &loweredMapForQuant};
1212 PrecisionConfiguration &precConfig = cctxQuant.precisionConfig;
1213 cctxQuant.precisionConfig.quantMode = QuantizationMode::Quantize;
1214 precConfig.quantConfig.infos = quantization::generateNodeProfilingInfos(
1215 profileBindings, F, loweredMapForProf);
1216 precConfig.quantConfig.assertAllNodesQuantized = true;
1217
1218 // Softmax is not supported in Int8QTy, so signal the quantizer it's OK to
1219 // keep it unquantized.
1220 precConfig.precisionModeKindSet.insert(Kinded::Kind::SoftMaxNodeKind);
1221
1222 F = mod->getFunction("convNetForImageRecognition");
1223 EEI_.compile(cctxQuant);
1224 input = mod->getPlaceholderByNameSlow("input");
1225
1226 // Generate the images used for testing.
1227 Tensor testImages(ElemKind::FloatTy, {batchSize, 8, 8, 1});
1228 Tensor testLabels(ElemKind::Int64ITy, {batchSize, 1});
1229 generateImageData(testImages, testLabels, mod->getPRNG());
1230 updateInputPlaceholders(inferBindings, {input}, {&testImages});
1231
1232 EEI_.run(inferBindings);
1233
1234 Tensor *res =
1235 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
1236 auto SMH = res->getHandle<>();
1237 for (dim_t i = 0; i < batchSize; i++) {
1238 bool isLine = testLabels.getHandle<int64_t>().at({i, 0}) == 0;
1239 auto lineWeight = SMH.at({i, 0});
1240 auto crossWeight = SMH.at({i, 1});
1241 EXPECT_TRUE((isLine && lineWeight > crossWeight) ||
1242 (!isLine && crossWeight > lineWeight));
1243 }
1244}
1245
1246/// Generate data for the regression test. Put a '1' in a random location in a
1247/// clear tensor and report the coordinates of that pixel.
1248static void generateRegressionTestData(Tensor &images, Tensor &labels,
1249 PseudoRNG &PRNG) {
1250 auto L = labels.getHandle<>();
1251 auto image = images.getHandle<>();
1252 unsigned numSamples = images.dims()[0];
1253 image.clear(0);
1254
1255 for (dim_t i = 0; i < numSamples; i++) {
1256 // Generate the X,Y coordinates to place our object.
1257 dim_t x = PRNG.nextRandInt(0, 9);
1258 dim_t y = PRNG.nextRandInt(0, 9);
1259 L.at({i, 0}) = x;
1260 L.at({i, 1}) = y;
1261 image.at({i, x, y, 0u}) = 1;
1262 }
1263}
1264
1265/// This is the "Where's Waldo" test. We place a pixel in a tensor and the
1266/// network reports the coordinate of the pixel.
1267TEST_P(MLTest, testFindPixelRegression) {
1268 CHECK_IF_ENABLED();
1269 EET_.setBackendName("Interpreter");
1270 ExecutionEngine EEP{"Interpreter"};
1271 engines_.emplace(engines_.begin(), &EEP);
1272 PlaceholderBindings inferBindings, trainingBindings, profileBindings;
1273
1274 const unsigned numSamples = 1000;
1275 const unsigned batchSize = 10;
1276
1277 // This variable records the number of the next sample to be used for
1278 // training.
1279 size_t sampleCounter = 0;
1280
1281 TrainingConfig TC;
1282 TC.learningRate = 0.01;
1283 TC.batchSize = batchSize;
1284 TC.momentum = 0.9;
1285 std::string fName;
1286 for (auto *EE : engines_) {
1287 auto &mod = EE->getModule();
1288 Function *F = mod.createFunction("main");
1289
1290 Placeholder *input = mod.createPlaceholder(
1291 ElemKind::FloatTy, {batchSize, 10, 10, 1}, "input", false);
1292 Placeholder *ex = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 2},
1293 "coordinates", false);
1294
1295 // A simple single-layer FC network could solve this program but we use a
1296 // two layer FC network to give the compiler something slightly more complex
1297 // to work with so we are adding another FC layer.
1298 auto *FC0 = F->createFullyConnected(inferBindings, "fc0", input, 6);
1299 auto *RL0 = F->createRELU("relu0", FC0);
1300 auto *FC1 = F->createFullyConnected(inferBindings, "fc1", RL0, 2);
1301 auto *R = F->createRegression("regression", FC1, ex);
1302 F->createSave("ret", R);
1303 fName = F->getName().str();
1304 }
1305
1306 auto *mod = &EET_.getModule();
1307 auto input = mod->getPlaceholderByNameSlow("input");
1308 auto ex = mod->getPlaceholderByNameSlow("coordinates");
1309
1310 auto *TF = glow::differentiate(mod->getFunction(fName), TC);
1311 auto tfName = TF->getName();
1312 EET_.compile(CompilationMode::Train);
1313 // Specify these to initialze to zero to prevent uninitialized memory issues.
1314 for (auto *PH : EET_.getModule().getPlaceholders()) {
1315 PH->setAllocZero();
1316 }
1317 trainingBindings.allocate(mod->getPlaceholders());
1318
1319 for (auto &PH : inferBindings.pairs()) {
1320 inferBindings.copyToTarget(PH.first->getName(), trainingBindings);
1321 }
1322 inferBindings.clear();
1323
1324 // -- STEP1 - train the network. --
1325 Tensor images(ElemKind::FloatTy, {numSamples, 10, 10, 1});
1326 Tensor labels(ElemKind::FloatTy, {numSamples, 2});
1327 generateRegressionTestData(images, labels, mod->getPRNG());
1328
1329 // Training:
1330 runBatch(EET_, trainingBindings, 400, sampleCounter, {input, ex},
1331 {&images, &labels}, tfName);
1332
1333 // -- STEP2 - Profile and quantize the network. --
1334 mod = &EEP.getModule();
1335 for (auto *PH : mod->getPlaceholders()) {
1336 PH->setAllocZero();
1337 }
1338 profileBindings.allocate(mod->getPlaceholders());
1339 Tensor profileImages(ElemKind::FloatTy, {batchSize, 10, 10, 1});
1340 Tensor profileLabels(ElemKind::FloatTy, {batchSize, 2});
1341 generateRegressionTestData(profileImages, profileLabels, mod->getPRNG());
1342 input = mod->getPlaceholderByNameSlow("input");
1343 updateInputPlaceholders(profileBindings, {input}, {&profileImages});
1344 LoweredInfoMap loweredMapForProf;
1345 CompilationContext cctxProf{&profileBindings, &loweredMapForProf};
1346 cctxProf.precisionConfig.quantMode = QuantizationMode::Profile;
1347
1348 trainingBindings.copyTrainableWeightsTo(profileBindings);
1349 EEP.compile(cctxProf);
1350 // Get new function after partitioning.
1351 auto F = EEP.getModule().getFunctions().front();
1352
1353 // Run the graph to capture the profile.
1354 runBatch(EEP, profileBindings, 100, sampleCounter, {input}, {&images}, fName);
1355
1356 // -- STEP3 - evaluate the quantized function. --
1357 mod = &EEI_.getModule();
1358 inferBindings.allocate(mod->getPlaceholders());
1359 trainingBindings.copyTrainableWeightsTo(inferBindings);
1360
1361 LoweredInfoMap loweredMapForQuant;
1362 CompilationContext cctxQuant{&inferBindings, &loweredMapForQuant};
1363 cctxQuant.precisionConfig.quantMode = QuantizationMode::Quantize;
1364 cctxQuant.loweredInfoMap = &loweredMapForQuant;
1365 cctxQuant.precisionConfig.quantConfig.infos =
1366 quantization::generateNodeProfilingInfos(profileBindings, F,
1367 loweredMapForProf);
1368 cctxQuant.precisionConfig.quantConfig.assertAllNodesQuantized = true;
1369
1370 F = mod->getFunction(fName);
1371 EEI_.compile(cctxQuant);
1372 input = mod->getPlaceholderByNameSlow("input");
1373
1374 // Generate the images used for testing.
1375 Tensor testImages(ElemKind::FloatTy, {batchSize, 10, 10, 1});
1376 Tensor testLabels(ElemKind::FloatTy, {batchSize, 2});
1377 generateRegressionTestData(testImages, testLabels, mod->getPRNG());
1378
1379 // Run the inference:
1380 updateInputPlaceholders(inferBindings, {input}, {&testImages});
1381 EEI_.run(inferBindings);
1382
1383 // A handle to the projected result.
1384 Tensor *res =
1385 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret"));
1386 auto RH = res->getHandle<>();
1387 // A handle to the true label.
1388 auto LH = testLabels.getHandle<>();
1389
1390 // Check how many of the coordinates that were reported are close to the real
1391 // values.
1392 unsigned correct = 0;
1393
1394 for (dim_t i = 0; i < batchSize; i++) {
1395 // Calculate the distance between the predicted value and correct value.
1396 auto dx = LH.at({i, 0}) - RH.at({i, 0});
1397 auto dy = LH.at({i, 1}) - RH.at({i, 1});
1398 auto distance = std::sqrt(std::pow(dx, 2) + std::pow(dy, 2));
1399
1400 // A correct answer is one where the projected distance is somewhat close.
1401 correct += distance < 3;
1402 }
1403
1404 // Expect 90% of the results to be correct.
1405 EXPECT_GE(correct, batchSize * 0.90);
1406}
1407
1408// Generate tests for a toy neural network that can recognize if a matrix 3x3
1409// is a rotation of another matrix 3x3.
1410// This is *not* about rotation matrices used for computer graphics, but a much
1411// simpler concept.
1412// Informally, let M1 and M2 be two 3x3 matrices, M2 is a rotation of M1 if it
1413// exists a way to rotate the cells of M1 to get M2. The rotations are all
1414// centered in the middle of the matrices.
1415// E.g.,
1416// Rotate clockwise 1 cell centered in 'e':
1417// --+
1418// | a b c | |
1419// M1 = | d e f | V
1420// | g h i |
1421// =>
1422// | d a b |
1423// M2 = | g e c |
1424// | h i f |
1425static void generateMatrixRotationRecognitionData(Tensor &matricesA,
1426 Tensor &matricesB,
1427 Tensor &expected,
1428 PseudoRNG &PRNG) {
1429
1430 using CellIdx = std::pair<uint8_t, uint8_t>;
1431 // List the indices in a clockwise ordering starting from the top left
1432 // corner.
1433 // Note: This does not include the cell in the middle given it is
1434 // never rotated.
1435 static const CellIdx clockwiseOrder[] = {{0, 0}, {0, 1}, {0, 2}, {1, 2},
1436 {2, 2}, {2, 1}, {2, 0}, {1, 0}};
1437 static const uint8_t possibleTargetCells =
1438 sizeof(clockwiseOrder) / sizeof(clockwiseOrder[0]);
1439 const unsigned numSamples = matricesA.dims()[0];
1440 assert(expected.dims()[0] == numSamples &&
1441 matricesB.dims()[0] == numSamples &&
1442 "Size of the tensors is incompatible");
1443 auto handleMatricesA = matricesA.getHandle<float>();
1444 auto handleMatricesB = matricesB.getHandle<float>();
1445 auto handleExpected = expected.getHandle<int64_t>();
1446
1447 handleMatricesA.randomize<int>(0, 1, PRNG);
1448 handleMatricesB.randomize<int>(0, 1, PRNG);
1449 for (unsigned idx = 0; idx < numSamples; ++idx) {
1450 // Toss a coin and create a rotation relationship or not.
1451 bool shouldHaveRotation = PRNG.nextRandInt(0, 1);
1452 handleExpected.at({idx, 0}) = shouldHaveRotation;
1453 if (shouldHaveRotation) {
1454 // On a 3x3 matrix we have 8 different possbile clockwise steps.
1455 // Pick one.
1456 size_t clockwiseSteps = PRNG.nextRandInt(0, possibleTargetCells - 1);
1457 // Generate the rotation matrix from A.
1458 // The center never changes.
1459 handleMatricesB.at({idx, 1, 1}) = handleMatricesA.at({idx, 1, 1});
1460 // Fetch the cell registered in the clockwiseOrder at the desired step.
1461 for (size_t i = 0; i != possibleTargetCells; ++i) {
1462 const CellIdx &sourceCellIdx = clockwiseOrder[i];
1463 const CellIdx &targetCellIdx =
1464 clockwiseOrder[(i + clockwiseSteps) % possibleTargetCells];
1465 handleMatricesB.at({idx, targetCellIdx.first, targetCellIdx.second}) =
1466 handleMatricesA.at(
1467 {idx, sourceCellIdx.first, sourceCellIdx.second});
1468 }
1469 }
1470 // Else:
1471 // There is a high probability that A and B don't have a rotation
1472 // relationship and thus, there is nothing to do.
1473 // Worse case, we mislabeled a relationship.
1474
1475 // Alternatively we could always alter one of the matrix such that it is
1476 // impossible to have a rotation between them (e.g., make sure the center
1477 // is different), but that would bias the kind of differences that could
1478 // occur.
1479 }
1480}
1481
1482TEST_P(MLTest, matrixRotationRecognition) {
1483 CHECK_IF_ENABLED();
1484 TrainingConfig TC;
1485 TC.learningRate = 0.15;
1486 TC.batchSize = 17;
1487 PlaceholderBindings inferBindings, trainingBindings;
1488
1489 // This variable records the number of the next sample to be used for
1490 // training.
1491 size_t sampleCounter = 0;
1492 Function *F;
1493 Placeholder *varMatricesA, *varMatricesB, *varExpected;
1494 for (auto *EE : engines_) {
1495 Module &mod = EE->getModule();
1496 F = mod.createFunction("MatrixRotationRecognition");
1497 varMatricesA = mod.createPlaceholder(
1498 ElemKind::FloatTy, {TC.batchSize, 3, 3}, "matrixA", false);
1499 varMatricesB = mod.createPlaceholder(
1500 ElemKind::FloatTy, {TC.batchSize, 3, 3}, "matrixB", false);
1501 varExpected = mod.createPlaceholder(ElemKind::Int64ITy, {TC.batchSize, 1},
1502 "expected", false);
1503
1504 // Simply concatenating the matrices first would probability be as effective
1505 // but we want to build something more complex than a straight chain of
1506 // operators to stress the scheduler.
1507 auto *FCA = F->createFullyConnected(inferBindings, "hidden_matrixA_fc",
1508 varMatricesA, 10);
1509 auto *FCB = F->createFullyConnected(inferBindings, "hidden_matrixB_fc",
1510 varMatricesB, 10);
1511 auto *ReLUA = F->createRELU("hidden_matrixA_ReLU", FCA);
1512 auto *ReLUB = F->createRELU("hidden_matrixB_ReLU", FCB);
1513 auto *concat = F->createConcat("hidden_concat_A_B", {ReLUA, ReLUB}, 1);
1514 auto *hiddenFC =
1515 F->createFullyConnected(inferBindings, "hidden_fc", concat, 30);
1516 auto *finalReLU = F->createRELU("hidden_concat_ReLU", hiddenFC);
1517 auto *finalFC =
1518 F->createFullyConnected(inferBindings, "output_fc", finalReLU, 2);
1519 auto *softMax = F->createSoftMax("output", finalFC, varExpected);
1520 F->createSave("result", softMax);
1521 }
1522 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1523 inferBindings.copyTrainableWeightsTo(trainingBindings);
1524 inferBindings.clear();
1525 inferBindings.allocate(EEI_.getModule().getPlaceholders());
1526 auto *res =
1527 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("result"));
1528 auto *TF = glow::differentiate(F, TC);
1529 auto tfName = TF->getName();
1530 auto fname = F->getName();
1531
1532 // Train the network.
1533 const unsigned numSamples = 50;
1534 Tensor matricesA(ElemKind::FloatTy, {numSamples, 3, 3});
1535 Tensor matricesB(ElemKind::FloatTy, {numSamples, 3, 3});
1536 Tensor expected(ElemKind::Int64ITy, {numSamples, 1});
1537 generateMatrixRotationRecognitionData(matricesA, matricesB, expected,
1538 EET_.getModule().getPRNG());
1539
1540 EET_.compile(CompilationMode::Train);
1541 // Training:
1542 runBatch(EET_, trainingBindings, 200, sampleCounter,
1543 {varMatricesA, varMatricesB, varExpected},
1544 {&matricesA, &matricesB, &expected}, tfName);
1545 trainingBindings.copyTrainableWeightsTo(inferBindings);
1546
1547 // Switch to inference mode.
1548 EEI_.compile(CompilationMode::Infer);
1549
1550 // At this point we should have overfitted the data.
1551 // Take a random batch and check that the values match what we expect.
1552 auto RHtrain = res->getHandle<>();
1553 auto batchSize = TC.batchSize;
1554 unsigned numBatches = numSamples / batchSize;
1555 unsigned batchStartIdx =
1556 EEI_.getModule().getPRNG().nextRandInt(0, numBatches - 1) * batchSize;
1557 varMatricesA = inferBindings.getPlaceholderByNameSlow("matrixA");
1558 varMatricesB = inferBindings.getPlaceholderByNameSlow("matrixB");
1559 auto batchMatricesA =
1560 matricesA.getUnowned({batchSize, 3, 3}, {batchStartIdx, 0, 0});
1561 auto batchMatricesB =
1562 matricesB.getUnowned({batchSize, 3, 3}, {batchStartIdx, 0, 0});
1563 updateInputPlaceholders(inferBindings, {varMatricesA, varMatricesB},
1564 {&batchMatricesA, &batchMatricesB});
1565 EEI_.run(inferBindings, fname);
1566
1567 unsigned errors = 0;
1568 // Check each output in the batch.
1569 for (dim_t i = 0; i != batchSize; i++) {
1570 // Note that the two softmax outputs always sum to 1, so we only look at
1571 // one. Index one is true if there is a rotation.
1572 float value = RHtrain.at({i, 1});
1573 bool hasRotation = expected.getHandle<int64_t>().at({batchStartIdx + i, 0});
1574 if ((value > 0.5) != hasRotation) {
1575 ++errors;
1576 }
1577 }
1578
1579 EXPECT_LE(errors, 1);
1580}
1581
1582/// Simple test case that learns the embedding table for a
1583/// SparseLengthsSum operator.
1584TEST_P(MLTest, learnSparseLengthsSumEmbeddings) {
1585 CHECK_IF_ENABLED();
1586 TrainingConfig TC;
1587 TC.learningRate = 0.3;
1588 TC.batchSize = 1;
1589
1590 PlaceholderBindings trainingBindings, inferBindings;
1591 Function *F;
1592 Placeholder *dataP, *indicesP, *lengthsP, *expectedP;
1593 PseudoRNG &PRNG = EET_.getModule().getPRNG();
1594 for (auto *EE : engines_) {
1595 Module &mod = EE->getModule();
1596
1597 // Create a model consisting of one SparseLengthsSum operator
1598 // followed by a Regression node to get some non-zero gradients.
1599 F = mod.createFunction("SparseLengthsSum");
1600 dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP",
1601 /*isTrainable=*/true);
1602 indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP",
1603 /*isTrainable=*/false);
1604 lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP",
1605 /*isTrainable=*/false);
1606 expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP",
1607 /*isTrainable=*/false);
1608
1609 auto *SLWS = F->createSparseLengthsSum("SLWS", dataP, indicesP, lengthsP);
1610 auto *reg = F->createRegression("reg", SLWS, expectedP);
1611 F->createSave("save", reg);
1612 }
1613 // Allocate and randomly initialize embeddings.
1614 auto DH = inferBindings.allocate(dataP)->getHandle();
1615 DH.randomize(-5.0, 5.0, PRNG);
1616
1617 // Allocate and set indices such that input embeddings are reversed.
1618 inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5,
1619 4, 3, 2, 1, 0};
1620
1621 // Allocate and set lengths.
1622 inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2};
1623
1624 // Allocate and set expected outputs. The embedding table will be adjusted
1625 // during training so that the final result is this.
1626 auto EH = inferBindings.allocate(expectedP)->getHandle();
1627 EH = {1, 2, 3, 4, 5};
1628
1629 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1630 inferBindings.copyTrainableWeightsTo(trainingBindings);
1631 inferBindings.copyToTarget("dataP", trainingBindings);
1632 inferBindings.copyToTarget("indicesP", trainingBindings);
1633 inferBindings.copyToTarget("lengthsP", trainingBindings);
1634 inferBindings.copyToTarget("expectedP", trainingBindings);
1635 inferBindings.clear();
1636 inferBindings.allocate(EEI_.getModule().getPlaceholders());
1637 EH = trainingBindings
1638 .get(trainingBindings.getPlaceholderByNameSlow("expectedP"))
1639 ->getHandle();
1640 auto *res =
1641 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save"));
1642
1643 // Train the network.
1644 auto *TF = glow::differentiate(F, TC);
1645 auto tfName = TF->getName();
1646 auto fname = F->getName();
1647 EET_.compile(CompilationMode::Train);
1648
1649 const size_t numIterations = 1000;
1650
1651 for (size_t i = 0; i < numIterations; ++i) {
1652 EET_.run(trainingBindings, tfName);
1653 }
1654 trainingBindings.copyTrainableWeightsTo(inferBindings);
1655 trainingBindings.copyToTarget("dataP", inferBindings);
1656 trainingBindings.copyToTarget("indicesP", inferBindings);
1657 trainingBindings.copyToTarget("lengthsP", inferBindings);
1658 trainingBindings.copyToTarget("expectedP", inferBindings);
1659 // Switch to inference mode and run the network.
1660 EEI_.compile(CompilationMode::Infer);
1661 EEI_.run(inferBindings, fname);
1662
1663 // Make sure that the network output matches expectations after training.
1664 auto RH = res->getHandle();
1665 for (size_t j = 0; j < EH.size(); ++j) {
1666 EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02);
1667 }
1668}
1669
1670/// Simple test case that learns the embedding table for a
1671/// SparseLengthsWeightedSum operator.
1672TEST_P(MLTest, learnSparseLengthsWeightedSumEmbeddings) {
1673 CHECK_IF_ENABLED();
1674 TrainingConfig TC;
1675 TC.learningRate = 0.3;
1676 TC.batchSize = 1;
1677
1678 PlaceholderBindings trainingBindings, inferBindings;
1679 Function *F;
1680 Placeholder *dataP, *indicesP, *lengthsP, *expectedP, *weightsP;
1681 PseudoRNG &PRNG = EET_.getModule().getPRNG();
1682 for (auto *EE : engines_) {
1683 Module &mod = EE->getModule();
1684
1685 // Create a model consisting of one SparseLengthsWeightedSum operator
1686 // followed by a Regression node to get some non-zero gradients.
1687 F = mod.createFunction("SparseLengthsWeightedSum");
1688 dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP",
1689 /*isTrainable=*/true);
1690 indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP",
1691 /*isTrainable=*/false);
1692 weightsP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "weightsP",
1693 /*isTrainable=*/false);
1694 lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP",
1695 /*isTrainable=*/false);
1696 expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP",
1697 /*isTrainable=*/false);
1698
1699 auto *SLWS = F->createSparseLengthsWeightedSum("SLWS", dataP, weightsP,
1700 indicesP, lengthsP);
1701 auto *reg = F->createRegression("reg", SLWS, expectedP);
1702 F->createSave("save", reg);
1703 }
1704 // Allocate and randomly initialize embeddings.
1705 auto DH = inferBindings.allocate(dataP)->getHandle();
1706 DH.randomize(-5.0, 5.0, PRNG);
1707
1708 // Allocate and set indices such that input embeddings are reversed.
1709 inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5,
1710 4, 3, 2, 1, 0};
1711 // Allocate and set weights.
1712 inferBindings.allocate(weightsP)->getHandle() = {
1713 0.75, 0.25, 0.75, 0.25, 0.75, 0.25, 0.75, 0.25, 0.75, 0.25};
1714
1715 // Allocate and set lengths.
1716 inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2};
1717
1718 // Allocate and set expected outputs. The embedding table will be adjusted
1719 // during training so that the final result is this.
1720 auto EH = inferBindings.allocate(expectedP)->getHandle();
1721 EH = {1, 2, 3, 4, 5};
1722
1723 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1724 inferBindings.copyTrainableWeightsTo(trainingBindings);
1725 inferBindings.copyToTarget("dataP", trainingBindings);
1726 inferBindings.copyToTarget("indicesP", trainingBindings);
1727 inferBindings.copyToTarget("weightsP", trainingBindings);
1728 inferBindings.copyToTarget("lengthsP", trainingBindings);
1729 inferBindings.copyToTarget("expectedP", trainingBindings);
1730 inferBindings.clear();
1731 inferBindings.allocate(EEI_.getModule().getPlaceholders());
1732 EH = trainingBindings
1733 .get(trainingBindings.getPlaceholderByNameSlow("expectedP"))
1734 ->getHandle();
1735 auto *res =
1736 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save"));
1737
1738 // Train the network.
1739 auto *TF = glow::differentiate(F, TC);
1740 auto tfName = TF->getName();
1741 auto fname = F->getName();
1742 EET_.compile(CompilationMode::Train);
1743
1744 const size_t numIterations = 1000;
1745
1746 for (size_t i = 0; i < numIterations; ++i) {
1747 EET_.run(trainingBindings, tfName);
1748 }
1749 trainingBindings.copyTrainableWeightsTo(inferBindings);
1750 trainingBindings.copyToTarget("dataP", inferBindings);
1751 trainingBindings.copyToTarget("indicesP", inferBindings);
1752 trainingBindings.copyToTarget("weightsP", inferBindings);
1753 trainingBindings.copyToTarget("lengthsP", inferBindings);
1754 trainingBindings.copyToTarget("expectedP", inferBindings);
1755 // Switch to inference mode and run the network.
1756 EEI_.compile(CompilationMode::Infer);
1757 EEI_.run(inferBindings, fname);
1758
1759 // Make sure that the network output matches expectations after training.
1760 auto RH = res->getHandle();
1761 for (size_t j = 0; j < EH.size(); ++j) {
1762 EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02);
1763 }
1764}
1765
1766/// Simple test case that learns the weights for a
1767/// SparseLengthsWeightedSum operator.
1768TEST_P(MLTest, learnSparseLengthsWeightedSumWeights) {
1769 CHECK_IF_ENABLED();
1770 TrainingConfig TC;
1771 TC.learningRate = 0.001;
1772 TC.batchSize = 1;
1773
1774 PlaceholderBindings trainingBindings, inferBindings;
1775 Function *F;
1776 Placeholder *dataP, *indicesP, *lengthsP, *expectedP, *weightsP;
1777 PseudoRNG &PRNG = EET_.getModule().getPRNG();
1778 for (auto *EE : engines_) {
1779 Module &mod = EE->getModule();
1780
1781 // Create a model consisting of one SparseLengthsWeightedSum operator
1782 // followed by a Regression node to get some non-zero gradients.
1783 F = mod.createFunction("SparseLengthsWeightedSum");
1784 dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP",
1785 /*isTrainable=*/false);
1786 indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP",
1787 /*isTrainable=*/false);
1788 weightsP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "weightsP",
1789 /*isTrainable=*/true);
1790 lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP",
1791 /*isTrainable=*/false);
1792 expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP",
1793 /*isTrainable=*/false);
1794
1795 auto *SLWS = F->createSparseLengthsWeightedSum("SLWS", dataP, weightsP,
1796 indicesP, lengthsP);
1797 auto *reg = F->createRegression("reg", SLWS, expectedP);
1798 F->createSave("save", reg);
1799 }
1800 // Allocate and set embeddings.
1801 inferBindings.allocate(dataP)->getHandle() = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
1802
1803 // Allocate and set indices such that input embeddings are reversed.
1804 inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5,
1805 4, 3, 2, 1, 0};
1806 // Allocate and randomly initialize weights.
1807 auto WH = inferBindings.allocate(weightsP)->getHandle();
1808 WH.randomize(-1.0, 1.0, PRNG);
1809
1810 // Allocate and set lengths.
1811 inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2};
1812
1813 // Allocate and set expected outputs. The weighs will be adjusted
1814 // during training so that the final result is this.
1815 auto EH = inferBindings.allocate(expectedP)->getHandle();
1816 EH = {10, 7, 6, 3, 2};
1817
1818 trainingBindings.allocate(EET_.getModule().getPlaceholders());
1819 inferBindings.copyTrainableWeightsTo(trainingBindings);
1820 inferBindings.copyToTarget("dataP", trainingBindings);
1821 inferBindings.copyToTarget("indicesP", trainingBindings);
1822 inferBindings.copyToTarget("weightsP", trainingBindings);
1823 inferBindings.copyToTarget("lengthsP", trainingBindings);
1824 inferBindings.copyToTarget("expectedP", trainingBindings);
1825 inferBindings.clear();
1826 inferBindings.allocate(EEI_.getModule().getPlaceholders());
1827 auto *res =
1828 inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save"));
1829 EH = trainingBindings
1830 .get(trainingBindings.getPlaceholderByNameSlow("expectedP"))
1831 ->getHandle();
1832 // Train the network.
1833 auto *TF = glow::differentiate(F, TC);
1834 auto tfName = TF->getName();
1835 auto fname = F->getName();
1836 EET_.compile(CompilationMode::Train);
1837
1838 const size_t numIterations = 1000;
1839
1840 for (size_t i = 0; i < numIterations; ++i) {
1841 EET_.run(trainingBindings, tfName);
1842 }
1843 trainingBindings.copyTrainableWeightsTo(inferBindings);
1844 trainingBindings.copyToTarget("dataP", inferBindings);
1845 trainingBindings.copyToTarget("indicesP", inferBindings);
1846 trainingBindings.copyToTarget("weightsP", inferBindings);
1847 trainingBindings.copyToTarget("lengthsP", inferBindings);
1848 trainingBindings.copyToTarget("expectedP", inferBindings);
1849 // Switch to inference mode and run the network.
1850 EEI_.compile(CompilationMode::Infer);
1851 EEI_.run(inferBindings, fname);
1852
1853 // Make sure that the network output matches expectations after training.
1854 auto RH = res->getHandle();
1855 for (size_t j = 0; j < EH.size(); ++j) {
1856 EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02);
1857 }
1858}
1859
1860INSTANTIATE_BACKEND_TEST(MLTest);
1861