1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #include "BackendTestUtils.h" |
17 | |
18 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
19 | #include "glow/Graph/Graph.h" |
20 | #include "glow/Quantization/Quantization.h" |
21 | |
22 | #include "gtest/gtest.h" |
23 | |
24 | #include "llvm/Support/Casting.h" |
25 | #include "llvm/Support/raw_ostream.h" |
26 | |
27 | #include <string> |
28 | |
29 | using namespace glow; |
30 | using llvm::isa; |
31 | |
32 | class TestRunnerBase : public ::testing::TestWithParam<std::string> { |
33 | public: |
34 | ExecutionEngine EEI_{GetParam()}; |
35 | ExecutionEngine EET_{GetParam()}; |
36 | std::vector<ExecutionEngine *> engines_; |
37 | void SetUp() override { |
38 | // The order here is intentional, the tests assume that EET is the last in |
39 | // the list. |
40 | engines_.push_back(&EEI_); |
41 | engines_.push_back(&EET_); |
42 | } |
43 | }; |
44 | |
45 | class MLTest : public TestRunnerBase {}; |
46 | |
47 | /// Use placeholders (and not variables) to learn the square root of two. |
48 | TEST_P(MLTest, learnSqrt2Placeholder) { |
49 | CHECK_IF_ENABLED(); |
50 | TrainingConfig TC; |
51 | PlaceholderBindings bindings; |
52 | |
53 | TC.learningRate = 0.03; |
54 | |
55 | auto &mod = EET_.getModule(); |
56 | Function *F = mod.createFunction("Square root of 2" ); |
57 | |
58 | auto *A = mod.createPlaceholder(ElemKind::FloatTy, {1}, "A" , true); |
59 | auto *inputTensor = bindings.allocate(A); |
60 | inputTensor->init(Tensor::InitKind::Broadcast, 1, mod.getPRNG()); |
61 | |
62 | auto *E = mod.createPlaceholder(ElemKind::FloatTy, {1}, "Ex" , false); |
63 | bindings.allocate(E)->getHandle() = {2}; |
64 | |
65 | auto *O = mod.createPlaceholder(ElemKind::FloatTy, {1}, "output" , false); |
66 | bindings.allocate(O); |
67 | |
68 | Node *M = F->createMul("Mult" , A, A); |
69 | M = F->createRegression("reg" , M, E); |
70 | SaveNode *SN = F->createSave("ret" , M); |
71 | |
72 | bindings.allocate(SN->getPlaceholder()); |
73 | |
74 | auto *TF = glow::differentiate(F, TC); |
75 | auto fName = TF->getName(); |
76 | EET_.compile(CompilationMode::Train); |
77 | |
78 | // Train the network: |
79 | for (int i = 0; i < 100; i++) { |
80 | EET_.run(bindings, fName); |
81 | } |
82 | |
83 | float res = inputTensor->getHandle().at({0}); |
84 | EXPECT_NEAR(res, 1.4142, 0.01); |
85 | } |
86 | |
87 | TEST_P(MLTest, trainASimpleNetwork) { |
88 | CHECK_IF_ENABLED(); |
89 | TrainingConfig TC; |
90 | PlaceholderBindings bindings; |
91 | |
92 | // This variable records the number of the next sample to be used for |
93 | // training. |
94 | size_t sampleCounter = 0; |
95 | |
96 | // Learning a single input vector. |
97 | TC.learningRate = 0.05; |
98 | Function *F; |
99 | Placeholder *A, *E; |
100 | for (auto *EE : engines_) { |
101 | auto &mod = EE->getModule(); |
102 | F = mod.createFunction("trainASimpleNetwork" ); |
103 | |
104 | // Create a variable with 1 input, which is a vector of 4 elements. |
105 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, 4}, "A" , false); |
106 | E = mod.createPlaceholder(ElemKind::FloatTy, {1, 4}, "E" , false); |
107 | Node *O = F->createFullyConnected(bindings, "fc1" , A, 10); |
108 | O = F->createSigmoid("sig1" , O); |
109 | O = F->createFullyConnected(bindings, "fc2" , O, 4); |
110 | O = F->createRegression("reg" , O, E); |
111 | F->createSave("return" , O); |
112 | } |
113 | // TODO if PHs aren't zeroed this will not always pass in release. Should |
114 | // check which operations are sensitive and update them to set AllocZero |
115 | // properly. |
116 | for (auto *PH : EET_.getModule().getPlaceholders()) { |
117 | PH->setAllocZero(); |
118 | } |
119 | PlaceholderBindings trainingBindings; |
120 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
121 | auto *resPH = EEI_.getModule().getPlaceholderByNameSlow("return" ); |
122 | |
123 | // Values for the input and output variables. |
124 | Tensor inputs(ElemKind::FloatTy, {1, 4}); |
125 | Tensor expected(ElemKind::FloatTy, {1, 4}); |
126 | inputs.getHandle<>() = {0.15f, 0.15f, 0.15f, 0.15f}; |
127 | expected.getHandle<>() = {0.9f, 0.9f, 0.9f, 0.9f}; |
128 | |
129 | auto *TF = glow::differentiate(F, TC); |
130 | auto tfName = TF->getName(); |
131 | auto fname = F->getName(); |
132 | EET_.compile(CompilationMode::Train); |
133 | |
134 | // Train the network. Learn 1000 batches. |
135 | runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, E}, |
136 | {&inputs, &expected}, tfName); |
137 | |
138 | // Testing the output vector. |
139 | PlaceholderBindings inferBindings; |
140 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
141 | A = EEI_.getModule().getPlaceholderByNameSlow("A" ); |
142 | EEI_.compile(CompilationMode::Infer); |
143 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
144 | updateInputPlaceholders(inferBindings, {A}, {&inputs}); |
145 | |
146 | EEI_.run(inferBindings, fname); |
147 | |
148 | auto RNWH = inferBindings.get(resPH)->getHandle(); |
149 | (void)RNWH; |
150 | |
151 | // Test the output: |
152 | EXPECT_NEAR(RNWH.at({0, 0}), 0.9, 0.05); |
153 | } |
154 | |
155 | TEST_P(MLTest, simpleRegression) { |
156 | CHECK_IF_ENABLED(); |
157 | TrainingConfig TC; |
158 | PlaceholderBindings trainingBindings, inferBindings; |
159 | |
160 | // This variable records the number of the next sample to be used for |
161 | // training. |
162 | size_t sampleCounter = 0; |
163 | |
164 | // Testing the regression layer. This test takes the first element from the |
165 | // input vector, adds one to it and places the result in the second element of |
166 | // the output vector. |
167 | const dim_t numInputs = 4; |
168 | |
169 | // Learning a single input vector. |
170 | TC.learningRate = 0.05; |
171 | |
172 | Tensor inputs(ElemKind::FloatTy, {1, numInputs}); |
173 | Tensor expected(ElemKind::FloatTy, {1, numInputs}); |
174 | Placeholder *A, *Ex; |
175 | Function *F; |
176 | for (auto *EE : engines_) { |
177 | auto &mod = EE->getModule(); |
178 | F = mod.createFunction("simpleRegression" ); |
179 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputs}, "A" , false); |
180 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, numInputs}, "E" , false); |
181 | Node *O = F->createFullyConnected(inferBindings, "fc" , A, 4); |
182 | O = F->createRELU("relu" , O); |
183 | O = F->createRegression("reg" , O, Ex); |
184 | F->createSave("result" , O); |
185 | } |
186 | auto resPH = EEI_.getModule().getPlaceholderByNameSlow("result" ); |
187 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
188 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
189 | inferBindings.clear(); |
190 | |
191 | auto I = inputs.getHandle<>(); |
192 | auto E = expected.getHandle<>(); |
193 | |
194 | auto *TF = glow::differentiate(F, TC); |
195 | auto tfName = TF->getName(); |
196 | auto fName = F->getName(); |
197 | EET_.compile(CompilationMode::Train); |
198 | |
199 | // Train the network: |
200 | for (int iter = 0; iter < 1000; iter++) { |
201 | float target = float(iter % 9); |
202 | I = {target, 0., 0., 0.}; |
203 | E = {0., target + 1, 0., 0.}; |
204 | runBatch(EET_, trainingBindings, 1, sampleCounter, {A, Ex}, |
205 | {&inputs, &expected}, tfName); |
206 | } |
207 | |
208 | // Verify the result of the regression layer. |
209 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
210 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
211 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
212 | EEI_.compile(CompilationMode::Infer); |
213 | |
214 | // Test the output: |
215 | for (int iter = 0; iter < 5; iter++) { |
216 | float target = iter % 9 + 1; |
217 | I = {target, 0., 0., 0.}; |
218 | updateInputPlaceholders(inferBindings, {A}, {&inputs}); |
219 | EEI_.run(inferBindings, fName); |
220 | auto *res = inferBindings.get(resPH); |
221 | auto resH = res->getHandle<>(); |
222 | (void)resH; |
223 | |
224 | EXPECT_NEAR(I.at({0, 0}) + 1, resH.at({0, 1}), 0.1); |
225 | } |
226 | } |
227 | |
228 | TEST_P(MLTest, learnXor) { |
229 | CHECK_IF_ENABLED(); |
230 | TrainingConfig TC; |
231 | PlaceholderBindings trainingBindings, inferBindings; |
232 | |
233 | unsigned numInputs = 10; |
234 | |
235 | // This variable records the number of the next sample to be used for |
236 | // training. |
237 | size_t sampleCounter = 0; |
238 | |
239 | // Learning a single input vector. |
240 | TC.learningRate = 0.05; |
241 | TC.batchSize = numInputs; |
242 | Placeholder *A, *Ex; |
243 | Function *F; |
244 | for (auto *EE : engines_) { |
245 | auto &mod = EE->getModule(); |
246 | F = mod.createFunction("learnXor" ); |
247 | |
248 | A = mod.createPlaceholder(ElemKind::FloatTy, {numInputs, 2}, "A" , false); |
249 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {numInputs, 1}, "Ex" , false); |
250 | |
251 | Node *O = F->createFullyConnected(inferBindings, "fc1" , A, 6); |
252 | O = F->createTanh("tanh1" , O); |
253 | O = F->createFullyConnected(inferBindings, "fc2" , O, 1); |
254 | O = F->createRegression("reg" , O, Ex); |
255 | F->createSave("ret" , O); |
256 | } |
257 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
258 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
259 | inferBindings.clear(); |
260 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
261 | |
262 | auto *res = |
263 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
264 | |
265 | // Prepare the training set and the testing set. |
266 | Tensor trainingSet(ElemKind::FloatTy, {numInputs, 2}); |
267 | Tensor trainingLabels(ElemKind::FloatTy, {numInputs, 1}); |
268 | |
269 | auto TS = trainingSet.getHandle<>(); |
270 | auto TL = trainingLabels.getHandle<>(); |
271 | |
272 | // Prepare the training data: |
273 | for (unsigned i = 0; i < numInputs; i++) { |
274 | int a = i % 2; |
275 | int b = (i >> 1) % 2; |
276 | TS.at({i, 0}) = a; |
277 | TS.at({i, 1}) = b; |
278 | TL.at({i, 0}) = a ^ b; |
279 | } |
280 | |
281 | auto *TF = glow::differentiate(F, TC); |
282 | auto tfName = TF->getName(); |
283 | auto fname = F->getName(); |
284 | EET_.compile(CompilationMode::Train); |
285 | |
286 | // Train the network: |
287 | runBatch(EET_, trainingBindings, 2500, sampleCounter, {A, Ex}, |
288 | {&trainingSet, &trainingLabels}, tfName); |
289 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
290 | EEI_.compile(CompilationMode::Infer); |
291 | |
292 | // Prepare the testing tensor: |
293 | for (unsigned i = 0; i < numInputs; i++) { |
294 | int a = (numInputs - i) % 2; |
295 | int b = ((numInputs - i) >> 1) % 2; |
296 | TS.at({i, 0}) = a; |
297 | TS.at({i, 1}) = b; |
298 | } |
299 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
300 | updateInputPlaceholders(inferBindings, {A}, {&trainingSet}); |
301 | EEI_.run(inferBindings, fname); |
302 | |
303 | auto resH = res->getHandle<>(); |
304 | |
305 | // Test the output: |
306 | for (dim_t i = 0; i < numInputs; i++) { |
307 | int a = TS.at({i, 0}); |
308 | int b = TS.at({i, 1}); |
309 | EXPECT_NEAR(resH.at({i, 0}), (a ^ b), 0.1); |
310 | } |
311 | } |
312 | |
313 | /// Learn the logarithmic function. |
314 | TEST_P(MLTest, learnLog) { |
315 | CHECK_IF_ENABLED(); |
316 | TrainingConfig TC; |
317 | PlaceholderBindings inferBindings, trainingBindings; |
318 | |
319 | // This variable records the number of the next sample to be used for |
320 | // training. |
321 | size_t sampleCounter = 0; |
322 | |
323 | unsigned numInputs = 50; |
324 | unsigned batchSize = 5; |
325 | TC.learningRate = 0.07; |
326 | TC.batchSize = batchSize; |
327 | Function *F; |
328 | Placeholder *A, *Ex; |
329 | for (auto *EE : engines_) { |
330 | auto &mod = EE->getModule(); |
331 | F = mod.createFunction("learnLog" ); |
332 | |
333 | A = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 1}, "A" , false); |
334 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 1}, "Ex" , false); |
335 | |
336 | Node *O = F->createFullyConnected(inferBindings, "fc1" , A, 4); |
337 | O = F->createTanh("tanh1" , O); |
338 | O = F->createFullyConnected(inferBindings, "fc2" , O, 1); |
339 | O = F->createRegression("reg" , O, Ex); |
340 | F->createSave("ret" , O); |
341 | } |
342 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
343 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
344 | inferBindings.clear(); |
345 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
346 | |
347 | auto *res = |
348 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
349 | |
350 | // Set the training data. |
351 | Tensor trainingSet(ElemKind::FloatTy, {numInputs, 1}); |
352 | Tensor trainingLabels(ElemKind::FloatTy, {numInputs, 1}); |
353 | |
354 | auto TS = trainingSet.getHandle<>(); |
355 | auto TL = trainingLabels.getHandle<>(); |
356 | |
357 | // Set the training data as floating number from range [0.75, 1.5). |
358 | const float LO = 0.75; // Lower bound of training data. |
359 | const float HI = 1.5; // Upper bound of training data. |
360 | for (dim_t i = 0; i < numInputs; i++) { |
361 | // Generate a floating number in the range of [LO,HI). |
362 | float a = LO + i * (HI - LO) / numInputs; |
363 | TS.at({i, 0}) = a; |
364 | TL.at({i, 0}) = std::log(a); |
365 | } |
366 | |
367 | auto *TF = glow::differentiate(F, TC); |
368 | auto tfName = TF->getName(); |
369 | auto fname = F->getName(); |
370 | EET_.compile(CompilationMode::Train); |
371 | |
372 | // Train the network: |
373 | runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, Ex}, |
374 | {&trainingSet, &trainingLabels}, tfName); |
375 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
376 | EEI_.compile(CompilationMode::Infer); |
377 | |
378 | // Set the testing data. |
379 | Tensor testSet(ElemKind::FloatTy, {batchSize, 1}); |
380 | |
381 | auto TES = testSet.getHandle<>(); |
382 | |
383 | const float LO_T = 0.85; // Lower bound of testing data. |
384 | const float HI_T = 1.45; // Upper bound of testing data. |
385 | |
386 | for (dim_t i = 0; i < batchSize; i++) { |
387 | // Generate a floating number in the range of [LO_T,HI_T). |
388 | float a = EEI_.getModule().getPRNG().nextRandReal(LO_T, HI_T); |
389 | TES.at({i, 0}) = a; |
390 | } |
391 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
392 | updateInputPlaceholders(inferBindings, {A}, {&testSet}); |
393 | EEI_.run(inferBindings, fname); |
394 | |
395 | auto resH = res->getHandle<>(); |
396 | |
397 | // Test the output: |
398 | for (dim_t i = 0; i < batchSize; i++) { |
399 | float a = TES.at({i, 0}); |
400 | EXPECT_NEAR(resH.at({i, 0}), (std::log(a)), 0.02); |
401 | } |
402 | } |
403 | |
404 | unsigned numSamples = 230; |
405 | |
406 | /// Generate data in two classes. The circle of dots that's close to the axis is |
407 | /// L0, and the rest of the dots, away from the axis are L1. |
408 | void generateCircleData(Tensor &coordinates, Tensor &labels, PseudoRNG &PRNG) { |
409 | auto C = coordinates.getHandle<>(); |
410 | auto L = labels.getHandle<int64_t>(); |
411 | |
412 | for (dim_t i = 0; i < numSamples / 2; i++) { |
413 | float r = PRNG.nextRand() * 0.4; |
414 | float a = PRNG.nextRand() * 3.141592 * 2; |
415 | float y = r * sin(a); |
416 | float x = r * cos(a); |
417 | |
418 | C.at({i * 2, 0u}) = x; |
419 | C.at({i * 2, 1u}) = y; |
420 | L.at({i * 2, 0}) = 1; |
421 | |
422 | r = PRNG.nextRand() * 0.4 + 0.8; |
423 | a = PRNG.nextRand() * 3.141592 * 2; |
424 | y = r * sin(a); |
425 | x = r * cos(a); |
426 | |
427 | C.at({i * 2 + 1, 0u}) = x; |
428 | C.at({i * 2 + 1, 1u}) = y; |
429 | L.at({i * 2 + 1, 0}) = 0; |
430 | } |
431 | } |
432 | |
433 | /// Test the fully connected layer and the softmax function. |
434 | /// Example from: |
435 | /// http://cs.stanford.edu/people/karpathy/convnetjs/demo/classify2d.html |
436 | TEST_P(MLTest, circle) { |
437 | CHECK_IF_ENABLED(); |
438 | TrainingConfig TC; |
439 | PlaceholderBindings trainingBindings, inferBindings; |
440 | |
441 | // This variable records the number of the next sample to be used for |
442 | // training. |
443 | size_t sampleCounter = 0; |
444 | |
445 | unsigned minibatchSize = 11; |
446 | |
447 | // Testing the softmax layer. |
448 | // Learning a single input vector. |
449 | TC.momentum = 0.9; |
450 | TC.learningRate = 0.01; |
451 | TC.batchSize = minibatchSize; |
452 | Function *F; |
453 | Placeholder *A, *S; |
454 | for (auto *EE : engines_) { |
455 | auto &mod = EE->getModule(); |
456 | F = mod.createFunction("circle" ); |
457 | A = mod.createPlaceholder(ElemKind::FloatTy, {minibatchSize, 2}, "A" , |
458 | false); |
459 | S = mod.createPlaceholder(ElemKind::Int64ITy, {minibatchSize, 1}, "S" , |
460 | false); |
461 | |
462 | auto *FCL0 = F->createFullyConnected(inferBindings, "fc1" , A, 8); |
463 | auto *T0 = F->createTanh("tanh1" , FCL0); |
464 | auto *FCL1 = F->createFullyConnected(inferBindings, "fc2" , T0, 2); |
465 | auto *SM = F->createSoftMax("soft" , FCL1, S); |
466 | F->createSave("ret" , SM); |
467 | } |
468 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
469 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
470 | inferBindings.clear(); |
471 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
472 | auto *res = |
473 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
474 | |
475 | auto *TF = glow::differentiate(F, TC); |
476 | auto tfName = TF->getName(); |
477 | auto fname = F->getName(); |
478 | EET_.compile(CompilationMode::Train); |
479 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
480 | |
481 | Tensor coordinates(ElemKind::FloatTy, {numSamples, 2}); |
482 | Tensor labels(ElemKind::Int64ITy, {numSamples, 1}); |
483 | generateCircleData(coordinates, labels, EET_.getModule().getPRNG()); |
484 | |
485 | // Training: |
486 | runBatch(EET_, trainingBindings, 4000, sampleCounter, {A, S}, |
487 | {&coordinates, &labels}, tfName); |
488 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
489 | EEI_.compile(CompilationMode::Infer); |
490 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
491 | // Print a diagram that depicts the network decision on a grid. |
492 | Tensor sample(ElemKind::FloatTy, {minibatchSize, 2}); |
493 | sample.zero(); |
494 | for (int x = -10; x < 10; x++) { |
495 | for (int y = -10; y < 10; y++) { |
496 | // Load the inputs: |
497 | sample.getHandle<>().at({0, 0}) = float(x) / 10; |
498 | sample.getHandle<>().at({0, 1}) = float(y) / 10; |
499 | |
500 | updateInputPlaceholders(inferBindings, {A}, {&sample}); |
501 | EEI_.run(inferBindings, fname); |
502 | |
503 | auto SMH = res->getHandle<>(); |
504 | auto A = SMH.at({0, 0}); |
505 | auto B = SMH.at({0, 1}); |
506 | |
507 | char ch = '='; |
508 | if (A > (B + 0.2)) { |
509 | ch = '+'; |
510 | } else if (B > (A + 0.2)) { |
511 | ch = '-'; |
512 | } |
513 | |
514 | llvm::outs() << ch; |
515 | } |
516 | llvm::outs() << "\n" ; |
517 | } |
518 | llvm::outs() << "\n" ; |
519 | |
520 | { |
521 | // The dot in the middle must be one. |
522 | sample.getHandle<>().at({0, 0}) = 0; |
523 | sample.getHandle<>().at({0, 1}) = 0; |
524 | updateInputPlaceholders(inferBindings, {A}, {&sample}); |
525 | EEI_.run(inferBindings, fname); |
526 | |
527 | auto SMH = res->getHandle<>(); |
528 | auto A = SMH.at({0, 0}); |
529 | auto B = SMH.at({0, 1}); |
530 | EXPECT_TRUE(B > (A + 0.2)); |
531 | } |
532 | |
533 | { |
534 | // Far away dot must be zero. |
535 | sample.getHandle<>().at({0, 0}) = 1; |
536 | sample.getHandle<>().at({0, 1}) = 1; |
537 | updateInputPlaceholders(inferBindings, {A}, {&sample}); |
538 | EEI_.run(inferBindings, fname); |
539 | auto SMH = res->getHandle<>(); |
540 | auto A = SMH.at({0, 0}); |
541 | auto B = SMH.at({0, 1}); |
542 | EXPECT_TRUE(A > (B + 0.2)); |
543 | } |
544 | } |
545 | |
546 | TEST_P(MLTest, learnSingleValueConcat) { |
547 | CHECK_IF_ENABLED(); |
548 | unsigned width = 6; |
549 | PlaceholderBindings inferBindings, trainingBindings; |
550 | |
551 | // This variable records the number of the next sample to be used for |
552 | // training. |
553 | size_t sampleCounter = 0; |
554 | |
555 | // Learning a single input vector. |
556 | TrainingConfig TC; |
557 | TC.momentum = 0.9; |
558 | TC.learningRate = 0.01; |
559 | Function *F; |
560 | Placeholder *A, *Ex, *B; |
561 | for (auto *EE : engines_) { |
562 | auto &mod = EE->getModule(); |
563 | F = mod.createFunction("learnSingleValueConcat" ); |
564 | |
565 | Ex = mod.createPlaceholder(ElemKind::FloatTy, {1, width * 2}, "Ex" , false); |
566 | |
567 | // Left side of the network: |
568 | A = mod.createPlaceholder(ElemKind::FloatTy, {1, width}, "A" , false); |
569 | Node *L = F->createFullyConnected(inferBindings, "fc1" , A, width); |
570 | L = F->createSigmoid("" , L); |
571 | |
572 | // Right side of the network: |
573 | B = mod.createPlaceholder(ElemKind::FloatTy, {1, width}, "B" , false); |
574 | Node *R = F->createFullyConnected(inferBindings, "fc2" , B, width); |
575 | R = F->createSigmoid("sig" , R); |
576 | |
577 | // Concat: |
578 | auto *C = F->createConcat("con" , {L, R}, 1); |
579 | auto *RN = F->createRegression("reg" , C, Ex); |
580 | F->createSave("ret" , RN); |
581 | } |
582 | |
583 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
584 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
585 | inferBindings.clear(); |
586 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
587 | auto *res = |
588 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
589 | |
590 | Tensor inputs(ElemKind::FloatTy, {1, width}); |
591 | Tensor expected(ElemKind::FloatTy, {1, width * 2}); |
592 | inputs.getHandle<>().clear(0.15); |
593 | expected.getHandle<>().clear(0.9); |
594 | auto *TF = glow::differentiate(F, TC); |
595 | auto tfName = TF->getName(); |
596 | EET_.compile(CompilationMode::Train); |
597 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
598 | |
599 | // Train the network: |
600 | runBatch(EET_, trainingBindings, 1000, sampleCounter, {A, B, Ex}, |
601 | {&inputs, &inputs, &expected}, tfName); |
602 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
603 | EEI_.compile(CompilationMode::Infer); |
604 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
605 | // Testing the output vector. |
606 | updateInputPlaceholders(inferBindings, {A}, {&inputs}); |
607 | EEI_.run(inferBindings); |
608 | auto RNWH = res->getHandle<>(); |
609 | (void)RNWH; |
610 | |
611 | // Test the output: |
612 | EXPECT_NEAR(RNWH.at({0, 0}), 0.9, 0.1); |
613 | } |
614 | |
615 | void buildGRU(PlaceholderBindings &bindings, Function *F, |
616 | const std::vector<NodeValue> &slicesX, unsigned hiddenSize, |
617 | unsigned outputSize, std::vector<NodeValue> &outputs) { |
618 | return F->createGRU(bindings, "GRU" , slicesX, 1, hiddenSize, outputSize, |
619 | outputs); |
620 | }; |
621 | |
622 | void buildRNN(PlaceholderBindings &bindings, Function *F, |
623 | const std::vector<NodeValue> &slicesX, unsigned hiddenSize, |
624 | unsigned outputSize, std::vector<NodeValue> &outputs) { |
625 | return F->createSimpleRNN(bindings, "SimpleRNN" , slicesX, 1, hiddenSize, |
626 | outputSize, outputs); |
627 | }; |
628 | |
629 | void buildLSTM(PlaceholderBindings &bindings, Function *F, |
630 | const std::vector<NodeValue> &slicesX, unsigned hiddenSize, |
631 | unsigned outputSize, std::vector<NodeValue> &outputs) { |
632 | return F->createLSTM(bindings, "LSTM" , slicesX, 1, hiddenSize, outputSize, |
633 | outputs); |
634 | }; |
635 | |
636 | using TCellGenerator = void (*)(PlaceholderBindings &, Function *, |
637 | const std::vector<NodeValue> &, unsigned, |
638 | unsigned, std::vector<NodeValue> &); |
639 | |
640 | void testRNNCell(TCellGenerator cell) { |
641 | TrainingConfig TC; |
642 | |
643 | // This variable records the number of the next sample to be used for |
644 | // training. |
645 | size_t sampleCounter = 0; |
646 | |
647 | PlaceholderBindings inferBindings, trainingBindings; |
648 | ExecutionEngine EEI, EET; |
649 | std::vector<ExecutionEngine *> engines; |
650 | engines.push_back(&EEI); |
651 | engines.push_back(&EET); |
652 | const unsigned NumVectors = 3; |
653 | const unsigned NumElements = 4; |
654 | // Learning a single input vector. |
655 | TC.learningRate = 0.05; |
656 | Function *F; |
657 | Placeholder *X, *Y; |
658 | for (auto *EE : engines) { |
659 | auto &mod = EE->getModule(); |
660 | F = mod.createFunction("testRNNCell" ); |
661 | |
662 | // Create a variable with 1 input, which is 3 consecutive vectors |
663 | // of 4 elements each. |
664 | X = mod.createPlaceholder(ElemKind::FloatTy, {1, NumVectors, NumElements}, |
665 | "X" , false); |
666 | Y = mod.createPlaceholder(ElemKind::FloatTy, {1, NumVectors}, "Y" , false); |
667 | inferBindings.allocate(X); |
668 | inferBindings.allocate(Y); |
669 | |
670 | // Extract a slice for each input. |
671 | std::vector<NodeValue> XSliced; |
672 | |
673 | for (unsigned i = 0; i < NumVectors; ++i) { |
674 | std::string Name{"X" }; |
675 | Name.append(std::to_string(i + 1)); |
676 | XSliced.push_back(F->createSlice(Name, X, {0, i, 0}, {1, i + 1, 4})); |
677 | } |
678 | |
679 | // Extract a slice for each output. |
680 | std::vector<Node *> YSliced; |
681 | |
682 | for (unsigned i = 0; i < NumVectors; ++i) { |
683 | std::string Name{"Y" }; |
684 | Name.append(std::to_string(i + 1)); |
685 | YSliced.push_back(F->createSlice(Name, Y, {0, i}, {1, i + 1})); |
686 | } |
687 | |
688 | const unsigned hiddenSize = 5; |
689 | const unsigned outputSize = 1; |
690 | |
691 | std::vector<NodeValue> outputNodes; |
692 | cell(inferBindings, F, XSliced, hiddenSize, outputSize, outputNodes); |
693 | |
694 | std::vector<NodeValue> regressionNodes; |
695 | for (unsigned t = 0; t < NumVectors; t++) { |
696 | regressionNodes.push_back( |
697 | F->createRegression("" , outputNodes[t], YSliced[t])); |
698 | }; |
699 | |
700 | auto *R = F->createConcat("O" , regressionNodes, 1); |
701 | F->createSave("result" , R); |
702 | } |
703 | // TODO if PHs aren't zeroed this will not always pass in release. Should |
704 | // check which operations are sensitive and update them to set AllocZero |
705 | // properly. |
706 | for (auto *PH : EEI.getModule().getPlaceholders()) { |
707 | PH->setAllocZero(); |
708 | } |
709 | for (auto *PH : EET.getModule().getPlaceholders()) { |
710 | PH->setAllocZero(); |
711 | } |
712 | trainingBindings.allocate(EET.getModule().getPlaceholders()); |
713 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
714 | inferBindings.clear(); |
715 | inferBindings.allocate(EEI.getModule().getPlaceholders()); |
716 | auto *res = |
717 | inferBindings.get(EEI.getModule().getPlaceholderByNameSlow("result" )); |
718 | |
719 | auto *TF = glow::differentiate(F, TC); |
720 | auto tfName = TF->getName(); |
721 | auto fname = F->getName(); |
722 | EET.compile(CompilationMode::Train); |
723 | |
724 | // Values for the input and output variables. |
725 | Tensor inputs(ElemKind::FloatTy, {1, NumVectors, NumElements}); |
726 | Tensor expected(ElemKind::FloatTy, {1, NumVectors}); |
727 | inputs.zero(); |
728 | expected.zero(); |
729 | for (dim_t i = 0; i < NumVectors; i++) { |
730 | inputs.getHandle<float_t>().at({0, i, 1}) = i; |
731 | expected.getHandle<float_t>().at({0, i}) = i; |
732 | } |
733 | |
734 | // Train the network. Learn 1000 batches. |
735 | runBatch(EET, trainingBindings, 1000, sampleCounter, {X, Y}, |
736 | {&inputs, &expected}, tfName); |
737 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
738 | // Testing the output vector. |
739 | EEI.compile(CompilationMode::Infer); |
740 | X = inferBindings.getPlaceholderByNameSlow("X" ); |
741 | updateInputPlaceholders(inferBindings, {X}, {&inputs}); |
742 | EEI.run(inferBindings, fname); |
743 | |
744 | auto RNWH = res->getHandle<>(); |
745 | (void)RNWH; |
746 | |
747 | // Test the output: |
748 | for (dim_t t = 0; t < NumVectors; ++t) { |
749 | EXPECT_NEAR(RNWH.at({0, t}), t, 0.05); |
750 | } |
751 | }; |
752 | |
753 | TEST_P(MLTest, trainASimpleRNN) { |
754 | CHECK_IF_ENABLED(); |
755 | testRNNCell(buildRNN); |
756 | }; |
757 | |
758 | TEST_P(MLTest, trainGRU) { |
759 | CHECK_IF_ENABLED(); |
760 | testRNNCell(buildGRU); |
761 | }; |
762 | |
763 | TEST_P(MLTest, trainLSTM) { |
764 | CHECK_IF_ENABLED(); |
765 | testRNNCell(buildLSTM); |
766 | }; |
767 | |
768 | TEST_P(MLTest, trainSimpleLinearRegression) { |
769 | CHECK_IF_ENABLED(); |
770 | TrainingConfig TC; |
771 | PlaceholderBindings bindings; |
772 | |
773 | // Given 1-D vectors x and y, find real numbers m and b such that |
774 | // m * x + b is approximately equal to y. |
775 | unsigned numSamples = 500; |
776 | |
777 | // This variable records the number of the next sample to be used for |
778 | // training. |
779 | size_t sampleCounter = 0; |
780 | |
781 | TC.learningRate = 0.1; |
782 | TC.batchSize = numSamples; |
783 | |
784 | auto &mod = EET_.getModule(); |
785 | Function *F = mod.createFunction( |
786 | "Gradient descent solution for simple linear regression" ); |
787 | |
788 | // These m and b are only used to generate training data. |
789 | float referenceM = 3.0; |
790 | float referenceB = 6.0; |
791 | |
792 | Tensor tensorX(ElemKind::FloatTy, {numSamples, 1}); |
793 | Tensor tensorY(ElemKind::FloatTy, {numSamples, 1}); |
794 | for (unsigned i = 0; i < numSamples; i++) { |
795 | float x_i = -2.0 + 4.0 * i / numSamples; |
796 | float y_i = referenceM * x_i + referenceB + mod.getPRNG().nextRand() / 10.0; |
797 | tensorX.getHandle<>().at({i, 0}) = x_i; |
798 | tensorY.getHandle<>().at({i, 0}) = y_i; |
799 | } |
800 | |
801 | // Create a variable with 1 input, which is a real number. |
802 | Placeholder *inputX = |
803 | mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1}, "input" , false); |
804 | Placeholder *expectedY = mod.createPlaceholder( |
805 | ElemKind::FloatTy, {numSamples, 1}, "expected" , false); |
806 | |
807 | FullyConnectedNode *FC = F->createFullyConnected(bindings, "fc" , inputX, 1); |
808 | Node *R = F->createRegression("reg" , FC, expectedY); |
809 | SaveNode *SN = F->createSave("return" , R); |
810 | |
811 | bindings.allocate(inputX); |
812 | bindings.allocate(expectedY); |
813 | bindings.allocate(SN->getPlaceholder()); |
814 | |
815 | Placeholder *M = llvm::cast<Placeholder>(FC->getWeights()); |
816 | Placeholder *B = llvm::cast<Placeholder>(FC->getBias()); |
817 | |
818 | auto *TF = glow::differentiate(F, TC); |
819 | auto tfName = TF->getName(); |
820 | EET_.compile(CompilationMode::Train); |
821 | |
822 | // Train the network doing 100 steps. Learn on 500 samples. |
823 | runBatch(EET_, bindings, 100, sampleCounter, {inputX, expectedY}, |
824 | {&tensorX, &tensorY}, tfName); |
825 | |
826 | // Testing trained m and b: |
827 | EXPECT_NEAR(bindings.get(M)->getHandle<>().at({0, 0}), referenceM, 0.01); |
828 | EXPECT_NEAR(bindings.get(B)->getHandle<>().at({0}), referenceB, 0.01); |
829 | } |
830 | |
831 | enum class Sport : size_t { BASKETBALL = 0, SOCCER = 1 }; |
832 | |
833 | void generatePlayerData(Tensor &players, Tensor &labels, |
834 | unsigned numTrainPlayers, PseudoRNG &PRNG) { |
835 | auto P = players.getHandle<>(); |
836 | auto L = labels.getHandle<int64_t>(); |
837 | |
838 | // Auto generate height/weights for basketball players. |
839 | for (dim_t i = 0; i < numTrainPlayers / 2; i++) { |
840 | auto heightInches = PRNG.nextRandInt(70, 88); |
841 | auto weightLbs = |
842 | 4 * heightInches + PRNG.nextRandInt(-85, -55); // [195, 297] |
843 | P.at({i, 0}) = heightInches; |
844 | P.at({i, 1}) = weightLbs; |
845 | L.at({i, 0}) = static_cast<size_t>(Sport::BASKETBALL); |
846 | } |
847 | |
848 | // Auto generate height/weights for soccer players. |
849 | for (dim_t i = numTrainPlayers / 2; i < numTrainPlayers; i++) { |
850 | auto heightInches = PRNG.nextRandInt(60, 76); |
851 | auto weightLbs = static_cast<unsigned>(2 * heightInches) + |
852 | PRNG.nextRandInt(20, 50); // [140, 202] |
853 | P.at({i, 0}) = heightInches; |
854 | P.at({i, 1}) = weightLbs; |
855 | L.at({i, 0}) = static_cast<size_t>(Sport::SOCCER); |
856 | } |
857 | } |
858 | |
859 | // Given a player's height and weight, classify them as a basketball or |
860 | // soccer player. |
861 | TEST_P(MLTest, classifyPlayerSport) { |
862 | CHECK_IF_ENABLED(); |
863 | const unsigned numTrainPlayers = 200; |
864 | const dim_t numFeatures = 2; |
865 | const dim_t numClasses = 2; |
866 | |
867 | TrainingConfig TC; |
868 | PlaceholderBindings inferBindings, trainingBindings; |
869 | |
870 | // This variable records the number of the next sample to be used for |
871 | // training. |
872 | size_t sampleCounter = 0; |
873 | |
874 | TC.learningRate = 0.05; |
875 | TC.batchSize = numTrainPlayers; |
876 | Function *F; |
877 | Placeholder *A, *S; |
878 | for (auto *EE : engines_) { |
879 | auto &mod = EE->getModule(); |
880 | F = mod.createFunction("classifyPlayers" ); |
881 | |
882 | A = mod.createPlaceholder(ElemKind::FloatTy, {numTrainPlayers, numFeatures}, |
883 | "A" , false); |
884 | S = mod.createPlaceholder(ElemKind::Int64ITy, {numTrainPlayers, 1}, "S" , |
885 | false); |
886 | |
887 | auto *FC = F->createFullyConnected(inferBindings, "fc" , A, numClasses); |
888 | auto *SM = F->createSoftMax("softmax" , FC, S); |
889 | F->createSave("result" , SM); |
890 | } |
891 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
892 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
893 | inferBindings.clear(); |
894 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
895 | |
896 | auto *TF = glow::differentiate(F, TC); |
897 | auto tfName = TF->getName(); |
898 | auto fname = F->getName(); |
899 | EET_.compile(CompilationMode::Train); |
900 | |
901 | Tensor players(ElemKind::FloatTy, {numTrainPlayers, numFeatures}); |
902 | Tensor labels(ElemKind::Int64ITy, {numTrainPlayers, 1}); |
903 | generatePlayerData(players, labels, numTrainPlayers, |
904 | EET_.getModule().getPRNG()); |
905 | |
906 | // Training: |
907 | runBatch(EET_, trainingBindings, 2000, sampleCounter, {A, S}, |
908 | {&players, &labels}, tfName); |
909 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
910 | EEI_.compile(CompilationMode::Infer); |
911 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
912 | std::vector<std::tuple<unsigned, unsigned, Sport>> testPlayers; |
913 | testPlayers.emplace_back(82, 240, Sport::BASKETBALL); |
914 | testPlayers.emplace_back(86, 260, Sport::BASKETBALL); |
915 | testPlayers.emplace_back(90, 270, Sport::BASKETBALL); |
916 | testPlayers.emplace_back(60, 160, Sport::SOCCER); |
917 | testPlayers.emplace_back(63, 155, Sport::SOCCER); |
918 | testPlayers.emplace_back(66, 170, Sport::SOCCER); |
919 | |
920 | Tensor testPlayersTensor(ElemKind::FloatTy, {numTrainPlayers, numFeatures}); |
921 | for (dim_t i = 0; i < testPlayers.size(); i++) { |
922 | testPlayersTensor.getHandle<>().at({i, 0}) = std::get<0>(testPlayers[i]); |
923 | testPlayersTensor.getHandle<>().at({i, 1}) = std::get<1>(testPlayers[i]); |
924 | } |
925 | |
926 | updateInputPlaceholders(inferBindings, {A}, {&testPlayersTensor}); |
927 | EEI_.run(inferBindings, fname); |
928 | |
929 | auto SMH = |
930 | inferBindings.get(inferBindings.getPlaceholderByNameSlow("result" )) |
931 | ->getHandle<>(); |
932 | for (dim_t i = 0; i < testPlayers.size(); i++) { |
933 | const dim_t sport = static_cast<dim_t>(std::get<2>(testPlayers[i])); |
934 | EXPECT_NEAR(SMH.at({i, sport}), 1.0, 0.1); |
935 | } |
936 | } |
937 | |
938 | TEST_P(MLTest, learnSinus) { |
939 | CHECK_IF_ENABLED(); |
940 | TrainingConfig TC; |
941 | PlaceholderBindings trainingBindings, inferBindings; |
942 | |
943 | // This variable records the number of the next sample to be used for |
944 | // training. |
945 | size_t sampleCounter = 0; |
946 | |
947 | // Try to learn the sin(x) function. |
948 | float epsilon = 0.1; |
949 | unsigned numSamples = 50; |
950 | Tensor tensorX(ElemKind::FloatTy, {numSamples, 1}); |
951 | Tensor tensorY(ElemKind::FloatTy, {numSamples, 1}); |
952 | |
953 | TC.learningRate = 0.2; |
954 | TC.batchSize = numSamples; |
955 | |
956 | // Function that should be learned by the NN |
957 | auto FF = [](float x) -> float { |
958 | // Return a shifted sin(x) value, so that it is in the range [0, 1]. |
959 | return (sin(x) + 1) / 2; |
960 | }; |
961 | Function *F; |
962 | Placeholder *inputX, *expectedY; |
963 | for (auto *EE : engines_) { |
964 | auto &mod = EE->getModule(); |
965 | F = mod.createFunction("Gradient descent solution for sin(x)" ); |
966 | |
967 | for (unsigned i = 0; i < numSamples; i++) { |
968 | // Scale x to cover the range [0, 4] as this leads to a good convergence |
969 | // during training. |
970 | float x = i / (numSamples / 4.0); |
971 | float y = FF(x); |
972 | tensorX.getHandle<>().at({i, 0}) = x; |
973 | tensorY.getHandle<>().at({i, 0}) = y; |
974 | } |
975 | |
976 | inputX = mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1}, "input" , |
977 | false); |
978 | |
979 | expectedY = mod.createPlaceholder(ElemKind::FloatTy, {numSamples, 1}, |
980 | "expected" , false); |
981 | |
982 | FullyConnectedNode *FC1 = |
983 | F->createFullyConnected(inferBindings, "fc1" , inputX, 10); |
984 | Node *O = F->createSigmoid("sigmoid1" , FC1); |
985 | FullyConnectedNode *FC2 = |
986 | F->createFullyConnected(inferBindings, "fc2" , O, 1); |
987 | Node *R = F->createRegression("reg" , FC2, expectedY); |
988 | F->createSave("return" , R); |
989 | } |
990 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
991 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
992 | inferBindings.clear(); |
993 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
994 | auto *res = |
995 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("return" )); |
996 | |
997 | auto *TF = glow::differentiate(F, TC); |
998 | auto tfName = TF->getName(); |
999 | auto fname = F->getName(); |
1000 | EET_.compile(CompilationMode::Train); |
1001 | |
1002 | // Learn on numSamples samples. |
1003 | runBatch(EET_, trainingBindings, 2700, sampleCounter, {inputX, expectedY}, |
1004 | {&tensorX, &tensorY}, tfName); |
1005 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1006 | // Create a test set, which is similar, but different from the training set. |
1007 | for (unsigned i = 0; i < numSamples; i++) { |
1008 | // Scale x to cover the range [0, 4.2] as this leads to a good convergence |
1009 | // during training. |
1010 | float x = i / (numSamples / 4.2) + 0.123456; |
1011 | float y = FF(x); |
1012 | tensorX.getHandle<>().at({i, 0}) = x; |
1013 | tensorY.getHandle<>().at({i, 0}) = y; |
1014 | } |
1015 | inputX = inferBindings.getPlaceholderByNameSlow("input" ); |
1016 | EEI_.compile(CompilationMode::Infer); |
1017 | updateInputPlaceholders(inferBindings, {inputX}, {&tensorX}); |
1018 | EEI_.run(inferBindings, fname); |
1019 | auto resH = res->getHandle<>(); |
1020 | |
1021 | for (dim_t i = 0; i < numSamples; i++) { |
1022 | float x = tensorX.getHandle().at({i, 0}); |
1023 | EXPECT_NEAR(resH.at({i, 0}), FF(x), epsilon); |
1024 | } |
1025 | } |
1026 | |
1027 | TEST_P(MLTest, nonLinearClassifier) { |
1028 | CHECK_IF_ENABLED(); |
1029 | // Test non-linear classification on a set of 2d points. Generate x and y in |
1030 | // (-1, 1) and classify according to XOR of the sign bit. |
1031 | unsigned batchSize = 46; |
1032 | unsigned numSamples = 230; |
1033 | |
1034 | // This variable records the number of the next sample to be used for |
1035 | // training. |
1036 | size_t sampleCounter = 0; |
1037 | |
1038 | PlaceholderBindings inferBindings, trainingBindings; |
1039 | TrainingConfig TC; |
1040 | TC.learningRate = 0.01; |
1041 | TC.momentum = 0.9; |
1042 | TC.batchSize = batchSize; |
1043 | Function *F; |
1044 | Placeholder *A, *S; |
1045 | for (auto *EE : engines_) { |
1046 | auto &mod = EE->getModule(); |
1047 | F = mod.createFunction("nonLinearClassifier" ); |
1048 | |
1049 | A = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 2}, "A" , false); |
1050 | S = mod.createPlaceholder(ElemKind::Int64ITy, {batchSize, 1}, "S" , false); |
1051 | |
1052 | auto *FCL0 = F->createFullyConnected(inferBindings, "fc1" , A, 8); |
1053 | auto *T0 = F->createTanh("tanh1" , FCL0); |
1054 | auto *FCL1 = F->createFullyConnected(inferBindings, "fc2" , T0, 8); |
1055 | auto *T1 = F->createTanh("tanh2" , FCL1); |
1056 | auto *FCL2 = F->createFullyConnected(inferBindings, "fc2" , T1, 2); |
1057 | auto *SM = F->createSoftMax("soft" , FCL2, S); |
1058 | F->createSave("ret" , SM); |
1059 | } |
1060 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1061 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1062 | inferBindings.clear(); |
1063 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
1064 | auto *res = |
1065 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
1066 | |
1067 | auto *TF = glow::differentiate(F, TC); |
1068 | auto tfName = TF->getName(); |
1069 | auto fname = F->getName(); |
1070 | EET_.compile(CompilationMode::Train); |
1071 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1072 | |
1073 | Tensor samples(ElemKind::FloatTy, {numSamples, 2}); |
1074 | Tensor labels(ElemKind::Int64ITy, {numSamples, 1}); |
1075 | |
1076 | for (dim_t i = 0; i < numSamples; i++) { |
1077 | float x = EET_.getModule().getPRNG().nextRand(); |
1078 | float y = EET_.getModule().getPRNG().nextRand(); |
1079 | dim_t label = (x < 0.0) ^ (y < 0.0); |
1080 | samples.getHandle<>().at({i, 0}) = x; |
1081 | samples.getHandle<>().at({i, 1}) = y; |
1082 | labels.getHandle<int64_t>().at({i, 0}) = label; |
1083 | } |
1084 | |
1085 | runBatch(EET_, trainingBindings, 500, sampleCounter, {A, S}, |
1086 | {&samples, &labels}, tfName); |
1087 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1088 | EEI_.compile(CompilationMode::Infer); |
1089 | A = inferBindings.getPlaceholderByNameSlow("A" ); |
1090 | std::vector<std::tuple<float, float, dim_t>> tests; |
1091 | tests.emplace_back(-0.8, -0.8, 0); |
1092 | tests.emplace_back(0.8, -0.8, 1); |
1093 | tests.emplace_back(-0.8, 0.8, 1); |
1094 | tests.emplace_back(0.8, 0.8, 0); |
1095 | auto RH = res->getHandle<>(); |
1096 | for (dim_t i = 0; i < tests.size(); i++) { |
1097 | Tensor T(ElemKind::FloatTy, {batchSize, 2}); |
1098 | T.getHandle<>().at({0, 0}) = std::get<0>(tests[i]); |
1099 | T.getHandle<>().at({0, 1}) = std::get<1>(tests[i]); |
1100 | updateInputPlaceholders(inferBindings, {A}, {&T}); |
1101 | EEI_.run(inferBindings, fname); |
1102 | EXPECT_NEAR(RH.at({0, std::get<2>(tests[i])}), 1.0, 0.2); |
1103 | } |
1104 | } |
1105 | |
1106 | /// Generate images in two classes. |
1107 | /// A "line" is labeled as 0 and a "cross" is labeled as 1. |
1108 | static void generateImageData(Tensor &images, Tensor &labels, PseudoRNG &PRNG) { |
1109 | auto L = labels.getHandle<int64_t>(); |
1110 | auto image = images.getHandle<>(); |
1111 | unsigned numSamples = images.dims()[0]; |
1112 | images.zero(); |
1113 | |
1114 | for (dim_t i = 0; i < numSamples; i++) { |
1115 | bool isLine = i % 2 == 0; |
1116 | L.at({i, 0}) = isLine ? 0 : 1; |
1117 | dim_t target = PRNG.nextRandInt(1, 6); |
1118 | if (isLine) { |
1119 | for (dim_t y = 0; y < 8; y++) |
1120 | image.at({i, target, y, 0u}) = 1; |
1121 | } else { |
1122 | for (dim_t pos = 0; pos < 8; pos++) { |
1123 | image.at({i, pos, target, 0u}) = 1; |
1124 | image.at({i, target, pos, 0u}) = 1; |
1125 | } |
1126 | } |
1127 | } |
1128 | } |
1129 | |
1130 | /// Test the convolutional layer. |
1131 | /// Use a simple convnet to learn two classes of images: Line and Cross. |
1132 | /// This test checks the results of the quantized network. |
1133 | TEST_P(MLTest, convNetForImageRecognition) { |
1134 | CHECK_IF_ENABLED(); |
1135 | EET_.setBackendName("Interpreter" ); |
1136 | ExecutionEngine EEP{"Interpreter" }; |
1137 | engines_.emplace(engines_.begin(), &EEP); |
1138 | const unsigned numSamples = 500; |
1139 | const unsigned batchSize = 7; |
1140 | PlaceholderBindings inferBindings, trainingBindings, profileBindings; |
1141 | |
1142 | // This variable records the number of the next sample to be used for |
1143 | // training. |
1144 | size_t sampleCounter = 0; |
1145 | |
1146 | TrainingConfig TC; |
1147 | TC.learningRate = 0.01; |
1148 | TC.batchSize = batchSize; |
1149 | TC.momentum = 0.9; |
1150 | std::string fName; |
1151 | for (auto *EE : engines_) { |
1152 | auto &mod = EE->getModule(); |
1153 | Function *F = mod.createFunction("convNetForImageRecognition" ); |
1154 | |
1155 | Placeholder *input = mod.createPlaceholder( |
1156 | ElemKind::FloatTy, {batchSize, 8, 8, 1}, "input" , false); |
1157 | |
1158 | Placeholder *ex = |
1159 | mod.createPlaceholder(ElemKind::Int64ITy, {batchSize, 1}, "exp" , false); |
1160 | |
1161 | auto *CV = F->createConv(inferBindings, "conv" , input, 1, 3, 1, 0, 1); |
1162 | auto *TANH = F->createTanh("tanh" , CV); |
1163 | auto *FCL = F->createFullyConnected(inferBindings, "fc" , TANH, 2); |
1164 | auto *SM = F->createSoftMax("sm" , FCL, ex); |
1165 | F->createSave("ret" , SM); |
1166 | fName = F->getName().str(); |
1167 | } |
1168 | |
1169 | auto *mod = &EET_.getModule(); |
1170 | auto input = mod->getPlaceholderByNameSlow("input" ); |
1171 | auto ex = mod->getPlaceholderByNameSlow("exp" ); |
1172 | |
1173 | auto *TF = glow::differentiate(mod->getFunction(fName), TC); |
1174 | auto tfName = TF->getName(); |
1175 | EET_.compile(CompilationMode::Train); |
1176 | trainingBindings.allocate(mod->getPlaceholders()); |
1177 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1178 | |
1179 | Tensor images(ElemKind::FloatTy, {numSamples, 8, 8, 1}); |
1180 | Tensor labels(ElemKind::Int64ITy, {numSamples, 1}); |
1181 | generateImageData(images, labels, mod->getPRNG()); |
1182 | |
1183 | // Training: |
1184 | runBatch(EET_, trainingBindings, 500, sampleCounter, {input, ex}, |
1185 | {&images, &labels}, tfName); |
1186 | |
1187 | mod = &EEP.getModule(); |
1188 | profileBindings.allocate(mod->getPlaceholders()); |
1189 | LoweredInfoMap loweredMapForProf; |
1190 | CompilationContext cctxProf{&profileBindings, &loweredMapForProf}; |
1191 | cctxProf.precisionConfig.quantMode = QuantizationMode::Profile; |
1192 | |
1193 | auto F = mod->getFunction(fName); |
1194 | input = mod->getPlaceholderByNameSlow("input" ); |
1195 | trainingBindings.copyTrainableWeightsTo(profileBindings); |
1196 | EEP.compile(cctxProf); |
1197 | // Since we are compiling in profiling mode the partitioner will create a new |
1198 | // function from the original. Get the new function. |
1199 | F = mod->getFunctions().front(); |
1200 | |
1201 | runBatch(EEP, profileBindings, 100, sampleCounter, {input}, {&images}, fName); |
1202 | |
1203 | // Evaluate on the quantized function: |
1204 | // Set the execution backend to the backend that we test. |
1205 | mod = &EEI_.getModule(); |
1206 | inferBindings.clear(); |
1207 | inferBindings.allocate(mod->getPlaceholders()); |
1208 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1209 | |
1210 | LoweredInfoMap loweredMapForQuant; |
1211 | CompilationContext cctxQuant{&inferBindings, &loweredMapForQuant}; |
1212 | PrecisionConfiguration &precConfig = cctxQuant.precisionConfig; |
1213 | cctxQuant.precisionConfig.quantMode = QuantizationMode::Quantize; |
1214 | precConfig.quantConfig.infos = quantization::generateNodeProfilingInfos( |
1215 | profileBindings, F, loweredMapForProf); |
1216 | precConfig.quantConfig.assertAllNodesQuantized = true; |
1217 | |
1218 | // Softmax is not supported in Int8QTy, so signal the quantizer it's OK to |
1219 | // keep it unquantized. |
1220 | precConfig.precisionModeKindSet.insert(Kinded::Kind::SoftMaxNodeKind); |
1221 | |
1222 | F = mod->getFunction("convNetForImageRecognition" ); |
1223 | EEI_.compile(cctxQuant); |
1224 | input = mod->getPlaceholderByNameSlow("input" ); |
1225 | |
1226 | // Generate the images used for testing. |
1227 | Tensor testImages(ElemKind::FloatTy, {batchSize, 8, 8, 1}); |
1228 | Tensor testLabels(ElemKind::Int64ITy, {batchSize, 1}); |
1229 | generateImageData(testImages, testLabels, mod->getPRNG()); |
1230 | updateInputPlaceholders(inferBindings, {input}, {&testImages}); |
1231 | |
1232 | EEI_.run(inferBindings); |
1233 | |
1234 | Tensor *res = |
1235 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
1236 | auto SMH = res->getHandle<>(); |
1237 | for (dim_t i = 0; i < batchSize; i++) { |
1238 | bool isLine = testLabels.getHandle<int64_t>().at({i, 0}) == 0; |
1239 | auto lineWeight = SMH.at({i, 0}); |
1240 | auto crossWeight = SMH.at({i, 1}); |
1241 | EXPECT_TRUE((isLine && lineWeight > crossWeight) || |
1242 | (!isLine && crossWeight > lineWeight)); |
1243 | } |
1244 | } |
1245 | |
1246 | /// Generate data for the regression test. Put a '1' in a random location in a |
1247 | /// clear tensor and report the coordinates of that pixel. |
1248 | static void generateRegressionTestData(Tensor &images, Tensor &labels, |
1249 | PseudoRNG &PRNG) { |
1250 | auto L = labels.getHandle<>(); |
1251 | auto image = images.getHandle<>(); |
1252 | unsigned numSamples = images.dims()[0]; |
1253 | image.clear(0); |
1254 | |
1255 | for (dim_t i = 0; i < numSamples; i++) { |
1256 | // Generate the X,Y coordinates to place our object. |
1257 | dim_t x = PRNG.nextRandInt(0, 9); |
1258 | dim_t y = PRNG.nextRandInt(0, 9); |
1259 | L.at({i, 0}) = x; |
1260 | L.at({i, 1}) = y; |
1261 | image.at({i, x, y, 0u}) = 1; |
1262 | } |
1263 | } |
1264 | |
1265 | /// This is the "Where's Waldo" test. We place a pixel in a tensor and the |
1266 | /// network reports the coordinate of the pixel. |
1267 | TEST_P(MLTest, testFindPixelRegression) { |
1268 | CHECK_IF_ENABLED(); |
1269 | EET_.setBackendName("Interpreter" ); |
1270 | ExecutionEngine EEP{"Interpreter" }; |
1271 | engines_.emplace(engines_.begin(), &EEP); |
1272 | PlaceholderBindings inferBindings, trainingBindings, profileBindings; |
1273 | |
1274 | const unsigned numSamples = 1000; |
1275 | const unsigned batchSize = 10; |
1276 | |
1277 | // This variable records the number of the next sample to be used for |
1278 | // training. |
1279 | size_t sampleCounter = 0; |
1280 | |
1281 | TrainingConfig TC; |
1282 | TC.learningRate = 0.01; |
1283 | TC.batchSize = batchSize; |
1284 | TC.momentum = 0.9; |
1285 | std::string fName; |
1286 | for (auto *EE : engines_) { |
1287 | auto &mod = EE->getModule(); |
1288 | Function *F = mod.createFunction("main" ); |
1289 | |
1290 | Placeholder *input = mod.createPlaceholder( |
1291 | ElemKind::FloatTy, {batchSize, 10, 10, 1}, "input" , false); |
1292 | Placeholder *ex = mod.createPlaceholder(ElemKind::FloatTy, {batchSize, 2}, |
1293 | "coordinates" , false); |
1294 | |
1295 | // A simple single-layer FC network could solve this program but we use a |
1296 | // two layer FC network to give the compiler something slightly more complex |
1297 | // to work with so we are adding another FC layer. |
1298 | auto *FC0 = F->createFullyConnected(inferBindings, "fc0" , input, 6); |
1299 | auto *RL0 = F->createRELU("relu0" , FC0); |
1300 | auto *FC1 = F->createFullyConnected(inferBindings, "fc1" , RL0, 2); |
1301 | auto *R = F->createRegression("regression" , FC1, ex); |
1302 | F->createSave("ret" , R); |
1303 | fName = F->getName().str(); |
1304 | } |
1305 | |
1306 | auto *mod = &EET_.getModule(); |
1307 | auto input = mod->getPlaceholderByNameSlow("input" ); |
1308 | auto ex = mod->getPlaceholderByNameSlow("coordinates" ); |
1309 | |
1310 | auto *TF = glow::differentiate(mod->getFunction(fName), TC); |
1311 | auto tfName = TF->getName(); |
1312 | EET_.compile(CompilationMode::Train); |
1313 | // Specify these to initialze to zero to prevent uninitialized memory issues. |
1314 | for (auto *PH : EET_.getModule().getPlaceholders()) { |
1315 | PH->setAllocZero(); |
1316 | } |
1317 | trainingBindings.allocate(mod->getPlaceholders()); |
1318 | |
1319 | for (auto &PH : inferBindings.pairs()) { |
1320 | inferBindings.copyToTarget(PH.first->getName(), trainingBindings); |
1321 | } |
1322 | inferBindings.clear(); |
1323 | |
1324 | // -- STEP1 - train the network. -- |
1325 | Tensor images(ElemKind::FloatTy, {numSamples, 10, 10, 1}); |
1326 | Tensor labels(ElemKind::FloatTy, {numSamples, 2}); |
1327 | generateRegressionTestData(images, labels, mod->getPRNG()); |
1328 | |
1329 | // Training: |
1330 | runBatch(EET_, trainingBindings, 400, sampleCounter, {input, ex}, |
1331 | {&images, &labels}, tfName); |
1332 | |
1333 | // -- STEP2 - Profile and quantize the network. -- |
1334 | mod = &EEP.getModule(); |
1335 | for (auto *PH : mod->getPlaceholders()) { |
1336 | PH->setAllocZero(); |
1337 | } |
1338 | profileBindings.allocate(mod->getPlaceholders()); |
1339 | Tensor profileImages(ElemKind::FloatTy, {batchSize, 10, 10, 1}); |
1340 | Tensor profileLabels(ElemKind::FloatTy, {batchSize, 2}); |
1341 | generateRegressionTestData(profileImages, profileLabels, mod->getPRNG()); |
1342 | input = mod->getPlaceholderByNameSlow("input" ); |
1343 | updateInputPlaceholders(profileBindings, {input}, {&profileImages}); |
1344 | LoweredInfoMap loweredMapForProf; |
1345 | CompilationContext cctxProf{&profileBindings, &loweredMapForProf}; |
1346 | cctxProf.precisionConfig.quantMode = QuantizationMode::Profile; |
1347 | |
1348 | trainingBindings.copyTrainableWeightsTo(profileBindings); |
1349 | EEP.compile(cctxProf); |
1350 | // Get new function after partitioning. |
1351 | auto F = EEP.getModule().getFunctions().front(); |
1352 | |
1353 | // Run the graph to capture the profile. |
1354 | runBatch(EEP, profileBindings, 100, sampleCounter, {input}, {&images}, fName); |
1355 | |
1356 | // -- STEP3 - evaluate the quantized function. -- |
1357 | mod = &EEI_.getModule(); |
1358 | inferBindings.allocate(mod->getPlaceholders()); |
1359 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1360 | |
1361 | LoweredInfoMap loweredMapForQuant; |
1362 | CompilationContext cctxQuant{&inferBindings, &loweredMapForQuant}; |
1363 | cctxQuant.precisionConfig.quantMode = QuantizationMode::Quantize; |
1364 | cctxQuant.loweredInfoMap = &loweredMapForQuant; |
1365 | cctxQuant.precisionConfig.quantConfig.infos = |
1366 | quantization::generateNodeProfilingInfos(profileBindings, F, |
1367 | loweredMapForProf); |
1368 | cctxQuant.precisionConfig.quantConfig.assertAllNodesQuantized = true; |
1369 | |
1370 | F = mod->getFunction(fName); |
1371 | EEI_.compile(cctxQuant); |
1372 | input = mod->getPlaceholderByNameSlow("input" ); |
1373 | |
1374 | // Generate the images used for testing. |
1375 | Tensor testImages(ElemKind::FloatTy, {batchSize, 10, 10, 1}); |
1376 | Tensor testLabels(ElemKind::FloatTy, {batchSize, 2}); |
1377 | generateRegressionTestData(testImages, testLabels, mod->getPRNG()); |
1378 | |
1379 | // Run the inference: |
1380 | updateInputPlaceholders(inferBindings, {input}, {&testImages}); |
1381 | EEI_.run(inferBindings); |
1382 | |
1383 | // A handle to the projected result. |
1384 | Tensor *res = |
1385 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("ret" )); |
1386 | auto RH = res->getHandle<>(); |
1387 | // A handle to the true label. |
1388 | auto LH = testLabels.getHandle<>(); |
1389 | |
1390 | // Check how many of the coordinates that were reported are close to the real |
1391 | // values. |
1392 | unsigned correct = 0; |
1393 | |
1394 | for (dim_t i = 0; i < batchSize; i++) { |
1395 | // Calculate the distance between the predicted value and correct value. |
1396 | auto dx = LH.at({i, 0}) - RH.at({i, 0}); |
1397 | auto dy = LH.at({i, 1}) - RH.at({i, 1}); |
1398 | auto distance = std::sqrt(std::pow(dx, 2) + std::pow(dy, 2)); |
1399 | |
1400 | // A correct answer is one where the projected distance is somewhat close. |
1401 | correct += distance < 3; |
1402 | } |
1403 | |
1404 | // Expect 90% of the results to be correct. |
1405 | EXPECT_GE(correct, batchSize * 0.90); |
1406 | } |
1407 | |
1408 | // Generate tests for a toy neural network that can recognize if a matrix 3x3 |
1409 | // is a rotation of another matrix 3x3. |
1410 | // This is *not* about rotation matrices used for computer graphics, but a much |
1411 | // simpler concept. |
1412 | // Informally, let M1 and M2 be two 3x3 matrices, M2 is a rotation of M1 if it |
1413 | // exists a way to rotate the cells of M1 to get M2. The rotations are all |
1414 | // centered in the middle of the matrices. |
1415 | // E.g., |
1416 | // Rotate clockwise 1 cell centered in 'e': |
1417 | // --+ |
1418 | // | a b c | | |
1419 | // M1 = | d e f | V |
1420 | // | g h i | |
1421 | // => |
1422 | // | d a b | |
1423 | // M2 = | g e c | |
1424 | // | h i f | |
1425 | static void generateMatrixRotationRecognitionData(Tensor &matricesA, |
1426 | Tensor &matricesB, |
1427 | Tensor &expected, |
1428 | PseudoRNG &PRNG) { |
1429 | |
1430 | using CellIdx = std::pair<uint8_t, uint8_t>; |
1431 | // List the indices in a clockwise ordering starting from the top left |
1432 | // corner. |
1433 | // Note: This does not include the cell in the middle given it is |
1434 | // never rotated. |
1435 | static const CellIdx clockwiseOrder[] = {{0, 0}, {0, 1}, {0, 2}, {1, 2}, |
1436 | {2, 2}, {2, 1}, {2, 0}, {1, 0}}; |
1437 | static const uint8_t possibleTargetCells = |
1438 | sizeof(clockwiseOrder) / sizeof(clockwiseOrder[0]); |
1439 | const unsigned numSamples = matricesA.dims()[0]; |
1440 | assert(expected.dims()[0] == numSamples && |
1441 | matricesB.dims()[0] == numSamples && |
1442 | "Size of the tensors is incompatible" ); |
1443 | auto handleMatricesA = matricesA.getHandle<float>(); |
1444 | auto handleMatricesB = matricesB.getHandle<float>(); |
1445 | auto handleExpected = expected.getHandle<int64_t>(); |
1446 | |
1447 | handleMatricesA.randomize<int>(0, 1, PRNG); |
1448 | handleMatricesB.randomize<int>(0, 1, PRNG); |
1449 | for (unsigned idx = 0; idx < numSamples; ++idx) { |
1450 | // Toss a coin and create a rotation relationship or not. |
1451 | bool shouldHaveRotation = PRNG.nextRandInt(0, 1); |
1452 | handleExpected.at({idx, 0}) = shouldHaveRotation; |
1453 | if (shouldHaveRotation) { |
1454 | // On a 3x3 matrix we have 8 different possbile clockwise steps. |
1455 | // Pick one. |
1456 | size_t clockwiseSteps = PRNG.nextRandInt(0, possibleTargetCells - 1); |
1457 | // Generate the rotation matrix from A. |
1458 | // The center never changes. |
1459 | handleMatricesB.at({idx, 1, 1}) = handleMatricesA.at({idx, 1, 1}); |
1460 | // Fetch the cell registered in the clockwiseOrder at the desired step. |
1461 | for (size_t i = 0; i != possibleTargetCells; ++i) { |
1462 | const CellIdx &sourceCellIdx = clockwiseOrder[i]; |
1463 | const CellIdx &targetCellIdx = |
1464 | clockwiseOrder[(i + clockwiseSteps) % possibleTargetCells]; |
1465 | handleMatricesB.at({idx, targetCellIdx.first, targetCellIdx.second}) = |
1466 | handleMatricesA.at( |
1467 | {idx, sourceCellIdx.first, sourceCellIdx.second}); |
1468 | } |
1469 | } |
1470 | // Else: |
1471 | // There is a high probability that A and B don't have a rotation |
1472 | // relationship and thus, there is nothing to do. |
1473 | // Worse case, we mislabeled a relationship. |
1474 | |
1475 | // Alternatively we could always alter one of the matrix such that it is |
1476 | // impossible to have a rotation between them (e.g., make sure the center |
1477 | // is different), but that would bias the kind of differences that could |
1478 | // occur. |
1479 | } |
1480 | } |
1481 | |
1482 | TEST_P(MLTest, matrixRotationRecognition) { |
1483 | CHECK_IF_ENABLED(); |
1484 | TrainingConfig TC; |
1485 | TC.learningRate = 0.15; |
1486 | TC.batchSize = 17; |
1487 | PlaceholderBindings inferBindings, trainingBindings; |
1488 | |
1489 | // This variable records the number of the next sample to be used for |
1490 | // training. |
1491 | size_t sampleCounter = 0; |
1492 | Function *F; |
1493 | Placeholder *varMatricesA, *varMatricesB, *varExpected; |
1494 | for (auto *EE : engines_) { |
1495 | Module &mod = EE->getModule(); |
1496 | F = mod.createFunction("MatrixRotationRecognition" ); |
1497 | varMatricesA = mod.createPlaceholder( |
1498 | ElemKind::FloatTy, {TC.batchSize, 3, 3}, "matrixA" , false); |
1499 | varMatricesB = mod.createPlaceholder( |
1500 | ElemKind::FloatTy, {TC.batchSize, 3, 3}, "matrixB" , false); |
1501 | varExpected = mod.createPlaceholder(ElemKind::Int64ITy, {TC.batchSize, 1}, |
1502 | "expected" , false); |
1503 | |
1504 | // Simply concatenating the matrices first would probability be as effective |
1505 | // but we want to build something more complex than a straight chain of |
1506 | // operators to stress the scheduler. |
1507 | auto *FCA = F->createFullyConnected(inferBindings, "hidden_matrixA_fc" , |
1508 | varMatricesA, 10); |
1509 | auto *FCB = F->createFullyConnected(inferBindings, "hidden_matrixB_fc" , |
1510 | varMatricesB, 10); |
1511 | auto *ReLUA = F->createRELU("hidden_matrixA_ReLU" , FCA); |
1512 | auto *ReLUB = F->createRELU("hidden_matrixB_ReLU" , FCB); |
1513 | auto *concat = F->createConcat("hidden_concat_A_B" , {ReLUA, ReLUB}, 1); |
1514 | auto *hiddenFC = |
1515 | F->createFullyConnected(inferBindings, "hidden_fc" , concat, 30); |
1516 | auto *finalReLU = F->createRELU("hidden_concat_ReLU" , hiddenFC); |
1517 | auto *finalFC = |
1518 | F->createFullyConnected(inferBindings, "output_fc" , finalReLU, 2); |
1519 | auto *softMax = F->createSoftMax("output" , finalFC, varExpected); |
1520 | F->createSave("result" , softMax); |
1521 | } |
1522 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1523 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1524 | inferBindings.clear(); |
1525 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
1526 | auto *res = |
1527 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("result" )); |
1528 | auto *TF = glow::differentiate(F, TC); |
1529 | auto tfName = TF->getName(); |
1530 | auto fname = F->getName(); |
1531 | |
1532 | // Train the network. |
1533 | const unsigned numSamples = 50; |
1534 | Tensor matricesA(ElemKind::FloatTy, {numSamples, 3, 3}); |
1535 | Tensor matricesB(ElemKind::FloatTy, {numSamples, 3, 3}); |
1536 | Tensor expected(ElemKind::Int64ITy, {numSamples, 1}); |
1537 | generateMatrixRotationRecognitionData(matricesA, matricesB, expected, |
1538 | EET_.getModule().getPRNG()); |
1539 | |
1540 | EET_.compile(CompilationMode::Train); |
1541 | // Training: |
1542 | runBatch(EET_, trainingBindings, 200, sampleCounter, |
1543 | {varMatricesA, varMatricesB, varExpected}, |
1544 | {&matricesA, &matricesB, &expected}, tfName); |
1545 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1546 | |
1547 | // Switch to inference mode. |
1548 | EEI_.compile(CompilationMode::Infer); |
1549 | |
1550 | // At this point we should have overfitted the data. |
1551 | // Take a random batch and check that the values match what we expect. |
1552 | auto RHtrain = res->getHandle<>(); |
1553 | auto batchSize = TC.batchSize; |
1554 | unsigned numBatches = numSamples / batchSize; |
1555 | unsigned batchStartIdx = |
1556 | EEI_.getModule().getPRNG().nextRandInt(0, numBatches - 1) * batchSize; |
1557 | varMatricesA = inferBindings.getPlaceholderByNameSlow("matrixA" ); |
1558 | varMatricesB = inferBindings.getPlaceholderByNameSlow("matrixB" ); |
1559 | auto batchMatricesA = |
1560 | matricesA.getUnowned({batchSize, 3, 3}, {batchStartIdx, 0, 0}); |
1561 | auto batchMatricesB = |
1562 | matricesB.getUnowned({batchSize, 3, 3}, {batchStartIdx, 0, 0}); |
1563 | updateInputPlaceholders(inferBindings, {varMatricesA, varMatricesB}, |
1564 | {&batchMatricesA, &batchMatricesB}); |
1565 | EEI_.run(inferBindings, fname); |
1566 | |
1567 | unsigned errors = 0; |
1568 | // Check each output in the batch. |
1569 | for (dim_t i = 0; i != batchSize; i++) { |
1570 | // Note that the two softmax outputs always sum to 1, so we only look at |
1571 | // one. Index one is true if there is a rotation. |
1572 | float value = RHtrain.at({i, 1}); |
1573 | bool hasRotation = expected.getHandle<int64_t>().at({batchStartIdx + i, 0}); |
1574 | if ((value > 0.5) != hasRotation) { |
1575 | ++errors; |
1576 | } |
1577 | } |
1578 | |
1579 | EXPECT_LE(errors, 1); |
1580 | } |
1581 | |
1582 | /// Simple test case that learns the embedding table for a |
1583 | /// SparseLengthsSum operator. |
1584 | TEST_P(MLTest, learnSparseLengthsSumEmbeddings) { |
1585 | CHECK_IF_ENABLED(); |
1586 | TrainingConfig TC; |
1587 | TC.learningRate = 0.3; |
1588 | TC.batchSize = 1; |
1589 | |
1590 | PlaceholderBindings trainingBindings, inferBindings; |
1591 | Function *F; |
1592 | Placeholder *dataP, *indicesP, *lengthsP, *expectedP; |
1593 | PseudoRNG &PRNG = EET_.getModule().getPRNG(); |
1594 | for (auto *EE : engines_) { |
1595 | Module &mod = EE->getModule(); |
1596 | |
1597 | // Create a model consisting of one SparseLengthsSum operator |
1598 | // followed by a Regression node to get some non-zero gradients. |
1599 | F = mod.createFunction("SparseLengthsSum" ); |
1600 | dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP" , |
1601 | /*isTrainable=*/true); |
1602 | indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP" , |
1603 | /*isTrainable=*/false); |
1604 | lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP" , |
1605 | /*isTrainable=*/false); |
1606 | expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP" , |
1607 | /*isTrainable=*/false); |
1608 | |
1609 | auto *SLWS = F->createSparseLengthsSum("SLWS" , dataP, indicesP, lengthsP); |
1610 | auto *reg = F->createRegression("reg" , SLWS, expectedP); |
1611 | F->createSave("save" , reg); |
1612 | } |
1613 | // Allocate and randomly initialize embeddings. |
1614 | auto DH = inferBindings.allocate(dataP)->getHandle(); |
1615 | DH.randomize(-5.0, 5.0, PRNG); |
1616 | |
1617 | // Allocate and set indices such that input embeddings are reversed. |
1618 | inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5, |
1619 | 4, 3, 2, 1, 0}; |
1620 | |
1621 | // Allocate and set lengths. |
1622 | inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2}; |
1623 | |
1624 | // Allocate and set expected outputs. The embedding table will be adjusted |
1625 | // during training so that the final result is this. |
1626 | auto EH = inferBindings.allocate(expectedP)->getHandle(); |
1627 | EH = {1, 2, 3, 4, 5}; |
1628 | |
1629 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1630 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1631 | inferBindings.copyToTarget("dataP" , trainingBindings); |
1632 | inferBindings.copyToTarget("indicesP" , trainingBindings); |
1633 | inferBindings.copyToTarget("lengthsP" , trainingBindings); |
1634 | inferBindings.copyToTarget("expectedP" , trainingBindings); |
1635 | inferBindings.clear(); |
1636 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
1637 | EH = trainingBindings |
1638 | .get(trainingBindings.getPlaceholderByNameSlow("expectedP" )) |
1639 | ->getHandle(); |
1640 | auto *res = |
1641 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save" )); |
1642 | |
1643 | // Train the network. |
1644 | auto *TF = glow::differentiate(F, TC); |
1645 | auto tfName = TF->getName(); |
1646 | auto fname = F->getName(); |
1647 | EET_.compile(CompilationMode::Train); |
1648 | |
1649 | const size_t numIterations = 1000; |
1650 | |
1651 | for (size_t i = 0; i < numIterations; ++i) { |
1652 | EET_.run(trainingBindings, tfName); |
1653 | } |
1654 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1655 | trainingBindings.copyToTarget("dataP" , inferBindings); |
1656 | trainingBindings.copyToTarget("indicesP" , inferBindings); |
1657 | trainingBindings.copyToTarget("lengthsP" , inferBindings); |
1658 | trainingBindings.copyToTarget("expectedP" , inferBindings); |
1659 | // Switch to inference mode and run the network. |
1660 | EEI_.compile(CompilationMode::Infer); |
1661 | EEI_.run(inferBindings, fname); |
1662 | |
1663 | // Make sure that the network output matches expectations after training. |
1664 | auto RH = res->getHandle(); |
1665 | for (size_t j = 0; j < EH.size(); ++j) { |
1666 | EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02); |
1667 | } |
1668 | } |
1669 | |
1670 | /// Simple test case that learns the embedding table for a |
1671 | /// SparseLengthsWeightedSum operator. |
1672 | TEST_P(MLTest, learnSparseLengthsWeightedSumEmbeddings) { |
1673 | CHECK_IF_ENABLED(); |
1674 | TrainingConfig TC; |
1675 | TC.learningRate = 0.3; |
1676 | TC.batchSize = 1; |
1677 | |
1678 | PlaceholderBindings trainingBindings, inferBindings; |
1679 | Function *F; |
1680 | Placeholder *dataP, *indicesP, *lengthsP, *expectedP, *weightsP; |
1681 | PseudoRNG &PRNG = EET_.getModule().getPRNG(); |
1682 | for (auto *EE : engines_) { |
1683 | Module &mod = EE->getModule(); |
1684 | |
1685 | // Create a model consisting of one SparseLengthsWeightedSum operator |
1686 | // followed by a Regression node to get some non-zero gradients. |
1687 | F = mod.createFunction("SparseLengthsWeightedSum" ); |
1688 | dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP" , |
1689 | /*isTrainable=*/true); |
1690 | indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP" , |
1691 | /*isTrainable=*/false); |
1692 | weightsP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "weightsP" , |
1693 | /*isTrainable=*/false); |
1694 | lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP" , |
1695 | /*isTrainable=*/false); |
1696 | expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP" , |
1697 | /*isTrainable=*/false); |
1698 | |
1699 | auto *SLWS = F->createSparseLengthsWeightedSum("SLWS" , dataP, weightsP, |
1700 | indicesP, lengthsP); |
1701 | auto *reg = F->createRegression("reg" , SLWS, expectedP); |
1702 | F->createSave("save" , reg); |
1703 | } |
1704 | // Allocate and randomly initialize embeddings. |
1705 | auto DH = inferBindings.allocate(dataP)->getHandle(); |
1706 | DH.randomize(-5.0, 5.0, PRNG); |
1707 | |
1708 | // Allocate and set indices such that input embeddings are reversed. |
1709 | inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5, |
1710 | 4, 3, 2, 1, 0}; |
1711 | // Allocate and set weights. |
1712 | inferBindings.allocate(weightsP)->getHandle() = { |
1713 | 0.75, 0.25, 0.75, 0.25, 0.75, 0.25, 0.75, 0.25, 0.75, 0.25}; |
1714 | |
1715 | // Allocate and set lengths. |
1716 | inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2}; |
1717 | |
1718 | // Allocate and set expected outputs. The embedding table will be adjusted |
1719 | // during training so that the final result is this. |
1720 | auto EH = inferBindings.allocate(expectedP)->getHandle(); |
1721 | EH = {1, 2, 3, 4, 5}; |
1722 | |
1723 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1724 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1725 | inferBindings.copyToTarget("dataP" , trainingBindings); |
1726 | inferBindings.copyToTarget("indicesP" , trainingBindings); |
1727 | inferBindings.copyToTarget("weightsP" , trainingBindings); |
1728 | inferBindings.copyToTarget("lengthsP" , trainingBindings); |
1729 | inferBindings.copyToTarget("expectedP" , trainingBindings); |
1730 | inferBindings.clear(); |
1731 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
1732 | EH = trainingBindings |
1733 | .get(trainingBindings.getPlaceholderByNameSlow("expectedP" )) |
1734 | ->getHandle(); |
1735 | auto *res = |
1736 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save" )); |
1737 | |
1738 | // Train the network. |
1739 | auto *TF = glow::differentiate(F, TC); |
1740 | auto tfName = TF->getName(); |
1741 | auto fname = F->getName(); |
1742 | EET_.compile(CompilationMode::Train); |
1743 | |
1744 | const size_t numIterations = 1000; |
1745 | |
1746 | for (size_t i = 0; i < numIterations; ++i) { |
1747 | EET_.run(trainingBindings, tfName); |
1748 | } |
1749 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1750 | trainingBindings.copyToTarget("dataP" , inferBindings); |
1751 | trainingBindings.copyToTarget("indicesP" , inferBindings); |
1752 | trainingBindings.copyToTarget("weightsP" , inferBindings); |
1753 | trainingBindings.copyToTarget("lengthsP" , inferBindings); |
1754 | trainingBindings.copyToTarget("expectedP" , inferBindings); |
1755 | // Switch to inference mode and run the network. |
1756 | EEI_.compile(CompilationMode::Infer); |
1757 | EEI_.run(inferBindings, fname); |
1758 | |
1759 | // Make sure that the network output matches expectations after training. |
1760 | auto RH = res->getHandle(); |
1761 | for (size_t j = 0; j < EH.size(); ++j) { |
1762 | EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02); |
1763 | } |
1764 | } |
1765 | |
1766 | /// Simple test case that learns the weights for a |
1767 | /// SparseLengthsWeightedSum operator. |
1768 | TEST_P(MLTest, learnSparseLengthsWeightedSumWeights) { |
1769 | CHECK_IF_ENABLED(); |
1770 | TrainingConfig TC; |
1771 | TC.learningRate = 0.001; |
1772 | TC.batchSize = 1; |
1773 | |
1774 | PlaceholderBindings trainingBindings, inferBindings; |
1775 | Function *F; |
1776 | Placeholder *dataP, *indicesP, *lengthsP, *expectedP, *weightsP; |
1777 | PseudoRNG &PRNG = EET_.getModule().getPRNG(); |
1778 | for (auto *EE : engines_) { |
1779 | Module &mod = EE->getModule(); |
1780 | |
1781 | // Create a model consisting of one SparseLengthsWeightedSum operator |
1782 | // followed by a Regression node to get some non-zero gradients. |
1783 | F = mod.createFunction("SparseLengthsWeightedSum" ); |
1784 | dataP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "dataP" , |
1785 | /*isTrainable=*/false); |
1786 | indicesP = mod.createPlaceholder(ElemKind::Int64ITy, {10}, "indicesP" , |
1787 | /*isTrainable=*/false); |
1788 | weightsP = mod.createPlaceholder(ElemKind::FloatTy, {10}, "weightsP" , |
1789 | /*isTrainable=*/true); |
1790 | lengthsP = mod.createPlaceholder(ElemKind::Int32ITy, {5}, "lengthsP" , |
1791 | /*isTrainable=*/false); |
1792 | expectedP = mod.createPlaceholder(ElemKind::FloatTy, {5}, "expectedP" , |
1793 | /*isTrainable=*/false); |
1794 | |
1795 | auto *SLWS = F->createSparseLengthsWeightedSum("SLWS" , dataP, weightsP, |
1796 | indicesP, lengthsP); |
1797 | auto *reg = F->createRegression("reg" , SLWS, expectedP); |
1798 | F->createSave("save" , reg); |
1799 | } |
1800 | // Allocate and set embeddings. |
1801 | inferBindings.allocate(dataP)->getHandle() = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; |
1802 | |
1803 | // Allocate and set indices such that input embeddings are reversed. |
1804 | inferBindings.allocate(indicesP)->getHandle<int64_t>() = {9, 8, 7, 6, 5, |
1805 | 4, 3, 2, 1, 0}; |
1806 | // Allocate and randomly initialize weights. |
1807 | auto WH = inferBindings.allocate(weightsP)->getHandle(); |
1808 | WH.randomize(-1.0, 1.0, PRNG); |
1809 | |
1810 | // Allocate and set lengths. |
1811 | inferBindings.allocate(lengthsP)->getHandle<int32_t>() = {2, 2, 2, 2, 2}; |
1812 | |
1813 | // Allocate and set expected outputs. The weighs will be adjusted |
1814 | // during training so that the final result is this. |
1815 | auto EH = inferBindings.allocate(expectedP)->getHandle(); |
1816 | EH = {10, 7, 6, 3, 2}; |
1817 | |
1818 | trainingBindings.allocate(EET_.getModule().getPlaceholders()); |
1819 | inferBindings.copyTrainableWeightsTo(trainingBindings); |
1820 | inferBindings.copyToTarget("dataP" , trainingBindings); |
1821 | inferBindings.copyToTarget("indicesP" , trainingBindings); |
1822 | inferBindings.copyToTarget("weightsP" , trainingBindings); |
1823 | inferBindings.copyToTarget("lengthsP" , trainingBindings); |
1824 | inferBindings.copyToTarget("expectedP" , trainingBindings); |
1825 | inferBindings.clear(); |
1826 | inferBindings.allocate(EEI_.getModule().getPlaceholders()); |
1827 | auto *res = |
1828 | inferBindings.get(EEI_.getModule().getPlaceholderByNameSlow("save" )); |
1829 | EH = trainingBindings |
1830 | .get(trainingBindings.getPlaceholderByNameSlow("expectedP" )) |
1831 | ->getHandle(); |
1832 | // Train the network. |
1833 | auto *TF = glow::differentiate(F, TC); |
1834 | auto tfName = TF->getName(); |
1835 | auto fname = F->getName(); |
1836 | EET_.compile(CompilationMode::Train); |
1837 | |
1838 | const size_t numIterations = 1000; |
1839 | |
1840 | for (size_t i = 0; i < numIterations; ++i) { |
1841 | EET_.run(trainingBindings, tfName); |
1842 | } |
1843 | trainingBindings.copyTrainableWeightsTo(inferBindings); |
1844 | trainingBindings.copyToTarget("dataP" , inferBindings); |
1845 | trainingBindings.copyToTarget("indicesP" , inferBindings); |
1846 | trainingBindings.copyToTarget("weightsP" , inferBindings); |
1847 | trainingBindings.copyToTarget("lengthsP" , inferBindings); |
1848 | trainingBindings.copyToTarget("expectedP" , inferBindings); |
1849 | // Switch to inference mode and run the network. |
1850 | EEI_.compile(CompilationMode::Infer); |
1851 | EEI_.run(inferBindings, fname); |
1852 | |
1853 | // Make sure that the network output matches expectations after training. |
1854 | auto RH = res->getHandle(); |
1855 | for (size_t j = 0; j < EH.size(); ++j) { |
1856 | EXPECT_NEAR(RH.raw(j), EH.raw(j), 0.02); |
1857 | } |
1858 | } |
1859 | |
1860 | INSTANTIATE_BACKEND_TEST(MLTest); |
1861 | |