1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#include "BackendTestUtils.h"
17
18#include "glow/Backends/DeviceManager.h"
19#include "glow/Backends/DummyDeviceManager.h"
20#include "glow/ExecutionEngine/ExecutionEngine.h"
21#include "glow/Optimizer/GraphOptimizer/GraphOptimizer.h"
22#include "glow/Runtime/RuntimeTypes.h"
23
24#include "gtest/gtest.h"
25
26#include <chrono>
27#include <future>
28
29using namespace glow;
30using namespace glow::runtime;
31
32template <typename ResultType>
33std::pair<std::promise<ResultType>, std::future<ResultType>> getFutureHelper() {
34 std::promise<ResultType> promise;
35 auto future = promise.get_future();
36 return std::make_pair(std::move(promise), std::move(future));
37}
38
39template <typename ResultType>
40void callbackHelper(std::promise<ResultType> &promise, ResultType res,
41 Error err) {
42 promise.set_value(!ERR_TO_BOOL(std::move(err)) ? std::move(res)
43 : ResultType());
44}
45
46class DeviceManagerTest : public ::testing::TestWithParam<std::string> {
47public:
48 void SetUp() override {
49 backendName = GetParam();
50 DeviceConfig config(backendName);
51 device.reset(DeviceManager::createDeviceManager(config));
52 ASSERT_TRUE(device.get());
53 ASSERT_FALSE(ERR_TO_BOOL(device->init()));
54 }
55
56 void TearDown() override { EXPECT_FALSE(ERR_TO_BOOL(device->stop())); }
57
58 std::string backendName;
59 std::unique_ptr<DeviceManager> device{nullptr};
60
61 void addToDevice(Module *module, FunctionMapTy functions) {
62
63 std::promise<const Module *> promise;
64 std::future<const Module *> future;
65 std::tie(promise, future) = getFutureHelper<const Module *>();
66
67 device->addNetwork(module, std::move(functions),
68 [&promise](const Module *module, Error err) {
69 callbackHelper(promise, module, std::move(err));
70 });
71
72 future.wait_for(std::chrono::seconds(2));
73 EXPECT_EQ(future.get(), module);
74 }
75
76 std::unique_ptr<ExecutionContext>
77 runFunction(std::string name, std::unique_ptr<ExecutionContext> context) {
78 std::promise<std::unique_ptr<ExecutionContext>> runPromise;
79 std::future<std::unique_ptr<ExecutionContext>> runFuture;
80
81 std::tie(runPromise, runFuture) =
82 getFutureHelper<std::unique_ptr<ExecutionContext>>();
83 device->runFunction(
84 name, std::move(context),
85 [&runPromise](RunIdentifierTy, Error err,
86 std::unique_ptr<ExecutionContext> context) {
87 callbackHelper(runPromise, std::move(context), std::move(err));
88 });
89
90 runFuture.wait_for(std::chrono::seconds(2));
91 context = runFuture.get();
92 return context;
93 }
94};
95
96std::unique_ptr<Module> makeBasicModule(std::string functionName = "main") {
97 std::unique_ptr<Module> module = glow::make_unique<Module>();
98
99 Function *F = module->createFunction(functionName);
100 auto *input = module->createPlaceholder(ElemKind::FloatTy, {1},
101 functionName + "_input", false);
102 auto *output = module->createPlaceholder(ElemKind::FloatTy, {1},
103 functionName + "_output", false);
104 auto *c =
105 module->createConstant(ElemKind::FloatTy, {1}, functionName + "_const");
106 auto *t = F->createTanh("tanh", input);
107 auto *m = F->createMax("max", c, t);
108 F->createSave("ret", m, output);
109
110 c->getPayloadMutable().getHandle().clear(0.25f);
111 return module;
112}
113
114FunctionMapTy
115compileFunctions(llvm::StringRef backendName, Module *module,
116 std::vector<std::unique_ptr<CompiledFunction>> &backing) {
117 FunctionMapTy results;
118 auto *backend = createBackend(backendName);
119 CompilationContext cctx;
120 cctx.compMode = CompilationMode::Infer;
121 for (auto *F : module->getFunctions()) {
122 EXIT_ON_ERR(::glow::optimizeFunction(F, *backend, cctx));
123 auto f = EXIT_ON_ERR(backend->compile(F, cctx.backendOpts));
124 backing.push_back(std::move(f));
125 results.emplace(F->getName(), backing.back().get());
126 }
127
128 delete backend;
129 return results;
130}
131
132TEST_P(DeviceManagerTest, Basic) {
133 auto module = makeBasicModule();
134 std::vector<std::unique_ptr<CompiledFunction>> backing;
135 FunctionMapTy functions =
136 compileFunctions(backendName, module.get(), backing);
137
138 std::unique_ptr<ExecutionContext> context =
139 glow::make_unique<ExecutionContext>();
140 context->getPlaceholderBindings()->allocate(module->getPlaceholders());
141
142 addToDevice(module.get(), std::move(functions));
143
144 Tensor input1(ElemKind::FloatTy, {1});
145 Tensor output1(ElemKind::FloatTy, {1});
146 input1.getHandle().clear(0.5);
147 output1.getHandle().clear(std::max(std::tanh(0.5), 0.25));
148
149 updateInputPlaceholders(*context->getPlaceholderBindings(),
150 {module->getPlaceholderByNameSlow("main_input")},
151 {&input1});
152
153 context = runFunction("main", std::move(context));
154 ASSERT_TRUE(context);
155 // We must ensure results are on host since we're using DeviceManager
156 // directly.
157 context->getPlaceholderBindings()->ensureOnHost();
158 Tensor *result1 = context->getPlaceholderBindings()->get(
159 module->getPlaceholderByNameSlow("main_output"));
160 ASSERT_TRUE(result1);
161 EXPECT_TRUE(result1->isEqual(output1));
162}
163
164// Test that the DeviceManager correctly supports virtual padding.
165TEST_P(DeviceManagerTest, PartialTensorCopy) {
166 // Temporarily disable this test for Habana.
167 if (backendName == "Habana") {
168 return;
169 }
170 std::unique_ptr<Module> module = glow::make_unique<Module>();
171
172 // Create function of batch size 2.
173 Function *F = module->createFunction("main");
174 auto *input =
175 module->createPlaceholder(ElemKind::FloatTy, {2}, "main_input", false);
176 auto *output =
177 module->createPlaceholder(ElemKind::FloatTy, {2}, "main_output", false);
178 auto *p = F->createTanh("tanh2", input);
179 F->createSave("ret", p, output);
180
181 std::vector<std::unique_ptr<CompiledFunction>> backing;
182 FunctionMapTy functions =
183 compileFunctions(backendName, module.get(), backing);
184
185 std::promise<const Module *> promise;
186 std::future<const Module *> future;
187 std::tie(promise, future) = getFutureHelper<const Module *>();
188
189 device->addNetwork(module.get(), std::move(functions),
190 [&promise](const Module *module, Error err) {
191 callbackHelper(promise, module, std::move(err));
192 });
193
194 future.wait_for(std::chrono::seconds(2));
195 EXPECT_EQ(future.get(), module.get());
196
197 std::unique_ptr<ExecutionContext> context =
198 glow::make_unique<ExecutionContext>();
199 context->getPlaceholderBindings()->allocate(output);
200
201 Tensor input1(ElemKind::FloatTy, {1});
202 auto size = input->getType()->getSizeInBytes() / 2;
203 Tensor virtualPaddedInput(input1.getUnsafePtr(), input->getType(), size);
204
205 Tensor output1(ElemKind::FloatTy, {1});
206 input1.getHandle().clear(0.5);
207 output1.getHandle().clear(std::max(std::tanh(0.5), 0.25));
208
209 context->getPlaceholderBindings()->insert(input,
210 std::move(virtualPaddedInput));
211 std::promise<std::unique_ptr<ExecutionContext>> runPromise;
212 std::future<std::unique_ptr<ExecutionContext>> runFuture;
213
214 std::tie(runPromise, runFuture) =
215 getFutureHelper<std::unique_ptr<ExecutionContext>>();
216 device->runFunction("main", std::move(context),
217 [&runPromise](RunIdentifierTy, Error err,
218 std::unique_ptr<ExecutionContext> context) {
219 callbackHelper(runPromise, std::move(context),
220 std::move(err));
221 });
222
223 runFuture.wait_for(std::chrono::seconds(2));
224 context = runFuture.get();
225 ASSERT_TRUE(context);
226 // We must ensure results are on host since we're using DeviceManager
227 // directly.
228 context->getPlaceholderBindings()->ensureOnHost();
229
230 Tensor *result1 = context->getPlaceholderBindings()->get(
231 module->getPlaceholderByNameSlow("main_output"));
232
233 ASSERT_TRUE(result1);
234 EXPECT_FLOAT_EQ(result1->getHandle().at({0}), std::max(std::tanh(0.5), 0.25));
235}
236
237// Test that the DeviceManager correctly supports
238// transferStaticPlaceholderToDevice
239TEST_P(DeviceManagerTest, TransferStaticPlaceholderTest) {
240 CHECK_IF_ENABLED();
241 std::unique_ptr<Module> module = glow::make_unique<Module>();
242
243 Function *F = module->createFunction("main");
244 auto *input =
245 module->createPlaceholder(ElemKind::FloatTy, {1}, "input", false);
246 auto *staticPlaceholder = module->createPlaceholder(
247 ElemKind::FloatTy, {1}, "static_placeholder", false);
248 staticPlaceholder->setStatic(true);
249 auto *output =
250 module->createPlaceholder(ElemKind::FloatTy, {1}, "main_output", false);
251 auto *p = F->createPow("pow", input, staticPlaceholder);
252 F->createSave("ret", p, output);
253
254 std::vector<std::unique_ptr<CompiledFunction>> backing;
255 FunctionMapTy functions =
256 compileFunctions(backendName, module.get(), backing);
257
258 std::promise<const Module *> promise;
259 std::future<const Module *> future;
260 std::tie(promise, future) = getFutureHelper<const Module *>();
261
262 device->addNetwork(module.get(), std::move(functions),
263 [&promise](const Module *module, Error err) {
264 callbackHelper(promise, module, std::move(err));
265 });
266
267 future.wait_for(std::chrono::seconds(2));
268 EXPECT_EQ(future.get(), module.get());
269
270 auto staticTensor = Tensor(staticPlaceholder->getType());
271 staticTensor.getHandle().clear(3.0);
272 std::promise<void> transferPromise;
273 Error transferError = Error::empty();
274 auto done = transferPromise.get_future();
275
276 device->transferStaticPlaceholderToDevice(
277 staticPlaceholder, &staticTensor,
278 [&transferPromise, &transferError](Error err) {
279 transferError = std::move(err);
280 transferPromise.set_value();
281 });
282 EXPECT_FALSE(ERR_TO_BOOL(std::move(transferError)));
283 std::unique_ptr<ExecutionContext> context =
284 glow::make_unique<ExecutionContext>();
285 context->getPlaceholderBindings()->allocate(output);
286
287 Tensor input1(ElemKind::FloatTy, {1});
288
289 Tensor output1(ElemKind::FloatTy, {1});
290 input1.getHandle().clear(2.0);
291
292 context->getPlaceholderBindings()->allocate(input);
293 context->getPlaceholderBindings()->get(input)->getHandle().clear(2.0);
294 std::promise<std::unique_ptr<ExecutionContext>> runPromise;
295 std::future<std::unique_ptr<ExecutionContext>> runFuture;
296
297 std::tie(runPromise, runFuture) =
298 getFutureHelper<std::unique_ptr<ExecutionContext>>();
299 device->runFunction("main", std::move(context),
300 [&runPromise](RunIdentifierTy, Error err,
301 std::unique_ptr<ExecutionContext> context) {
302 callbackHelper(runPromise, std::move(context),
303 std::move(err));
304 });
305
306 runFuture.wait_for(std::chrono::seconds(2));
307 context = runFuture.get();
308 ASSERT_TRUE(context);
309 // We must ensure results are on host since we're using DeviceManager
310 // directly.
311 context->getPlaceholderBindings()->ensureOnHost();
312
313 Tensor *result = context->getPlaceholderBindings()->get(output);
314
315 ASSERT_TRUE(result);
316 EXPECT_NEAR(result->getHandle().at({0}), 8.0, 1E-5);
317}
318
319TEST_P(DeviceManagerTest, MultiRun) {
320 CHECK_IF_ENABLED();
321 auto module = makeBasicModule();
322 std::vector<std::unique_ptr<CompiledFunction>> backing;
323 FunctionMapTy functions =
324 compileFunctions(backendName, module.get(), backing);
325
326 addToDevice(module.get(), std::move(functions));
327
328 std::unique_ptr<ExecutionContext> context1 =
329 glow::make_unique<ExecutionContext>();
330 std::unique_ptr<ExecutionContext> context2 =
331 glow::make_unique<ExecutionContext>();
332 context1->getPlaceholderBindings()->allocate(module->getPlaceholders());
333 context2->getPlaceholderBindings()->allocate(module->getPlaceholders());
334
335 Tensor input1(ElemKind::FloatTy, {1});
336 Tensor input2(ElemKind::FloatTy, {1});
337 input1.getHandle().clear(2.0f);
338 input2.getHandle().clear(3.0f);
339
340 Tensor output1(ElemKind::FloatTy, {1});
341 Tensor output2(ElemKind::FloatTy, {1});
342 output1.getHandle().clear(std::max(std::tanh(2.0f), 0.25f));
343 output2.getHandle().clear(std::max(std::tanh(3.0f), 0.25f));
344
345 updateInputPlaceholders(*context1->getPlaceholderBindings(),
346 {module->getPlaceholderByNameSlow("main_input")},
347 {&input1});
348 updateInputPlaceholders(*context2->getPlaceholderBindings(),
349 {module->getPlaceholderByNameSlow("main_input")},
350 {&input2});
351
352 std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2;
353 std::future<std::unique_ptr<ExecutionContext>> runF1, runF2;
354 std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
355 std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
356
357 device->runFunction("main", std::move(context1),
358 [&runP1](RunIdentifierTy, Error err,
359 std::unique_ptr<ExecutionContext> context) {
360 callbackHelper(runP1, std::move(context),
361 std::move(err));
362 });
363
364 device->runFunction("main", std::move(context2),
365 [&runP2](RunIdentifierTy, Error err,
366 std::unique_ptr<ExecutionContext> context) {
367 callbackHelper(runP2, std::move(context),
368 std::move(err));
369 });
370
371 context1 = runF1.get();
372 context2 = runF2.get();
373 ASSERT_TRUE(context1);
374 ASSERT_TRUE(context2);
375 EXPECT_NE(context1, context2);
376 // We must ensure results are on host since we're using DeviceManager
377 // directly.
378 context1->getPlaceholderBindings()->ensureOnHost();
379 context2->getPlaceholderBindings()->ensureOnHost();
380
381 Tensor *result1 = context1->getPlaceholderBindings()->get(
382 module->getPlaceholderByNameSlow("main_output"));
383 Tensor *result2 = context2->getPlaceholderBindings()->get(
384 module->getPlaceholderByNameSlow("main_output"));
385 ASSERT_TRUE(result1);
386 ASSERT_TRUE(result2);
387 EXPECT_TRUE(result1->isEqual(output1));
388 EXPECT_TRUE(result2->isEqual(output2));
389}
390
391TEST_P(DeviceManagerTest, MultiFunction) {
392 CHECK_IF_ENABLED();
393
394 auto module = makeBasicModule("func1");
395
396 std::unique_ptr<ExecutionContext> context1 =
397 glow::make_unique<ExecutionContext>();
398 std::unique_ptr<ExecutionContext> context2 =
399 glow::make_unique<ExecutionContext>();
400 context1->getPlaceholderBindings()->allocate(module->getPlaceholders());
401
402 Function *F = module->createFunction("func2");
403 auto *inP = module->getPlaceholderByNameSlow("func1_input");
404 auto *outP =
405 module->createPlaceholder(ElemKind::FloatTy, {1}, "func2_output", false);
406 auto *p = F->createTanh("tanh2", inP);
407 F->createSave("ret2", p, outP);
408 // Add extra tanh and fcs to the second function, we do not care about it's
409 // output but this makes the two functions have different memory requirements.
410 auto *c = module->createConstant(ElemKind::FloatTy, {1}, "add_constant");
411 auto *sideTan = F->createTanh("tanh_extra", c);
412 auto *fc = F->createFullyConnected(*context2->getPlaceholderBindings(), "fc",
413 sideTan, 1000);
414 auto *fc2 = F->createFullyConnected(*context2->getPlaceholderBindings(),
415 "fc2", fc, 1);
416 auto res = F->createSave("side_save", fc2);
417
418 context2->getPlaceholderBindings()->allocate(inP);
419 context2->getPlaceholderBindings()->allocate(outP);
420 context2->getPlaceholderBindings()->allocate(res->getPlaceholder());
421
422 std::vector<std::unique_ptr<CompiledFunction>> backing;
423 FunctionMapTy functions =
424 compileFunctions(backendName, module.get(), backing);
425 EXPECT_EQ(functions.size(), 2);
426
427 std::promise<const Module *> promise;
428 std::future<const Module *> future;
429 std::tie(promise, future) = getFutureHelper<const Module *>();
430 device->addNetwork(module.get(), std::move(functions),
431 [&promise](const Module *module, Error err) {
432 callbackHelper(promise, module, std::move(err));
433 });
434 future.wait_for(std::chrono::seconds(2));
435 EXPECT_EQ(future.get(), module.get());
436
437 Tensor input(ElemKind::FloatTy, {1});
438 input.getHandle().clear(0.5f);
439 Tensor output1(ElemKind::FloatTy, {1});
440 output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
441 Tensor output2(ElemKind::FloatTy, {1});
442 output2.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
443
444 updateInputPlaceholders(*context1->getPlaceholderBindings(),
445 {module->getPlaceholderByNameSlow("func1_input")},
446 {&input});
447 updateInputPlaceholders(*context2->getPlaceholderBindings(),
448 {module->getPlaceholderByNameSlow("func1_input")},
449 {&input});
450
451 std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2;
452 std::future<std::unique_ptr<ExecutionContext>> runF1, runF2;
453 std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
454 std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
455
456 device->runFunction("func1", std::move(context1),
457 [&runP1](RunIdentifierTy, Error err,
458 std::unique_ptr<ExecutionContext> context) {
459 callbackHelper(runP1, std::move(context),
460 std::move(err));
461 });
462
463 device->runFunction("func2", std::move(context2),
464 [&runP2](RunIdentifierTy, Error err,
465 std::unique_ptr<ExecutionContext> context) {
466 callbackHelper(runP2, std::move(context),
467 std::move(err));
468 });
469
470 context1 = runF1.get();
471 context2 = runF2.get();
472 ASSERT_TRUE(context1);
473 ASSERT_TRUE(context2);
474 EXPECT_NE(context1, context2);
475 context1->getPlaceholderBindings()->ensureOnHost();
476 context2->getPlaceholderBindings()->ensureOnHost();
477
478 Tensor *result1 = context1->getPlaceholderBindings()->get(
479 module->getPlaceholderByNameSlow("func1_output"));
480 Tensor *result2 = context2->getPlaceholderBindings()->get(
481 module->getPlaceholderByNameSlow("func2_output"));
482 ASSERT_TRUE(result1);
483 ASSERT_TRUE(result2);
484 EXPECT_TRUE(result1->isEqual(output1));
485 EXPECT_TRUE(result2->isEqual(output2));
486}
487
488TEST_P(DeviceManagerTest, MultiModule) {
489 auto module1 = makeBasicModule("func1");
490 auto module2 = makeBasicModule("func2");
491
492 std::vector<std::unique_ptr<CompiledFunction>> backing;
493 FunctionMapTy functions1 =
494 compileFunctions(backendName, module1.get(), backing);
495 FunctionMapTy functions2 =
496 compileFunctions(backendName, module2.get(), backing);
497
498 std::promise<const Module *> promise;
499 std::future<const Module *> future;
500 std::tie(promise, future) = getFutureHelper<const Module *>();
501 device->addNetwork(module1.get(), std::move(functions1),
502 [&promise](const Module *module, Error err) {
503 callbackHelper(promise, module, std::move(err));
504 });
505 future.wait_for(std::chrono::seconds(2));
506 EXPECT_EQ(future.get(), module1.get());
507
508 std::tie(promise, future) = getFutureHelper<const Module *>();
509 device->addNetwork(module2.get(), std::move(functions2),
510 [&promise](const Module *module, Error err) {
511 callbackHelper(promise, module, std::move(err));
512 });
513 future.wait_for(std::chrono::seconds(2));
514 EXPECT_EQ(future.get(), module2.get());
515
516 std::unique_ptr<ExecutionContext> context1 =
517 glow::make_unique<ExecutionContext>();
518 context1->getPlaceholderBindings()->allocate(module1->getPlaceholders());
519 Tensor input(ElemKind::FloatTy, {1});
520 input.getHandle().clear(0.5f);
521 Tensor output(ElemKind::FloatTy, {1});
522 output.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
523
524 updateInputPlaceholders(*context1->getPlaceholderBindings(),
525 {module1->getPlaceholderByNameSlow("func1_input")},
526 {&input});
527
528 std::unique_ptr<ExecutionContext> context2 =
529 glow::make_unique<ExecutionContext>();
530 context2->getPlaceholderBindings()->allocate(module2->getPlaceholders());
531 updateInputPlaceholders(*context2->getPlaceholderBindings(),
532 {module2->getPlaceholderByNameSlow("func2_input")},
533 {&input});
534
535 std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2;
536 std::future<std::unique_ptr<ExecutionContext>> runF1, runF2;
537 std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
538 std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
539
540 device->runFunction("func1", std::move(context1),
541 [&runP1](RunIdentifierTy, Error err,
542 std::unique_ptr<ExecutionContext> context) {
543 callbackHelper(runP1, std::move(context),
544 std::move(err));
545 });
546
547 device->runFunction("func2", std::move(context2),
548 [&runP2](RunIdentifierTy, Error err,
549 std::unique_ptr<ExecutionContext> context) {
550 callbackHelper(runP2, std::move(context),
551 std::move(err));
552 });
553
554 context1 = runF1.get();
555 context2 = runF2.get();
556 ASSERT_TRUE(context1);
557 ASSERT_TRUE(context2);
558 EXPECT_NE(context1, context2);
559 context1->getPlaceholderBindings()->ensureOnHost();
560 context2->getPlaceholderBindings()->ensureOnHost();
561
562 Tensor *result1 = context1->getPlaceholderBindings()->get(
563 module1->getPlaceholderByNameSlow("func1_output"));
564 ASSERT_TRUE(result1);
565 EXPECT_TRUE(result1->isEqual(output));
566
567 Tensor *result2 = context2->getPlaceholderBindings()->get(
568 module2->getPlaceholderByNameSlow("func2_output"));
569 ASSERT_TRUE(result2);
570 EXPECT_TRUE(result2->isEqual(output));
571}
572
573TEST_P(DeviceManagerTest, ReuseModule) {
574 auto module = makeBasicModule("func1");
575
576 std::unique_ptr<ExecutionContext> context1 =
577 glow::make_unique<ExecutionContext>();
578 std::unique_ptr<ExecutionContext> context2 =
579 glow::make_unique<ExecutionContext>();
580 context1->getPlaceholderBindings()->allocate(module->getPlaceholders());
581
582 Function *F = module->createFunction("func2");
583 auto *inP = module->getPlaceholderByNameSlow("func1_input");
584 auto *outP =
585 module->createPlaceholder(ElemKind::FloatTy, {1}, "func2_output", false);
586 auto *p = F->createTanh("tanh2", inP);
587 F->createSave("ret2", p, outP);
588
589 context2->getPlaceholderBindings()->allocate(inP);
590 context2->getPlaceholderBindings()->allocate(outP);
591
592 std::vector<std::unique_ptr<CompiledFunction>> backing;
593 FunctionMapTy functions =
594 compileFunctions(backendName, module.get(), backing);
595 EXPECT_EQ(functions.size(), 2);
596
597 // Split the function map into two parts.
598 FunctionMapTy functions2;
599 functions2.emplace("func2", std::move(functions["func2"]));
600 functions.erase("func2");
601 EXPECT_EQ(functions.size(), 1);
602 EXPECT_EQ(functions2.size(), 1);
603
604 std::promise<const Module *> promise;
605 std::future<const Module *> future;
606 std::tie(promise, future) = getFutureHelper<const Module *>();
607 device->addNetwork(module.get(), std::move(functions),
608 [&promise](const Module *module, Error err) {
609 callbackHelper(promise, module, std::move(err));
610 });
611 future.wait_for(std::chrono::seconds(2));
612 EXPECT_EQ(future.get(), module.get());
613
614 std::tie(promise, future) = getFutureHelper<const Module *>();
615 device->addNetwork(module.get(), std::move(functions2),
616 [&promise](const Module *module, Error err) {
617 callbackHelper(promise, module, std::move(err));
618 });
619 future.wait_for(std::chrono::seconds(2));
620 EXPECT_EQ(future.get(), module.get());
621
622 Tensor input(ElemKind::FloatTy, {1});
623 input.getHandle().clear(0.5f);
624 Tensor output1(ElemKind::FloatTy, {1});
625 output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
626 Tensor output2(ElemKind::FloatTy, {1});
627 output2.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
628
629 updateInputPlaceholders(*context1->getPlaceholderBindings(),
630 {module->getPlaceholderByNameSlow("func1_input")},
631 {&input});
632 updateInputPlaceholders(*context2->getPlaceholderBindings(),
633 {module->getPlaceholderByNameSlow("func1_input")},
634 {&input});
635
636 std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2;
637 std::future<std::unique_ptr<ExecutionContext>> runF1, runF2;
638 std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
639 std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>();
640
641 device->runFunction("func1", std::move(context1),
642 [&runP1](RunIdentifierTy, Error err,
643 std::unique_ptr<ExecutionContext> context) {
644 callbackHelper(runP1, std::move(context),
645 std::move(err));
646 });
647
648 device->runFunction("func2", std::move(context2),
649 [&runP2](RunIdentifierTy, Error err,
650 std::unique_ptr<ExecutionContext> context) {
651 callbackHelper(runP2, std::move(context),
652 std::move(err));
653 });
654
655 context1 = runF1.get();
656 context2 = runF2.get();
657 ASSERT_TRUE(context1);
658 ASSERT_TRUE(context2);
659 EXPECT_NE(context1, context2);
660 context1->getPlaceholderBindings()->ensureOnHost();
661 context2->getPlaceholderBindings()->ensureOnHost();
662
663 Tensor *result1 = context1->getPlaceholderBindings()->get(
664 module->getPlaceholderByNameSlow("func1_output"));
665 ASSERT_TRUE(result1);
666 EXPECT_TRUE(result1->isEqual(output1));
667
668 Tensor *result2 = context2->getPlaceholderBindings()->get(
669 module->getPlaceholderByNameSlow("func2_output"));
670 ASSERT_TRUE(result2);
671 EXPECT_TRUE(result2->isEqual(output2));
672}
673
674TEST(DeviceManagerTest, SetDeviceMemory) {
675 // Test Interpreter.
676 auto interpreterConfigEmpty = DeviceConfig("Interpreter");
677 auto interpreterConfigFull = DeviceConfig("Interpreter");
678 interpreterConfigFull.setDeviceMemory(32768);
679 // Only deviceConfig setting.
680 auto interpreterDeviceSetByDeviceConfig = std::unique_ptr<DeviceManager>(
681 DeviceManager::createDeviceManager(interpreterConfigFull));
682 EXPECT_EQ(interpreterDeviceSetByDeviceConfig->getMaximumMemory(), 32768);
683 // No setting at all, default memory size.
684 auto interpreterDeviceDefault = std::unique_ptr<DeviceManager>(
685 DeviceManager::createDeviceManager(interpreterConfigEmpty));
686 EXPECT_EQ(interpreterDeviceDefault->getMaximumMemory(), 2000000000);
687}
688
689TEST(DeviceManagerTest, AvailableMemory) {
690 std::vector<std::unique_ptr<CompiledFunction>> backing;
691 std::promise<const Module *> promise;
692 std::future<const Module *> future;
693
694 auto module = makeBasicModule();
695 auto compiledFunctions = compileFunctions("CPU", module.get(), backing);
696
697 uint64_t expectedBytes{0};
698 for (const auto &f : backing) {
699 expectedBytes += f->getRuntimeBundle().getConstantWeightSize();
700 }
701
702 auto config = DeviceConfig("CPU");
703 config.setDeviceMemory(expectedBytes);
704 auto cpuCoreDevice = std::unique_ptr<DeviceManager>(
705 DeviceManager::createDeviceManager(config));
706 ASSERT_FALSE(ERR_TO_BOOL(cpuCoreDevice->init()));
707
708 EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes);
709 EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), expectedBytes);
710 EXPECT_TRUE(cpuCoreDevice->isMemoryAvailable(expectedBytes));
711 EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(expectedBytes + 1));
712
713 std::tie(promise, future) = getFutureHelper<const Module *>();
714 cpuCoreDevice->addNetwork(module.get(), compiledFunctions,
715 [&promise](const Module *module, Error err) {
716 callbackHelper(promise, module, std::move(err));
717 });
718
719 future.wait_for(std::chrono::seconds(2));
720 EXPECT_EQ(future.get(), module.get());
721
722 EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes);
723 EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0);
724 EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(expectedBytes));
725 EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(1));
726
727 // Let's try again.
728 auto module2 = makeBasicModule();
729 std::tie(promise, future) = getFutureHelper<const Module *>();
730 cpuCoreDevice->addNetwork(module2.get(),
731 compileFunctions("CPU", module2.get(), backing),
732 [&promise](const Module *module, Error err) {
733 callbackHelper(promise, module, std::move(err));
734 });
735
736 future.wait_for(std::chrono::seconds(2));
737 auto *resultModule = future.get();
738 EXPECT_NE(resultModule, module2.get());
739 EXPECT_NE(resultModule, module.get());
740 EXPECT_EQ(resultModule, nullptr);
741
742 EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes);
743 EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0);
744
745 // Evict the first network.
746 std::promise<std::string> evictPromise;
747 std::future<std::string> evictFuture;
748 std::tie(evictPromise, evictFuture) = getFutureHelper<std::string>();
749 cpuCoreDevice->evictNetwork(
750 "main", [&evictPromise](std::string functionName, Error err) {
751 callbackHelper(evictPromise, functionName, std::move(err));
752 });
753 evictFuture.wait_for(std::chrono::seconds(2));
754 EXPECT_EQ(evictFuture.get(), "main");
755
756 // And try again, this time with available space.
757 std::tie(promise, future) = getFutureHelper<const Module *>();
758 cpuCoreDevice->addNetwork(module2.get(),
759 compileFunctions("CPU", module2.get(), backing),
760 [&promise](const Module *module, Error err) {
761 callbackHelper(promise, module, std::move(err));
762 });
763
764 future.wait_for(std::chrono::seconds(2));
765 EXPECT_EQ(future.get(), module2.get());
766
767 EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes);
768 EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0);
769
770 EXPECT_FALSE(ERR_TO_BOOL(cpuCoreDevice->stop()));
771
772 // Test CPU DeviceConfig.
773 auto cpuConfigEmpty = DeviceConfig("CPU");
774 auto cpuConfigFull = DeviceConfig("CPU");
775 cpuConfigFull.setDeviceMemory(32768);
776 // Only deviceConfig setting.
777 auto cpuDeviceSetByDeviceConfig = std::unique_ptr<DeviceManager>(
778 DeviceManager::createDeviceManager(cpuConfigFull));
779 EXPECT_EQ(cpuDeviceSetByDeviceConfig->getMaximumMemory(), 32768);
780 // No setting at all, default memory size.
781 auto cpuDeviceDefault = std::unique_ptr<DeviceManager>(
782 DeviceManager::createDeviceManager(cpuConfigEmpty));
783 EXPECT_EQ(cpuDeviceDefault->getMaximumMemory(), 2000000000);
784}
785
786TEST(DeviceManagerTest, DummyDeviceManager) {
787 DummyDeviceManager deviceManager{DeviceConfig("Interpreter")};
788 ASSERT_FALSE(ERR_TO_BOOL(deviceManager.init()));
789
790 auto module = makeBasicModule();
791 std::vector<std::unique_ptr<CompiledFunction>> backing;
792 FunctionMapTy functions =
793 compileFunctions("Interpreter", module.get(), backing);
794
795 std::promise<const Module *> promise;
796 std::future<const Module *> future;
797 std::tie(promise, future) = getFutureHelper<const Module *>();
798 deviceManager.addNetwork(module.get(), std::move(functions),
799 [&promise](const Module *module, Error err) {
800 callbackHelper(promise, module, std::move(err));
801 });
802 // no need to wait.
803 EXPECT_EQ(future.get(), module.get());
804
805 std::unique_ptr<ExecutionContext> context1 =
806 glow::make_unique<ExecutionContext>();
807 context1->getPlaceholderBindings()->allocate(module->getPlaceholders());
808
809 Tensor input1(ElemKind::FloatTy, {1});
810 Tensor output1(ElemKind::FloatTy, {1});
811 input1.getHandle().clear(0.5f);
812 output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f));
813
814 updateInputPlaceholders(*context1->getPlaceholderBindings(),
815 {module->getPlaceholderByNameSlow("main_input")},
816 {&input1});
817
818 std::promise<std::unique_ptr<ExecutionContext>> runPromise;
819 std::future<std::unique_ptr<ExecutionContext>> runFuture;
820
821 std::tie(runPromise, runFuture) =
822 getFutureHelper<std::unique_ptr<ExecutionContext>>();
823 deviceManager.runFunction(
824 "main", std::move(context1),
825 [&runPromise](RunIdentifierTy, Error err,
826 std::unique_ptr<ExecutionContext> context) {
827 callbackHelper(runPromise, std::move(context), std::move(err));
828 });
829
830 runFuture.wait_for(std::chrono::seconds(2));
831 std::unique_ptr<ExecutionContext> context2 = runFuture.get();
832
833 ASSERT_TRUE(context2);
834
835 Tensor *result = context2->getPlaceholderBindings()->get(
836 module->getPlaceholderByNameSlow("main_output"));
837 ASSERT_TRUE(result);
838 EXPECT_TRUE(result->isEqual(output1));
839
840 EXPECT_FALSE(ERR_TO_BOOL(deviceManager.stop()));
841}
842
843/// Check that the device can move data to and from the host.
844/// Disable if your device does not support Device Resident Tensors.
845TEST_P(DeviceManagerTest, DeviceResidentTensors) {
846 CHECK_IF_ENABLED();
847 Tensor T = {1.2f, 12.1f, 51.0f, 1515.2f};
848 Tensor R = {1.2f, 12.1f, 51.0f, 1515.2f};
849
850 ASSERT_FALSE(T.isDeviceResident());
851
852 device->transferToDevice(T, nullptr);
853
854 ASSERT_TRUE(T.isDeviceResident());
855
856 device->transferFromDevice(T);
857
858 ASSERT_FALSE(T.isDeviceResident());
859
860 ASSERT_TRUE(T.isEqual(R));
861}
862
863/// A mock DeviceManager for use in Device Resident Tensor tests.
864class MockDM : public DeviceManager {
865public:
866 MockDM() : DeviceManager(DeviceConfig("MockDM")) {}
867 void addNetwork(const Module *module, FunctionMapTy functions,
868 ReadyCBTy readyCB) override {}
869
870 void evictNetwork(
871 std::string functionName,
872 EvictFunctionCBTy evictCB = [](std::string, Error) {}) override {}
873
874 runtime::RunIdentifierTy
875 runFunction(std::string functionName,
876 std::unique_ptr<ExecutionContext> context,
877 runtime::ResultCBTy resultCB) override {
878 return 0;
879 }
880
881 uint64_t getMaximumMemory() const override { return 0; }
882
883 uint64_t getAvailableMemory() const override { return 0; }
884
885 bool isMemoryAvailable(uint64_t estimate) const override { return 0; }
886
887 void transferToDevice(
888 Tensor &tensor, void *locationContext = nullptr,
889 std::function<void(Error)> resultCB = [](Error) {}) override {
890 if (locationContext == nullptr) {
891 locationContext = malloc(tensor.getSizeInBytes());
892 }
893 memcpy(locationContext, tensor.getUnsafePtr(), tensor.getSizeInBytes());
894 tensor.moveToDevice(this, locationContext);
895 }
896
897 void transferFromDevice(
898 Tensor &tensor, bool release = true,
899 std::function<void(Error)> resultCB = [](Error) {}) override {
900 memcpy(tensor.getUnsafePtr(), tensor.getLocationContext(),
901 tensor.getSizeInBytes());
902 free(tensor.getLocationContext());
903 tensor.clearDeviceResidency();
904 }
905
906 bool releaseDeviceTensor(void *locationContext) override { return true; }
907};
908
909TEST_P(DeviceManagerTest, CanHandleDeviceResidentTensors) {
910 CHECK_IF_ENABLED();
911
912 MockDM mockDM;
913
914 auto module = makeBasicModule();
915 std::vector<std::unique_ptr<CompiledFunction>> backing;
916 FunctionMapTy functions =
917 compileFunctions(backendName, module.get(), backing);
918
919 addToDevice(module.get(), std::move(functions));
920
921 std::unique_ptr<ExecutionContext> context =
922 glow::make_unique<ExecutionContext>();
923 context->getPlaceholderBindings()->allocate(module->getPlaceholders());
924
925 Tensor input1(ElemKind::FloatTy, {1});
926 Tensor output1(ElemKind::FloatTy, {1});
927 input1.getHandle().clear(0.5);
928 output1.getHandle().clear(std::max(std::tanh(0.5), 0.25));
929
930 updateInputPlaceholders(*context->getPlaceholderBindings(),
931 {module->getPlaceholderByNameSlow("main_input")},
932 {&input1});
933
934 mockDM.transferToDevice(*context->getPlaceholderBindings()->get(
935 module->getPlaceholderByNameSlow("main_input")));
936
937 context = runFunction("main", std::move(context));
938 ASSERT_TRUE(context);
939 Tensor *result1 = context->getPlaceholderBindings()->get(
940 module->getPlaceholderByNameSlow("main_output"));
941 ASSERT_TRUE(result1);
942}
943
944TEST_P(DeviceManagerTest, TensorCopyRawToDevice) {
945 MockDM mockDM;
946
947 Tensor input1(ElemKind::FloatTy, {10});
948 Tensor input2(ElemKind::FloatTy, {10});
949
950 input1.getHandle().clear(1);
951 input2.getHandle().clear(2);
952
953 float *deviceMemory = (float *)malloc(sizeof(float) * 10);
954 mockDM.transferToDevice(input1, deviceMemory);
955
956 for (int i = 0; i < 10; ++i) {
957 EXPECT_EQ(deviceMemory[i], 1);
958 }
959
960 input1.copyRawToDevice(&input2);
961
962 for (int i = 0; i < 10; ++i) {
963 EXPECT_EQ(deviceMemory[i], 2);
964 }
965
966 mockDM.transferFromDevice(input1);
967 auto inputHandle = input1.getHandle();
968 for (unsigned i = 0; i < 10; ++i) {
969 EXPECT_EQ(inputHandle.at({i}), 2);
970 }
971}
972
973INSTANTIATE_BACKEND_TEST(DeviceManagerTest);
974