1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #include "BackendTestUtils.h" |
17 | |
18 | #include "glow/Backends/DeviceManager.h" |
19 | #include "glow/Backends/DummyDeviceManager.h" |
20 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
21 | #include "glow/Optimizer/GraphOptimizer/GraphOptimizer.h" |
22 | #include "glow/Runtime/RuntimeTypes.h" |
23 | |
24 | #include "gtest/gtest.h" |
25 | |
26 | #include <chrono> |
27 | #include <future> |
28 | |
29 | using namespace glow; |
30 | using namespace glow::runtime; |
31 | |
32 | template <typename ResultType> |
33 | std::pair<std::promise<ResultType>, std::future<ResultType>> getFutureHelper() { |
34 | std::promise<ResultType> promise; |
35 | auto future = promise.get_future(); |
36 | return std::make_pair(std::move(promise), std::move(future)); |
37 | } |
38 | |
39 | template <typename ResultType> |
40 | void callbackHelper(std::promise<ResultType> &promise, ResultType res, |
41 | Error err) { |
42 | promise.set_value(!ERR_TO_BOOL(std::move(err)) ? std::move(res) |
43 | : ResultType()); |
44 | } |
45 | |
46 | class DeviceManagerTest : public ::testing::TestWithParam<std::string> { |
47 | public: |
48 | void SetUp() override { |
49 | backendName = GetParam(); |
50 | DeviceConfig config(backendName); |
51 | device.reset(DeviceManager::createDeviceManager(config)); |
52 | ASSERT_TRUE(device.get()); |
53 | ASSERT_FALSE(ERR_TO_BOOL(device->init())); |
54 | } |
55 | |
56 | void TearDown() override { EXPECT_FALSE(ERR_TO_BOOL(device->stop())); } |
57 | |
58 | std::string backendName; |
59 | std::unique_ptr<DeviceManager> device{nullptr}; |
60 | |
61 | void addToDevice(Module *module, FunctionMapTy functions) { |
62 | |
63 | std::promise<const Module *> promise; |
64 | std::future<const Module *> future; |
65 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
66 | |
67 | device->addNetwork(module, std::move(functions), |
68 | [&promise](const Module *module, Error err) { |
69 | callbackHelper(promise, module, std::move(err)); |
70 | }); |
71 | |
72 | future.wait_for(std::chrono::seconds(2)); |
73 | EXPECT_EQ(future.get(), module); |
74 | } |
75 | |
76 | std::unique_ptr<ExecutionContext> |
77 | runFunction(std::string name, std::unique_ptr<ExecutionContext> context) { |
78 | std::promise<std::unique_ptr<ExecutionContext>> runPromise; |
79 | std::future<std::unique_ptr<ExecutionContext>> runFuture; |
80 | |
81 | std::tie(runPromise, runFuture) = |
82 | getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
83 | device->runFunction( |
84 | name, std::move(context), |
85 | [&runPromise](RunIdentifierTy, Error err, |
86 | std::unique_ptr<ExecutionContext> context) { |
87 | callbackHelper(runPromise, std::move(context), std::move(err)); |
88 | }); |
89 | |
90 | runFuture.wait_for(std::chrono::seconds(2)); |
91 | context = runFuture.get(); |
92 | return context; |
93 | } |
94 | }; |
95 | |
96 | std::unique_ptr<Module> makeBasicModule(std::string functionName = "main" ) { |
97 | std::unique_ptr<Module> module = glow::make_unique<Module>(); |
98 | |
99 | Function *F = module->createFunction(functionName); |
100 | auto *input = module->createPlaceholder(ElemKind::FloatTy, {1}, |
101 | functionName + "_input" , false); |
102 | auto *output = module->createPlaceholder(ElemKind::FloatTy, {1}, |
103 | functionName + "_output" , false); |
104 | auto *c = |
105 | module->createConstant(ElemKind::FloatTy, {1}, functionName + "_const" ); |
106 | auto *t = F->createTanh("tanh" , input); |
107 | auto *m = F->createMax("max" , c, t); |
108 | F->createSave("ret" , m, output); |
109 | |
110 | c->getPayloadMutable().getHandle().clear(0.25f); |
111 | return module; |
112 | } |
113 | |
114 | FunctionMapTy |
115 | compileFunctions(llvm::StringRef backendName, Module *module, |
116 | std::vector<std::unique_ptr<CompiledFunction>> &backing) { |
117 | FunctionMapTy results; |
118 | auto *backend = createBackend(backendName); |
119 | CompilationContext cctx; |
120 | cctx.compMode = CompilationMode::Infer; |
121 | for (auto *F : module->getFunctions()) { |
122 | EXIT_ON_ERR(::glow::optimizeFunction(F, *backend, cctx)); |
123 | auto f = EXIT_ON_ERR(backend->compile(F, cctx.backendOpts)); |
124 | backing.push_back(std::move(f)); |
125 | results.emplace(F->getName(), backing.back().get()); |
126 | } |
127 | |
128 | delete backend; |
129 | return results; |
130 | } |
131 | |
132 | TEST_P(DeviceManagerTest, Basic) { |
133 | auto module = makeBasicModule(); |
134 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
135 | FunctionMapTy functions = |
136 | compileFunctions(backendName, module.get(), backing); |
137 | |
138 | std::unique_ptr<ExecutionContext> context = |
139 | glow::make_unique<ExecutionContext>(); |
140 | context->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
141 | |
142 | addToDevice(module.get(), std::move(functions)); |
143 | |
144 | Tensor input1(ElemKind::FloatTy, {1}); |
145 | Tensor output1(ElemKind::FloatTy, {1}); |
146 | input1.getHandle().clear(0.5); |
147 | output1.getHandle().clear(std::max(std::tanh(0.5), 0.25)); |
148 | |
149 | updateInputPlaceholders(*context->getPlaceholderBindings(), |
150 | {module->getPlaceholderByNameSlow("main_input" )}, |
151 | {&input1}); |
152 | |
153 | context = runFunction("main" , std::move(context)); |
154 | ASSERT_TRUE(context); |
155 | // We must ensure results are on host since we're using DeviceManager |
156 | // directly. |
157 | context->getPlaceholderBindings()->ensureOnHost(); |
158 | Tensor *result1 = context->getPlaceholderBindings()->get( |
159 | module->getPlaceholderByNameSlow("main_output" )); |
160 | ASSERT_TRUE(result1); |
161 | EXPECT_TRUE(result1->isEqual(output1)); |
162 | } |
163 | |
164 | // Test that the DeviceManager correctly supports virtual padding. |
165 | TEST_P(DeviceManagerTest, PartialTensorCopy) { |
166 | // Temporarily disable this test for Habana. |
167 | if (backendName == "Habana" ) { |
168 | return; |
169 | } |
170 | std::unique_ptr<Module> module = glow::make_unique<Module>(); |
171 | |
172 | // Create function of batch size 2. |
173 | Function *F = module->createFunction("main" ); |
174 | auto *input = |
175 | module->createPlaceholder(ElemKind::FloatTy, {2}, "main_input" , false); |
176 | auto *output = |
177 | module->createPlaceholder(ElemKind::FloatTy, {2}, "main_output" , false); |
178 | auto *p = F->createTanh("tanh2" , input); |
179 | F->createSave("ret" , p, output); |
180 | |
181 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
182 | FunctionMapTy functions = |
183 | compileFunctions(backendName, module.get(), backing); |
184 | |
185 | std::promise<const Module *> promise; |
186 | std::future<const Module *> future; |
187 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
188 | |
189 | device->addNetwork(module.get(), std::move(functions), |
190 | [&promise](const Module *module, Error err) { |
191 | callbackHelper(promise, module, std::move(err)); |
192 | }); |
193 | |
194 | future.wait_for(std::chrono::seconds(2)); |
195 | EXPECT_EQ(future.get(), module.get()); |
196 | |
197 | std::unique_ptr<ExecutionContext> context = |
198 | glow::make_unique<ExecutionContext>(); |
199 | context->getPlaceholderBindings()->allocate(output); |
200 | |
201 | Tensor input1(ElemKind::FloatTy, {1}); |
202 | auto size = input->getType()->getSizeInBytes() / 2; |
203 | Tensor virtualPaddedInput(input1.getUnsafePtr(), input->getType(), size); |
204 | |
205 | Tensor output1(ElemKind::FloatTy, {1}); |
206 | input1.getHandle().clear(0.5); |
207 | output1.getHandle().clear(std::max(std::tanh(0.5), 0.25)); |
208 | |
209 | context->getPlaceholderBindings()->insert(input, |
210 | std::move(virtualPaddedInput)); |
211 | std::promise<std::unique_ptr<ExecutionContext>> runPromise; |
212 | std::future<std::unique_ptr<ExecutionContext>> runFuture; |
213 | |
214 | std::tie(runPromise, runFuture) = |
215 | getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
216 | device->runFunction("main" , std::move(context), |
217 | [&runPromise](RunIdentifierTy, Error err, |
218 | std::unique_ptr<ExecutionContext> context) { |
219 | callbackHelper(runPromise, std::move(context), |
220 | std::move(err)); |
221 | }); |
222 | |
223 | runFuture.wait_for(std::chrono::seconds(2)); |
224 | context = runFuture.get(); |
225 | ASSERT_TRUE(context); |
226 | // We must ensure results are on host since we're using DeviceManager |
227 | // directly. |
228 | context->getPlaceholderBindings()->ensureOnHost(); |
229 | |
230 | Tensor *result1 = context->getPlaceholderBindings()->get( |
231 | module->getPlaceholderByNameSlow("main_output" )); |
232 | |
233 | ASSERT_TRUE(result1); |
234 | EXPECT_FLOAT_EQ(result1->getHandle().at({0}), std::max(std::tanh(0.5), 0.25)); |
235 | } |
236 | |
237 | // Test that the DeviceManager correctly supports |
238 | // transferStaticPlaceholderToDevice |
239 | TEST_P(DeviceManagerTest, TransferStaticPlaceholderTest) { |
240 | CHECK_IF_ENABLED(); |
241 | std::unique_ptr<Module> module = glow::make_unique<Module>(); |
242 | |
243 | Function *F = module->createFunction("main" ); |
244 | auto *input = |
245 | module->createPlaceholder(ElemKind::FloatTy, {1}, "input" , false); |
246 | auto *staticPlaceholder = module->createPlaceholder( |
247 | ElemKind::FloatTy, {1}, "static_placeholder" , false); |
248 | staticPlaceholder->setStatic(true); |
249 | auto *output = |
250 | module->createPlaceholder(ElemKind::FloatTy, {1}, "main_output" , false); |
251 | auto *p = F->createPow("pow" , input, staticPlaceholder); |
252 | F->createSave("ret" , p, output); |
253 | |
254 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
255 | FunctionMapTy functions = |
256 | compileFunctions(backendName, module.get(), backing); |
257 | |
258 | std::promise<const Module *> promise; |
259 | std::future<const Module *> future; |
260 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
261 | |
262 | device->addNetwork(module.get(), std::move(functions), |
263 | [&promise](const Module *module, Error err) { |
264 | callbackHelper(promise, module, std::move(err)); |
265 | }); |
266 | |
267 | future.wait_for(std::chrono::seconds(2)); |
268 | EXPECT_EQ(future.get(), module.get()); |
269 | |
270 | auto staticTensor = Tensor(staticPlaceholder->getType()); |
271 | staticTensor.getHandle().clear(3.0); |
272 | std::promise<void> transferPromise; |
273 | Error transferError = Error::empty(); |
274 | auto done = transferPromise.get_future(); |
275 | |
276 | device->transferStaticPlaceholderToDevice( |
277 | staticPlaceholder, &staticTensor, |
278 | [&transferPromise, &transferError](Error err) { |
279 | transferError = std::move(err); |
280 | transferPromise.set_value(); |
281 | }); |
282 | EXPECT_FALSE(ERR_TO_BOOL(std::move(transferError))); |
283 | std::unique_ptr<ExecutionContext> context = |
284 | glow::make_unique<ExecutionContext>(); |
285 | context->getPlaceholderBindings()->allocate(output); |
286 | |
287 | Tensor input1(ElemKind::FloatTy, {1}); |
288 | |
289 | Tensor output1(ElemKind::FloatTy, {1}); |
290 | input1.getHandle().clear(2.0); |
291 | |
292 | context->getPlaceholderBindings()->allocate(input); |
293 | context->getPlaceholderBindings()->get(input)->getHandle().clear(2.0); |
294 | std::promise<std::unique_ptr<ExecutionContext>> runPromise; |
295 | std::future<std::unique_ptr<ExecutionContext>> runFuture; |
296 | |
297 | std::tie(runPromise, runFuture) = |
298 | getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
299 | device->runFunction("main" , std::move(context), |
300 | [&runPromise](RunIdentifierTy, Error err, |
301 | std::unique_ptr<ExecutionContext> context) { |
302 | callbackHelper(runPromise, std::move(context), |
303 | std::move(err)); |
304 | }); |
305 | |
306 | runFuture.wait_for(std::chrono::seconds(2)); |
307 | context = runFuture.get(); |
308 | ASSERT_TRUE(context); |
309 | // We must ensure results are on host since we're using DeviceManager |
310 | // directly. |
311 | context->getPlaceholderBindings()->ensureOnHost(); |
312 | |
313 | Tensor *result = context->getPlaceholderBindings()->get(output); |
314 | |
315 | ASSERT_TRUE(result); |
316 | EXPECT_NEAR(result->getHandle().at({0}), 8.0, 1E-5); |
317 | } |
318 | |
319 | TEST_P(DeviceManagerTest, MultiRun) { |
320 | CHECK_IF_ENABLED(); |
321 | auto module = makeBasicModule(); |
322 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
323 | FunctionMapTy functions = |
324 | compileFunctions(backendName, module.get(), backing); |
325 | |
326 | addToDevice(module.get(), std::move(functions)); |
327 | |
328 | std::unique_ptr<ExecutionContext> context1 = |
329 | glow::make_unique<ExecutionContext>(); |
330 | std::unique_ptr<ExecutionContext> context2 = |
331 | glow::make_unique<ExecutionContext>(); |
332 | context1->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
333 | context2->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
334 | |
335 | Tensor input1(ElemKind::FloatTy, {1}); |
336 | Tensor input2(ElemKind::FloatTy, {1}); |
337 | input1.getHandle().clear(2.0f); |
338 | input2.getHandle().clear(3.0f); |
339 | |
340 | Tensor output1(ElemKind::FloatTy, {1}); |
341 | Tensor output2(ElemKind::FloatTy, {1}); |
342 | output1.getHandle().clear(std::max(std::tanh(2.0f), 0.25f)); |
343 | output2.getHandle().clear(std::max(std::tanh(3.0f), 0.25f)); |
344 | |
345 | updateInputPlaceholders(*context1->getPlaceholderBindings(), |
346 | {module->getPlaceholderByNameSlow("main_input" )}, |
347 | {&input1}); |
348 | updateInputPlaceholders(*context2->getPlaceholderBindings(), |
349 | {module->getPlaceholderByNameSlow("main_input" )}, |
350 | {&input2}); |
351 | |
352 | std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2; |
353 | std::future<std::unique_ptr<ExecutionContext>> runF1, runF2; |
354 | std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
355 | std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
356 | |
357 | device->runFunction("main" , std::move(context1), |
358 | [&runP1](RunIdentifierTy, Error err, |
359 | std::unique_ptr<ExecutionContext> context) { |
360 | callbackHelper(runP1, std::move(context), |
361 | std::move(err)); |
362 | }); |
363 | |
364 | device->runFunction("main" , std::move(context2), |
365 | [&runP2](RunIdentifierTy, Error err, |
366 | std::unique_ptr<ExecutionContext> context) { |
367 | callbackHelper(runP2, std::move(context), |
368 | std::move(err)); |
369 | }); |
370 | |
371 | context1 = runF1.get(); |
372 | context2 = runF2.get(); |
373 | ASSERT_TRUE(context1); |
374 | ASSERT_TRUE(context2); |
375 | EXPECT_NE(context1, context2); |
376 | // We must ensure results are on host since we're using DeviceManager |
377 | // directly. |
378 | context1->getPlaceholderBindings()->ensureOnHost(); |
379 | context2->getPlaceholderBindings()->ensureOnHost(); |
380 | |
381 | Tensor *result1 = context1->getPlaceholderBindings()->get( |
382 | module->getPlaceholderByNameSlow("main_output" )); |
383 | Tensor *result2 = context2->getPlaceholderBindings()->get( |
384 | module->getPlaceholderByNameSlow("main_output" )); |
385 | ASSERT_TRUE(result1); |
386 | ASSERT_TRUE(result2); |
387 | EXPECT_TRUE(result1->isEqual(output1)); |
388 | EXPECT_TRUE(result2->isEqual(output2)); |
389 | } |
390 | |
391 | TEST_P(DeviceManagerTest, MultiFunction) { |
392 | CHECK_IF_ENABLED(); |
393 | |
394 | auto module = makeBasicModule("func1" ); |
395 | |
396 | std::unique_ptr<ExecutionContext> context1 = |
397 | glow::make_unique<ExecutionContext>(); |
398 | std::unique_ptr<ExecutionContext> context2 = |
399 | glow::make_unique<ExecutionContext>(); |
400 | context1->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
401 | |
402 | Function *F = module->createFunction("func2" ); |
403 | auto *inP = module->getPlaceholderByNameSlow("func1_input" ); |
404 | auto *outP = |
405 | module->createPlaceholder(ElemKind::FloatTy, {1}, "func2_output" , false); |
406 | auto *p = F->createTanh("tanh2" , inP); |
407 | F->createSave("ret2" , p, outP); |
408 | // Add extra tanh and fcs to the second function, we do not care about it's |
409 | // output but this makes the two functions have different memory requirements. |
410 | auto *c = module->createConstant(ElemKind::FloatTy, {1}, "add_constant" ); |
411 | auto *sideTan = F->createTanh("tanh_extra" , c); |
412 | auto *fc = F->createFullyConnected(*context2->getPlaceholderBindings(), "fc" , |
413 | sideTan, 1000); |
414 | auto *fc2 = F->createFullyConnected(*context2->getPlaceholderBindings(), |
415 | "fc2" , fc, 1); |
416 | auto res = F->createSave("side_save" , fc2); |
417 | |
418 | context2->getPlaceholderBindings()->allocate(inP); |
419 | context2->getPlaceholderBindings()->allocate(outP); |
420 | context2->getPlaceholderBindings()->allocate(res->getPlaceholder()); |
421 | |
422 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
423 | FunctionMapTy functions = |
424 | compileFunctions(backendName, module.get(), backing); |
425 | EXPECT_EQ(functions.size(), 2); |
426 | |
427 | std::promise<const Module *> promise; |
428 | std::future<const Module *> future; |
429 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
430 | device->addNetwork(module.get(), std::move(functions), |
431 | [&promise](const Module *module, Error err) { |
432 | callbackHelper(promise, module, std::move(err)); |
433 | }); |
434 | future.wait_for(std::chrono::seconds(2)); |
435 | EXPECT_EQ(future.get(), module.get()); |
436 | |
437 | Tensor input(ElemKind::FloatTy, {1}); |
438 | input.getHandle().clear(0.5f); |
439 | Tensor output1(ElemKind::FloatTy, {1}); |
440 | output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
441 | Tensor output2(ElemKind::FloatTy, {1}); |
442 | output2.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
443 | |
444 | updateInputPlaceholders(*context1->getPlaceholderBindings(), |
445 | {module->getPlaceholderByNameSlow("func1_input" )}, |
446 | {&input}); |
447 | updateInputPlaceholders(*context2->getPlaceholderBindings(), |
448 | {module->getPlaceholderByNameSlow("func1_input" )}, |
449 | {&input}); |
450 | |
451 | std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2; |
452 | std::future<std::unique_ptr<ExecutionContext>> runF1, runF2; |
453 | std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
454 | std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
455 | |
456 | device->runFunction("func1" , std::move(context1), |
457 | [&runP1](RunIdentifierTy, Error err, |
458 | std::unique_ptr<ExecutionContext> context) { |
459 | callbackHelper(runP1, std::move(context), |
460 | std::move(err)); |
461 | }); |
462 | |
463 | device->runFunction("func2" , std::move(context2), |
464 | [&runP2](RunIdentifierTy, Error err, |
465 | std::unique_ptr<ExecutionContext> context) { |
466 | callbackHelper(runP2, std::move(context), |
467 | std::move(err)); |
468 | }); |
469 | |
470 | context1 = runF1.get(); |
471 | context2 = runF2.get(); |
472 | ASSERT_TRUE(context1); |
473 | ASSERT_TRUE(context2); |
474 | EXPECT_NE(context1, context2); |
475 | context1->getPlaceholderBindings()->ensureOnHost(); |
476 | context2->getPlaceholderBindings()->ensureOnHost(); |
477 | |
478 | Tensor *result1 = context1->getPlaceholderBindings()->get( |
479 | module->getPlaceholderByNameSlow("func1_output" )); |
480 | Tensor *result2 = context2->getPlaceholderBindings()->get( |
481 | module->getPlaceholderByNameSlow("func2_output" )); |
482 | ASSERT_TRUE(result1); |
483 | ASSERT_TRUE(result2); |
484 | EXPECT_TRUE(result1->isEqual(output1)); |
485 | EXPECT_TRUE(result2->isEqual(output2)); |
486 | } |
487 | |
488 | TEST_P(DeviceManagerTest, MultiModule) { |
489 | auto module1 = makeBasicModule("func1" ); |
490 | auto module2 = makeBasicModule("func2" ); |
491 | |
492 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
493 | FunctionMapTy functions1 = |
494 | compileFunctions(backendName, module1.get(), backing); |
495 | FunctionMapTy functions2 = |
496 | compileFunctions(backendName, module2.get(), backing); |
497 | |
498 | std::promise<const Module *> promise; |
499 | std::future<const Module *> future; |
500 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
501 | device->addNetwork(module1.get(), std::move(functions1), |
502 | [&promise](const Module *module, Error err) { |
503 | callbackHelper(promise, module, std::move(err)); |
504 | }); |
505 | future.wait_for(std::chrono::seconds(2)); |
506 | EXPECT_EQ(future.get(), module1.get()); |
507 | |
508 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
509 | device->addNetwork(module2.get(), std::move(functions2), |
510 | [&promise](const Module *module, Error err) { |
511 | callbackHelper(promise, module, std::move(err)); |
512 | }); |
513 | future.wait_for(std::chrono::seconds(2)); |
514 | EXPECT_EQ(future.get(), module2.get()); |
515 | |
516 | std::unique_ptr<ExecutionContext> context1 = |
517 | glow::make_unique<ExecutionContext>(); |
518 | context1->getPlaceholderBindings()->allocate(module1->getPlaceholders()); |
519 | Tensor input(ElemKind::FloatTy, {1}); |
520 | input.getHandle().clear(0.5f); |
521 | Tensor output(ElemKind::FloatTy, {1}); |
522 | output.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
523 | |
524 | updateInputPlaceholders(*context1->getPlaceholderBindings(), |
525 | {module1->getPlaceholderByNameSlow("func1_input" )}, |
526 | {&input}); |
527 | |
528 | std::unique_ptr<ExecutionContext> context2 = |
529 | glow::make_unique<ExecutionContext>(); |
530 | context2->getPlaceholderBindings()->allocate(module2->getPlaceholders()); |
531 | updateInputPlaceholders(*context2->getPlaceholderBindings(), |
532 | {module2->getPlaceholderByNameSlow("func2_input" )}, |
533 | {&input}); |
534 | |
535 | std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2; |
536 | std::future<std::unique_ptr<ExecutionContext>> runF1, runF2; |
537 | std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
538 | std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
539 | |
540 | device->runFunction("func1" , std::move(context1), |
541 | [&runP1](RunIdentifierTy, Error err, |
542 | std::unique_ptr<ExecutionContext> context) { |
543 | callbackHelper(runP1, std::move(context), |
544 | std::move(err)); |
545 | }); |
546 | |
547 | device->runFunction("func2" , std::move(context2), |
548 | [&runP2](RunIdentifierTy, Error err, |
549 | std::unique_ptr<ExecutionContext> context) { |
550 | callbackHelper(runP2, std::move(context), |
551 | std::move(err)); |
552 | }); |
553 | |
554 | context1 = runF1.get(); |
555 | context2 = runF2.get(); |
556 | ASSERT_TRUE(context1); |
557 | ASSERT_TRUE(context2); |
558 | EXPECT_NE(context1, context2); |
559 | context1->getPlaceholderBindings()->ensureOnHost(); |
560 | context2->getPlaceholderBindings()->ensureOnHost(); |
561 | |
562 | Tensor *result1 = context1->getPlaceholderBindings()->get( |
563 | module1->getPlaceholderByNameSlow("func1_output" )); |
564 | ASSERT_TRUE(result1); |
565 | EXPECT_TRUE(result1->isEqual(output)); |
566 | |
567 | Tensor *result2 = context2->getPlaceholderBindings()->get( |
568 | module2->getPlaceholderByNameSlow("func2_output" )); |
569 | ASSERT_TRUE(result2); |
570 | EXPECT_TRUE(result2->isEqual(output)); |
571 | } |
572 | |
573 | TEST_P(DeviceManagerTest, ReuseModule) { |
574 | auto module = makeBasicModule("func1" ); |
575 | |
576 | std::unique_ptr<ExecutionContext> context1 = |
577 | glow::make_unique<ExecutionContext>(); |
578 | std::unique_ptr<ExecutionContext> context2 = |
579 | glow::make_unique<ExecutionContext>(); |
580 | context1->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
581 | |
582 | Function *F = module->createFunction("func2" ); |
583 | auto *inP = module->getPlaceholderByNameSlow("func1_input" ); |
584 | auto *outP = |
585 | module->createPlaceholder(ElemKind::FloatTy, {1}, "func2_output" , false); |
586 | auto *p = F->createTanh("tanh2" , inP); |
587 | F->createSave("ret2" , p, outP); |
588 | |
589 | context2->getPlaceholderBindings()->allocate(inP); |
590 | context2->getPlaceholderBindings()->allocate(outP); |
591 | |
592 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
593 | FunctionMapTy functions = |
594 | compileFunctions(backendName, module.get(), backing); |
595 | EXPECT_EQ(functions.size(), 2); |
596 | |
597 | // Split the function map into two parts. |
598 | FunctionMapTy functions2; |
599 | functions2.emplace("func2" , std::move(functions["func2" ])); |
600 | functions.erase("func2" ); |
601 | EXPECT_EQ(functions.size(), 1); |
602 | EXPECT_EQ(functions2.size(), 1); |
603 | |
604 | std::promise<const Module *> promise; |
605 | std::future<const Module *> future; |
606 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
607 | device->addNetwork(module.get(), std::move(functions), |
608 | [&promise](const Module *module, Error err) { |
609 | callbackHelper(promise, module, std::move(err)); |
610 | }); |
611 | future.wait_for(std::chrono::seconds(2)); |
612 | EXPECT_EQ(future.get(), module.get()); |
613 | |
614 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
615 | device->addNetwork(module.get(), std::move(functions2), |
616 | [&promise](const Module *module, Error err) { |
617 | callbackHelper(promise, module, std::move(err)); |
618 | }); |
619 | future.wait_for(std::chrono::seconds(2)); |
620 | EXPECT_EQ(future.get(), module.get()); |
621 | |
622 | Tensor input(ElemKind::FloatTy, {1}); |
623 | input.getHandle().clear(0.5f); |
624 | Tensor output1(ElemKind::FloatTy, {1}); |
625 | output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
626 | Tensor output2(ElemKind::FloatTy, {1}); |
627 | output2.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
628 | |
629 | updateInputPlaceholders(*context1->getPlaceholderBindings(), |
630 | {module->getPlaceholderByNameSlow("func1_input" )}, |
631 | {&input}); |
632 | updateInputPlaceholders(*context2->getPlaceholderBindings(), |
633 | {module->getPlaceholderByNameSlow("func1_input" )}, |
634 | {&input}); |
635 | |
636 | std::promise<std::unique_ptr<ExecutionContext>> runP1, runP2; |
637 | std::future<std::unique_ptr<ExecutionContext>> runF1, runF2; |
638 | std::tie(runP1, runF1) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
639 | std::tie(runP2, runF2) = getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
640 | |
641 | device->runFunction("func1" , std::move(context1), |
642 | [&runP1](RunIdentifierTy, Error err, |
643 | std::unique_ptr<ExecutionContext> context) { |
644 | callbackHelper(runP1, std::move(context), |
645 | std::move(err)); |
646 | }); |
647 | |
648 | device->runFunction("func2" , std::move(context2), |
649 | [&runP2](RunIdentifierTy, Error err, |
650 | std::unique_ptr<ExecutionContext> context) { |
651 | callbackHelper(runP2, std::move(context), |
652 | std::move(err)); |
653 | }); |
654 | |
655 | context1 = runF1.get(); |
656 | context2 = runF2.get(); |
657 | ASSERT_TRUE(context1); |
658 | ASSERT_TRUE(context2); |
659 | EXPECT_NE(context1, context2); |
660 | context1->getPlaceholderBindings()->ensureOnHost(); |
661 | context2->getPlaceholderBindings()->ensureOnHost(); |
662 | |
663 | Tensor *result1 = context1->getPlaceholderBindings()->get( |
664 | module->getPlaceholderByNameSlow("func1_output" )); |
665 | ASSERT_TRUE(result1); |
666 | EXPECT_TRUE(result1->isEqual(output1)); |
667 | |
668 | Tensor *result2 = context2->getPlaceholderBindings()->get( |
669 | module->getPlaceholderByNameSlow("func2_output" )); |
670 | ASSERT_TRUE(result2); |
671 | EXPECT_TRUE(result2->isEqual(output2)); |
672 | } |
673 | |
674 | TEST(DeviceManagerTest, SetDeviceMemory) { |
675 | // Test Interpreter. |
676 | auto interpreterConfigEmpty = DeviceConfig("Interpreter" ); |
677 | auto interpreterConfigFull = DeviceConfig("Interpreter" ); |
678 | interpreterConfigFull.setDeviceMemory(32768); |
679 | // Only deviceConfig setting. |
680 | auto interpreterDeviceSetByDeviceConfig = std::unique_ptr<DeviceManager>( |
681 | DeviceManager::createDeviceManager(interpreterConfigFull)); |
682 | EXPECT_EQ(interpreterDeviceSetByDeviceConfig->getMaximumMemory(), 32768); |
683 | // No setting at all, default memory size. |
684 | auto interpreterDeviceDefault = std::unique_ptr<DeviceManager>( |
685 | DeviceManager::createDeviceManager(interpreterConfigEmpty)); |
686 | EXPECT_EQ(interpreterDeviceDefault->getMaximumMemory(), 2000000000); |
687 | } |
688 | |
689 | TEST(DeviceManagerTest, AvailableMemory) { |
690 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
691 | std::promise<const Module *> promise; |
692 | std::future<const Module *> future; |
693 | |
694 | auto module = makeBasicModule(); |
695 | auto compiledFunctions = compileFunctions("CPU" , module.get(), backing); |
696 | |
697 | uint64_t expectedBytes{0}; |
698 | for (const auto &f : backing) { |
699 | expectedBytes += f->getRuntimeBundle().getConstantWeightSize(); |
700 | } |
701 | |
702 | auto config = DeviceConfig("CPU" ); |
703 | config.setDeviceMemory(expectedBytes); |
704 | auto cpuCoreDevice = std::unique_ptr<DeviceManager>( |
705 | DeviceManager::createDeviceManager(config)); |
706 | ASSERT_FALSE(ERR_TO_BOOL(cpuCoreDevice->init())); |
707 | |
708 | EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes); |
709 | EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), expectedBytes); |
710 | EXPECT_TRUE(cpuCoreDevice->isMemoryAvailable(expectedBytes)); |
711 | EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(expectedBytes + 1)); |
712 | |
713 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
714 | cpuCoreDevice->addNetwork(module.get(), compiledFunctions, |
715 | [&promise](const Module *module, Error err) { |
716 | callbackHelper(promise, module, std::move(err)); |
717 | }); |
718 | |
719 | future.wait_for(std::chrono::seconds(2)); |
720 | EXPECT_EQ(future.get(), module.get()); |
721 | |
722 | EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes); |
723 | EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0); |
724 | EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(expectedBytes)); |
725 | EXPECT_FALSE(cpuCoreDevice->isMemoryAvailable(1)); |
726 | |
727 | // Let's try again. |
728 | auto module2 = makeBasicModule(); |
729 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
730 | cpuCoreDevice->addNetwork(module2.get(), |
731 | compileFunctions("CPU" , module2.get(), backing), |
732 | [&promise](const Module *module, Error err) { |
733 | callbackHelper(promise, module, std::move(err)); |
734 | }); |
735 | |
736 | future.wait_for(std::chrono::seconds(2)); |
737 | auto *resultModule = future.get(); |
738 | EXPECT_NE(resultModule, module2.get()); |
739 | EXPECT_NE(resultModule, module.get()); |
740 | EXPECT_EQ(resultModule, nullptr); |
741 | |
742 | EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes); |
743 | EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0); |
744 | |
745 | // Evict the first network. |
746 | std::promise<std::string> evictPromise; |
747 | std::future<std::string> evictFuture; |
748 | std::tie(evictPromise, evictFuture) = getFutureHelper<std::string>(); |
749 | cpuCoreDevice->evictNetwork( |
750 | "main" , [&evictPromise](std::string functionName, Error err) { |
751 | callbackHelper(evictPromise, functionName, std::move(err)); |
752 | }); |
753 | evictFuture.wait_for(std::chrono::seconds(2)); |
754 | EXPECT_EQ(evictFuture.get(), "main" ); |
755 | |
756 | // And try again, this time with available space. |
757 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
758 | cpuCoreDevice->addNetwork(module2.get(), |
759 | compileFunctions("CPU" , module2.get(), backing), |
760 | [&promise](const Module *module, Error err) { |
761 | callbackHelper(promise, module, std::move(err)); |
762 | }); |
763 | |
764 | future.wait_for(std::chrono::seconds(2)); |
765 | EXPECT_EQ(future.get(), module2.get()); |
766 | |
767 | EXPECT_EQ(cpuCoreDevice->getMaximumMemory(), expectedBytes); |
768 | EXPECT_EQ(cpuCoreDevice->getAvailableMemory(), 0); |
769 | |
770 | EXPECT_FALSE(ERR_TO_BOOL(cpuCoreDevice->stop())); |
771 | |
772 | // Test CPU DeviceConfig. |
773 | auto cpuConfigEmpty = DeviceConfig("CPU" ); |
774 | auto cpuConfigFull = DeviceConfig("CPU" ); |
775 | cpuConfigFull.setDeviceMemory(32768); |
776 | // Only deviceConfig setting. |
777 | auto cpuDeviceSetByDeviceConfig = std::unique_ptr<DeviceManager>( |
778 | DeviceManager::createDeviceManager(cpuConfigFull)); |
779 | EXPECT_EQ(cpuDeviceSetByDeviceConfig->getMaximumMemory(), 32768); |
780 | // No setting at all, default memory size. |
781 | auto cpuDeviceDefault = std::unique_ptr<DeviceManager>( |
782 | DeviceManager::createDeviceManager(cpuConfigEmpty)); |
783 | EXPECT_EQ(cpuDeviceDefault->getMaximumMemory(), 2000000000); |
784 | } |
785 | |
786 | TEST(DeviceManagerTest, DummyDeviceManager) { |
787 | DummyDeviceManager deviceManager{DeviceConfig("Interpreter" )}; |
788 | ASSERT_FALSE(ERR_TO_BOOL(deviceManager.init())); |
789 | |
790 | auto module = makeBasicModule(); |
791 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
792 | FunctionMapTy functions = |
793 | compileFunctions("Interpreter" , module.get(), backing); |
794 | |
795 | std::promise<const Module *> promise; |
796 | std::future<const Module *> future; |
797 | std::tie(promise, future) = getFutureHelper<const Module *>(); |
798 | deviceManager.addNetwork(module.get(), std::move(functions), |
799 | [&promise](const Module *module, Error err) { |
800 | callbackHelper(promise, module, std::move(err)); |
801 | }); |
802 | // no need to wait. |
803 | EXPECT_EQ(future.get(), module.get()); |
804 | |
805 | std::unique_ptr<ExecutionContext> context1 = |
806 | glow::make_unique<ExecutionContext>(); |
807 | context1->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
808 | |
809 | Tensor input1(ElemKind::FloatTy, {1}); |
810 | Tensor output1(ElemKind::FloatTy, {1}); |
811 | input1.getHandle().clear(0.5f); |
812 | output1.getHandle().clear(std::max(std::tanh(0.5f), 0.25f)); |
813 | |
814 | updateInputPlaceholders(*context1->getPlaceholderBindings(), |
815 | {module->getPlaceholderByNameSlow("main_input" )}, |
816 | {&input1}); |
817 | |
818 | std::promise<std::unique_ptr<ExecutionContext>> runPromise; |
819 | std::future<std::unique_ptr<ExecutionContext>> runFuture; |
820 | |
821 | std::tie(runPromise, runFuture) = |
822 | getFutureHelper<std::unique_ptr<ExecutionContext>>(); |
823 | deviceManager.runFunction( |
824 | "main" , std::move(context1), |
825 | [&runPromise](RunIdentifierTy, Error err, |
826 | std::unique_ptr<ExecutionContext> context) { |
827 | callbackHelper(runPromise, std::move(context), std::move(err)); |
828 | }); |
829 | |
830 | runFuture.wait_for(std::chrono::seconds(2)); |
831 | std::unique_ptr<ExecutionContext> context2 = runFuture.get(); |
832 | |
833 | ASSERT_TRUE(context2); |
834 | |
835 | Tensor *result = context2->getPlaceholderBindings()->get( |
836 | module->getPlaceholderByNameSlow("main_output" )); |
837 | ASSERT_TRUE(result); |
838 | EXPECT_TRUE(result->isEqual(output1)); |
839 | |
840 | EXPECT_FALSE(ERR_TO_BOOL(deviceManager.stop())); |
841 | } |
842 | |
843 | /// Check that the device can move data to and from the host. |
844 | /// Disable if your device does not support Device Resident Tensors. |
845 | TEST_P(DeviceManagerTest, DeviceResidentTensors) { |
846 | CHECK_IF_ENABLED(); |
847 | Tensor T = {1.2f, 12.1f, 51.0f, 1515.2f}; |
848 | Tensor R = {1.2f, 12.1f, 51.0f, 1515.2f}; |
849 | |
850 | ASSERT_FALSE(T.isDeviceResident()); |
851 | |
852 | device->transferToDevice(T, nullptr); |
853 | |
854 | ASSERT_TRUE(T.isDeviceResident()); |
855 | |
856 | device->transferFromDevice(T); |
857 | |
858 | ASSERT_FALSE(T.isDeviceResident()); |
859 | |
860 | ASSERT_TRUE(T.isEqual(R)); |
861 | } |
862 | |
863 | /// A mock DeviceManager for use in Device Resident Tensor tests. |
864 | class MockDM : public DeviceManager { |
865 | public: |
866 | MockDM() : DeviceManager(DeviceConfig("MockDM" )) {} |
867 | void addNetwork(const Module *module, FunctionMapTy functions, |
868 | ReadyCBTy readyCB) override {} |
869 | |
870 | void evictNetwork( |
871 | std::string functionName, |
872 | EvictFunctionCBTy evictCB = [](std::string, Error) {}) override {} |
873 | |
874 | runtime::RunIdentifierTy |
875 | runFunction(std::string functionName, |
876 | std::unique_ptr<ExecutionContext> context, |
877 | runtime::ResultCBTy resultCB) override { |
878 | return 0; |
879 | } |
880 | |
881 | uint64_t getMaximumMemory() const override { return 0; } |
882 | |
883 | uint64_t getAvailableMemory() const override { return 0; } |
884 | |
885 | bool isMemoryAvailable(uint64_t estimate) const override { return 0; } |
886 | |
887 | void transferToDevice( |
888 | Tensor &tensor, void *locationContext = nullptr, |
889 | std::function<void(Error)> resultCB = [](Error) {}) override { |
890 | if (locationContext == nullptr) { |
891 | locationContext = malloc(tensor.getSizeInBytes()); |
892 | } |
893 | memcpy(locationContext, tensor.getUnsafePtr(), tensor.getSizeInBytes()); |
894 | tensor.moveToDevice(this, locationContext); |
895 | } |
896 | |
897 | void transferFromDevice( |
898 | Tensor &tensor, bool release = true, |
899 | std::function<void(Error)> resultCB = [](Error) {}) override { |
900 | memcpy(tensor.getUnsafePtr(), tensor.getLocationContext(), |
901 | tensor.getSizeInBytes()); |
902 | free(tensor.getLocationContext()); |
903 | tensor.clearDeviceResidency(); |
904 | } |
905 | |
906 | bool releaseDeviceTensor(void *locationContext) override { return true; } |
907 | }; |
908 | |
909 | TEST_P(DeviceManagerTest, CanHandleDeviceResidentTensors) { |
910 | CHECK_IF_ENABLED(); |
911 | |
912 | MockDM mockDM; |
913 | |
914 | auto module = makeBasicModule(); |
915 | std::vector<std::unique_ptr<CompiledFunction>> backing; |
916 | FunctionMapTy functions = |
917 | compileFunctions(backendName, module.get(), backing); |
918 | |
919 | addToDevice(module.get(), std::move(functions)); |
920 | |
921 | std::unique_ptr<ExecutionContext> context = |
922 | glow::make_unique<ExecutionContext>(); |
923 | context->getPlaceholderBindings()->allocate(module->getPlaceholders()); |
924 | |
925 | Tensor input1(ElemKind::FloatTy, {1}); |
926 | Tensor output1(ElemKind::FloatTy, {1}); |
927 | input1.getHandle().clear(0.5); |
928 | output1.getHandle().clear(std::max(std::tanh(0.5), 0.25)); |
929 | |
930 | updateInputPlaceholders(*context->getPlaceholderBindings(), |
931 | {module->getPlaceholderByNameSlow("main_input" )}, |
932 | {&input1}); |
933 | |
934 | mockDM.transferToDevice(*context->getPlaceholderBindings()->get( |
935 | module->getPlaceholderByNameSlow("main_input" ))); |
936 | |
937 | context = runFunction("main" , std::move(context)); |
938 | ASSERT_TRUE(context); |
939 | Tensor *result1 = context->getPlaceholderBindings()->get( |
940 | module->getPlaceholderByNameSlow("main_output" )); |
941 | ASSERT_TRUE(result1); |
942 | } |
943 | |
944 | TEST_P(DeviceManagerTest, TensorCopyRawToDevice) { |
945 | MockDM mockDM; |
946 | |
947 | Tensor input1(ElemKind::FloatTy, {10}); |
948 | Tensor input2(ElemKind::FloatTy, {10}); |
949 | |
950 | input1.getHandle().clear(1); |
951 | input2.getHandle().clear(2); |
952 | |
953 | float *deviceMemory = (float *)malloc(sizeof(float) * 10); |
954 | mockDM.transferToDevice(input1, deviceMemory); |
955 | |
956 | for (int i = 0; i < 10; ++i) { |
957 | EXPECT_EQ(deviceMemory[i], 1); |
958 | } |
959 | |
960 | input1.copyRawToDevice(&input2); |
961 | |
962 | for (int i = 0; i < 10; ++i) { |
963 | EXPECT_EQ(deviceMemory[i], 2); |
964 | } |
965 | |
966 | mockDM.transferFromDevice(input1); |
967 | auto inputHandle = input1.getHandle(); |
968 | for (unsigned i = 0; i < 10; ++i) { |
969 | EXPECT_EQ(inputHandle.at({i}), 2); |
970 | } |
971 | } |
972 | |
973 | INSTANTIATE_BACKEND_TEST(DeviceManagerTest); |
974 | |