1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "glow/ExecutionEngine/ExecutionEngine.h"
18#include "glow/Backend/Backend.h"
19#include "glow/Graph/Graph.h"
20#include "glow/Graph/PlaceholderBindings.h"
21#include "glow/Optimizer/GraphOptimizer/GraphOptimizer.h"
22#include "glow/Support/Error.h"
23
24#include "llvm/ADT/STLExtras.h"
25
26#include <future>
27
28using namespace glow;
29
30ExecutionEngine::ExecutionEngine(llvm::StringRef backend, uint64_t deviceMemory,
31 bool ignoreUserDeviceConfig,
32 unsigned numDevices)
33 : deviceMemory_(deviceMemory),
34 ignoreUserDeviceConfig_(ignoreUserDeviceConfig) {
35 setBackendName(backend, numDevices);
36}
37
38/// Set the code generator to the given \p backend.
39void ExecutionEngine::setBackendName(llvm::StringRef backend,
40 size_t numDevices) {
41 clear();
42 module_.reset(new Module);
43 rawModule_ = module_.get();
44 backendName_ = backend.str();
45
46 if (hostManager_) {
47 EXIT_ON_ERR(hostManager_->clearHost());
48 hostManager_.reset();
49 }
50
51 std::vector<std::unique_ptr<runtime::DeviceConfig>> configs;
52 if (!ignoreUserDeviceConfig_ &&
53 loadDeviceConfigsFromFile(configs, deviceMemory_)) {
54 // Warning if there is more than a single device configured.
55 if (configs.size() != 1) {
56 LOG(WARNING) << "Found " << configs.size()
57 << " devices configured for the ExecutionEngine";
58 }
59 // Verify that all configured devices match the expected backend name.
60 CHECK(backendName_ == configs[0]->backendName)
61 << "Expected backend name to match the ExecutionEngine";
62 } else {
63 for (size_t i = 0; i < numDevices; i++) {
64 auto config = glow::make_unique<runtime::DeviceConfig>(backendName_);
65 if (deviceMemory_) {
66 config->setDeviceMemory(deviceMemory_);
67 }
68 configs.push_back(std::move(config));
69 }
70 }
71 hostManager_ = glow::make_unique<runtime::HostManager>(std::move(configs));
72}
73
74llvm::StringRef ExecutionEngine::getBackendName() const { return backendName_; }
75
76Function *ExecutionEngine::getSingleFunctionFromModule() const {
77 auto &fList = getModule().getFunctions();
78 assert(fList.size() == 1 && "More than one Function in Module.");
79 return *fList.begin();
80}
81
82ExecutionEngine::~ExecutionEngine() { clear(); }
83
84void ExecutionEngine::clear() {
85 if (hostManager_) {
86 EXIT_ON_ERR(hostManager_->clearHost());
87 }
88 compiledFunctions_.clear();
89 module_.reset(nullptr);
90}
91
92void glow::updateInputPlaceholders(PlaceholderBindings &bindings,
93 llvm::ArrayRef<Placeholder *> ph,
94 llvm::ArrayRef<Tensor *> inputs) {
95 assert(inputs.size() == ph.size() &&
96 "The number of inputs does not match the number of Placeholders");
97
98 for (int i = 0, e = ph.size(); i < e; i++) {
99 assert(ph[i] && "Invalid value");
100 auto *backingTensor = bindings.get(ph[i]);
101 assert(backingTensor && "Can't find the placeholder");
102 auto dim = inputs[i]->dims();
103 (void)dim;
104 assert(backingTensor->getType().isEqual(inputs[i]->getType()) &&
105 "Mismatch on Placeholder and Tensor types.");
106 backingTensor->assign(inputs[i]);
107 }
108}
109
110void glow::updateInputPlaceholdersByName(PlaceholderBindings &bindings,
111 Module *mod,
112 llvm::ArrayRef<llvm::StringRef> ph,
113 llvm::ArrayRef<Tensor *> inputs) {
114 assert(inputs.size() == ph.size() &&
115 "The number of inputs does not match the number of Placeholders");
116
117 for (int i = 0, e = ph.size(); i < e; i++) {
118 Placeholder *p = mod->getPlaceholderByNameSlow(legalizeName(ph[i]));
119 Tensor *t = inputs[i];
120 assert(t && "Invalid tensor.");
121 assert(p && "Invalid placeholder.");
122 updateInputPlaceholders(bindings, {p}, {t});
123 }
124}
125
126void ExecutionEngine::runInternal(ExecutionContext &context,
127 llvm::StringRef name) {
128 std::unique_ptr<ExecutionContext> contextPtr(&context);
129 std::unique_ptr<ExecutionContext> contextOut;
130 std::promise<void> runPromise;
131 auto fut = runPromise.get_future();
132 Error runErr = Error::empty();
133 hostManager_->runNetwork(name, std::move(contextPtr),
134 [&runPromise, &runErr, &contextOut](
135 runtime::RunIdentifierTy, Error err,
136 std::unique_ptr<ExecutionContext> contextPtr) {
137 contextOut = std::move(contextPtr);
138 runErr = std::move(err);
139 runPromise.set_value();
140 });
141
142 fut.wait();
143 if (ensureOutputsOnHost_) {
144 contextOut->getPlaceholderBindings()->ensureOnHost();
145 }
146 // Don't delete context.
147 contextOut.release();
148 EXIT_ON_ERR(std::move(runErr));
149}
150
151void ExecutionEngine::run(ExecutionContext &context) {
152 assert((compiledFunctions_.size() == 1 || allowMultiFunction_) &&
153 "Expected exactly one compiled function.");
154 runInternal(context, *compiledFunctions_.begin());
155}
156
157void ExecutionEngine::run(ExecutionContext &context, llvm::StringRef name) {
158 runInternal(context, name);
159}
160
161void ExecutionEngine::run(PlaceholderBindings &bindings) {
162 assert((compiledFunctions_.size() == 1 || allowMultiFunction_) &&
163 "Expected exactly one compiled function.");
164 std::unique_ptr<PlaceholderBindings> bindingsPtr(&bindings);
165 ExecutionContext context(std::move(bindingsPtr));
166 runInternal(context, *compiledFunctions_.begin());
167 // Don't delete bindings.
168 context.movePlaceholderBindings().release();
169}
170
171void ExecutionEngine::run(PlaceholderBindings &bindings, llvm::StringRef name) {
172 std::unique_ptr<PlaceholderBindings> bindingsPtr(&bindings);
173 ExecutionContext context(std::move(bindingsPtr));
174 runInternal(context, name);
175 // Don't delete bindings.
176 context.movePlaceholderBindings().release();
177}
178
179void glow::runBatch(ExecutionEngine &EE, PlaceholderBindings &bindings,
180 size_t iterations, size_t &sampleCounter,
181 llvm::ArrayRef<Placeholder *> ph,
182 llvm::ArrayRef<Tensor *> inputs, llvm::StringRef name) {
183 // This is the size of one batch (the number of samples in the batch).
184 size_t batchSize = ph[0]->getType()->dims()[0];
185
186 assert(!inputs.empty() && "No inputs");
187 assert(inputs.size() == ph.size() &&
188 "The number of inputs does not match the number of placeholders");
189
190 // For each iteration in the batch:
191 for (size_t j = 0; j < iterations; j++) {
192
193 // Update the input placeholders.
194 for (int i = 0, e = ph.size(); i < e; i++) {
195 assert(ph[i] && "Invalid value");
196 auto *backingTensor = bindings.get(ph[i]);
197 assert(backingTensor && "Can't find the backing tensor");
198
199 auto dim = inputs[i]->dims();
200 assert(backingTensor->dims().drop_front() == dim.drop_front() &&
201 "Invalid slice size");
202 // Extract the n'th slice, that must be a tensor.
203 size_t slc = sampleCounter % dim[0];
204 // Pick up one slice from the input tensors, and load it into the
205 // corresponding network Placeholder.
206 backingTensor->copyConsecutiveSlices(inputs[i], slc);
207 }
208
209 // Run the network.
210 if (name == "") {
211 EE.run(bindings);
212 } else {
213 EE.run(bindings, name);
214 }
215 sampleCounter += batchSize;
216 }
217}
218
219void glow::evalBatch(
220 ExecutionEngine &EE, PlaceholderBindings &bindings, size_t numMinibatchRuns,
221 size_t &sampleCounter, Placeholder *inputPH, Placeholder *outputPH,
222 Tensor &samplesInput, Tensor &labelsInput, llvm::StringRef name,
223 std::function<void(const Tensor &sampleIn, const Tensor &sampleOut,
224 const Tensor &label, size_t sampleIndex)> &&cb) {
225 // The number of samples in a minibatch (a single function run)
226 size_t minibatchSize = inputPH->getType()->dims()[0];
227
228 assert(samplesInput.dims()[0] == labelsInput.dims()[0] &&
229 "The number of sample inputs does not match the number of labels");
230
231 auto LIH = labelsInput.getHandle<int64_t>();
232
233 // For each iteration in the batch:
234 for (size_t j = 0; j < numMinibatchRuns; j++) {
235 assert(inputPH && "Invalid value");
236 auto *backingTensor = bindings.get(inputPH);
237 assert(backingTensor && "Can't find the backing tensor");
238 auto dim = samplesInput.dims();
239 assert(backingTensor->dims().drop_front() == dim.drop_front() &&
240 "Invalid slice size");
241 // Extract the n'th slice, that must be a tensor.
242 size_t slc = sampleCounter % dim[0];
243 // Pick up one slice from the input tensors, and load it into the
244 // corresponding network Placeholder.
245 backingTensor->copyConsecutiveSlices(&samplesInput, slc);
246
247 // Run the network.
248 if (name == "") {
249 EE.run(bindings);
250 } else {
251 EE.run(bindings, name);
252 }
253
254 for (unsigned i = 0; i < minibatchSize; i++) {
255 auto sampleInputTensor = backingTensor->getHandle().extractSlice(i);
256 auto sampleOutputTensor =
257 bindings.get(outputPH)->getHandle().extractSlice(i);
258 // If index is out of bounds of samples/labels first dimension, it is
259 // wrapped around to be consistent with "copyConsecutiveSlices".
260 auto labelTensor = LIH.extractSlice((sampleCounter + i) % dim[0]);
261
262 cb(sampleInputTensor, sampleOutputTensor, labelTensor, sampleCounter + i);
263 }
264 sampleCounter += minibatchSize;
265 }
266}
267
268void ExecutionEngine::compile(CompilationMode mode) {
269 CompilationContext cctx;
270 cctx.compMode = mode;
271 if (skipModuleStrip_) {
272 cctx.skipModuleStrip = true;
273 }
274 compile(cctx);
275}
276
277void ExecutionEngine::compile(CompilationContext &cctx) {
278 assert(module_.get() && "Compile has already been called.");
279
280 if (skipModuleStrip_) {
281 cctx.skipModuleStrip = true;
282 }
283
284 if (cctx.prepartitionedConfig) {
285 if (cctx.prepartitionedConfig->funcs.size() > 1) {
286 allowMultiFunction_ = true;
287 }
288
289 // If we are compiling a prepartitioned model then we should add the name of
290 // the prepartitioned config, which is later used as the root of the DAG,
291 // and also used to kick off running.
292 compiledFunctions_.insert(cctx.prepartitionedConfig->funcName);
293 }
294
295 for (auto *F : module_->getFunctions()) {
296 // Check to see if this Function is part of the prepartitioned config if it
297 // exists, as we do not kick off execution for the partitions individually.
298 bool skipAdding = false;
299 if (cctx.prepartitionedConfig) {
300 auto &pFuns = cctx.prepartitionedConfig->funcs;
301 skipAdding = std::find(pFuns.begin(), pFuns.end(), F) != pFuns.end();
302 }
303 if (!skipAdding) {
304 compiledFunctions_.insert(F->getName().str());
305 }
306 }
307
308 EXIT_ON_ERR(hostManager_->addNetwork(std::move(module_), cctx));
309}
310
311Backend &ExecutionEngine::getBackend(llvm::StringRef backendName) const {
312 return hostManager_->getBackend(backendName);
313}
314
315Backend &ExecutionEngine::getBackend() const {
316 return hostManager_->getBackend(backendName_);
317}
318