1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
18 | #include "glow/Backend/Backend.h" |
19 | #include "glow/Graph/Graph.h" |
20 | #include "glow/Graph/PlaceholderBindings.h" |
21 | #include "glow/Optimizer/GraphOptimizer/GraphOptimizer.h" |
22 | #include "glow/Support/Error.h" |
23 | |
24 | #include "llvm/ADT/STLExtras.h" |
25 | |
26 | #include <future> |
27 | |
28 | using namespace glow; |
29 | |
30 | ExecutionEngine::ExecutionEngine(llvm::StringRef backend, uint64_t deviceMemory, |
31 | bool ignoreUserDeviceConfig, |
32 | unsigned numDevices) |
33 | : deviceMemory_(deviceMemory), |
34 | ignoreUserDeviceConfig_(ignoreUserDeviceConfig) { |
35 | setBackendName(backend, numDevices); |
36 | } |
37 | |
38 | /// Set the code generator to the given \p backend. |
39 | void ExecutionEngine::setBackendName(llvm::StringRef backend, |
40 | size_t numDevices) { |
41 | clear(); |
42 | module_.reset(new Module); |
43 | rawModule_ = module_.get(); |
44 | backendName_ = backend.str(); |
45 | |
46 | if (hostManager_) { |
47 | EXIT_ON_ERR(hostManager_->clearHost()); |
48 | hostManager_.reset(); |
49 | } |
50 | |
51 | std::vector<std::unique_ptr<runtime::DeviceConfig>> configs; |
52 | if (!ignoreUserDeviceConfig_ && |
53 | loadDeviceConfigsFromFile(configs, deviceMemory_)) { |
54 | // Warning if there is more than a single device configured. |
55 | if (configs.size() != 1) { |
56 | LOG(WARNING) << "Found " << configs.size() |
57 | << " devices configured for the ExecutionEngine" ; |
58 | } |
59 | // Verify that all configured devices match the expected backend name. |
60 | CHECK(backendName_ == configs[0]->backendName) |
61 | << "Expected backend name to match the ExecutionEngine" ; |
62 | } else { |
63 | for (size_t i = 0; i < numDevices; i++) { |
64 | auto config = glow::make_unique<runtime::DeviceConfig>(backendName_); |
65 | if (deviceMemory_) { |
66 | config->setDeviceMemory(deviceMemory_); |
67 | } |
68 | configs.push_back(std::move(config)); |
69 | } |
70 | } |
71 | hostManager_ = glow::make_unique<runtime::HostManager>(std::move(configs)); |
72 | } |
73 | |
74 | llvm::StringRef ExecutionEngine::getBackendName() const { return backendName_; } |
75 | |
76 | Function *ExecutionEngine::getSingleFunctionFromModule() const { |
77 | auto &fList = getModule().getFunctions(); |
78 | assert(fList.size() == 1 && "More than one Function in Module." ); |
79 | return *fList.begin(); |
80 | } |
81 | |
82 | ExecutionEngine::~ExecutionEngine() { clear(); } |
83 | |
84 | void ExecutionEngine::clear() { |
85 | if (hostManager_) { |
86 | EXIT_ON_ERR(hostManager_->clearHost()); |
87 | } |
88 | compiledFunctions_.clear(); |
89 | module_.reset(nullptr); |
90 | } |
91 | |
92 | void glow::updateInputPlaceholders(PlaceholderBindings &bindings, |
93 | llvm::ArrayRef<Placeholder *> ph, |
94 | llvm::ArrayRef<Tensor *> inputs) { |
95 | assert(inputs.size() == ph.size() && |
96 | "The number of inputs does not match the number of Placeholders" ); |
97 | |
98 | for (int i = 0, e = ph.size(); i < e; i++) { |
99 | assert(ph[i] && "Invalid value" ); |
100 | auto *backingTensor = bindings.get(ph[i]); |
101 | assert(backingTensor && "Can't find the placeholder" ); |
102 | auto dim = inputs[i]->dims(); |
103 | (void)dim; |
104 | assert(backingTensor->getType().isEqual(inputs[i]->getType()) && |
105 | "Mismatch on Placeholder and Tensor types." ); |
106 | backingTensor->assign(inputs[i]); |
107 | } |
108 | } |
109 | |
110 | void glow::updateInputPlaceholdersByName(PlaceholderBindings &bindings, |
111 | Module *mod, |
112 | llvm::ArrayRef<llvm::StringRef> ph, |
113 | llvm::ArrayRef<Tensor *> inputs) { |
114 | assert(inputs.size() == ph.size() && |
115 | "The number of inputs does not match the number of Placeholders" ); |
116 | |
117 | for (int i = 0, e = ph.size(); i < e; i++) { |
118 | Placeholder *p = mod->getPlaceholderByNameSlow(legalizeName(ph[i])); |
119 | Tensor *t = inputs[i]; |
120 | assert(t && "Invalid tensor." ); |
121 | assert(p && "Invalid placeholder." ); |
122 | updateInputPlaceholders(bindings, {p}, {t}); |
123 | } |
124 | } |
125 | |
126 | void ExecutionEngine::runInternal(ExecutionContext &context, |
127 | llvm::StringRef name) { |
128 | std::unique_ptr<ExecutionContext> contextPtr(&context); |
129 | std::unique_ptr<ExecutionContext> contextOut; |
130 | std::promise<void> runPromise; |
131 | auto fut = runPromise.get_future(); |
132 | Error runErr = Error::empty(); |
133 | hostManager_->runNetwork(name, std::move(contextPtr), |
134 | [&runPromise, &runErr, &contextOut]( |
135 | runtime::RunIdentifierTy, Error err, |
136 | std::unique_ptr<ExecutionContext> contextPtr) { |
137 | contextOut = std::move(contextPtr); |
138 | runErr = std::move(err); |
139 | runPromise.set_value(); |
140 | }); |
141 | |
142 | fut.wait(); |
143 | if (ensureOutputsOnHost_) { |
144 | contextOut->getPlaceholderBindings()->ensureOnHost(); |
145 | } |
146 | // Don't delete context. |
147 | contextOut.release(); |
148 | EXIT_ON_ERR(std::move(runErr)); |
149 | } |
150 | |
151 | void ExecutionEngine::run(ExecutionContext &context) { |
152 | assert((compiledFunctions_.size() == 1 || allowMultiFunction_) && |
153 | "Expected exactly one compiled function." ); |
154 | runInternal(context, *compiledFunctions_.begin()); |
155 | } |
156 | |
157 | void ExecutionEngine::run(ExecutionContext &context, llvm::StringRef name) { |
158 | runInternal(context, name); |
159 | } |
160 | |
161 | void ExecutionEngine::run(PlaceholderBindings &bindings) { |
162 | assert((compiledFunctions_.size() == 1 || allowMultiFunction_) && |
163 | "Expected exactly one compiled function." ); |
164 | std::unique_ptr<PlaceholderBindings> bindingsPtr(&bindings); |
165 | ExecutionContext context(std::move(bindingsPtr)); |
166 | runInternal(context, *compiledFunctions_.begin()); |
167 | // Don't delete bindings. |
168 | context.movePlaceholderBindings().release(); |
169 | } |
170 | |
171 | void ExecutionEngine::run(PlaceholderBindings &bindings, llvm::StringRef name) { |
172 | std::unique_ptr<PlaceholderBindings> bindingsPtr(&bindings); |
173 | ExecutionContext context(std::move(bindingsPtr)); |
174 | runInternal(context, name); |
175 | // Don't delete bindings. |
176 | context.movePlaceholderBindings().release(); |
177 | } |
178 | |
179 | void glow::runBatch(ExecutionEngine &EE, PlaceholderBindings &bindings, |
180 | size_t iterations, size_t &sampleCounter, |
181 | llvm::ArrayRef<Placeholder *> ph, |
182 | llvm::ArrayRef<Tensor *> inputs, llvm::StringRef name) { |
183 | // This is the size of one batch (the number of samples in the batch). |
184 | size_t batchSize = ph[0]->getType()->dims()[0]; |
185 | |
186 | assert(!inputs.empty() && "No inputs" ); |
187 | assert(inputs.size() == ph.size() && |
188 | "The number of inputs does not match the number of placeholders" ); |
189 | |
190 | // For each iteration in the batch: |
191 | for (size_t j = 0; j < iterations; j++) { |
192 | |
193 | // Update the input placeholders. |
194 | for (int i = 0, e = ph.size(); i < e; i++) { |
195 | assert(ph[i] && "Invalid value" ); |
196 | auto *backingTensor = bindings.get(ph[i]); |
197 | assert(backingTensor && "Can't find the backing tensor" ); |
198 | |
199 | auto dim = inputs[i]->dims(); |
200 | assert(backingTensor->dims().drop_front() == dim.drop_front() && |
201 | "Invalid slice size" ); |
202 | // Extract the n'th slice, that must be a tensor. |
203 | size_t slc = sampleCounter % dim[0]; |
204 | // Pick up one slice from the input tensors, and load it into the |
205 | // corresponding network Placeholder. |
206 | backingTensor->copyConsecutiveSlices(inputs[i], slc); |
207 | } |
208 | |
209 | // Run the network. |
210 | if (name == "" ) { |
211 | EE.run(bindings); |
212 | } else { |
213 | EE.run(bindings, name); |
214 | } |
215 | sampleCounter += batchSize; |
216 | } |
217 | } |
218 | |
219 | void glow::evalBatch( |
220 | ExecutionEngine &EE, PlaceholderBindings &bindings, size_t numMinibatchRuns, |
221 | size_t &sampleCounter, Placeholder *inputPH, Placeholder *outputPH, |
222 | Tensor &samplesInput, Tensor &labelsInput, llvm::StringRef name, |
223 | std::function<void(const Tensor &sampleIn, const Tensor &sampleOut, |
224 | const Tensor &label, size_t sampleIndex)> &&cb) { |
225 | // The number of samples in a minibatch (a single function run) |
226 | size_t minibatchSize = inputPH->getType()->dims()[0]; |
227 | |
228 | assert(samplesInput.dims()[0] == labelsInput.dims()[0] && |
229 | "The number of sample inputs does not match the number of labels" ); |
230 | |
231 | auto LIH = labelsInput.getHandle<int64_t>(); |
232 | |
233 | // For each iteration in the batch: |
234 | for (size_t j = 0; j < numMinibatchRuns; j++) { |
235 | assert(inputPH && "Invalid value" ); |
236 | auto *backingTensor = bindings.get(inputPH); |
237 | assert(backingTensor && "Can't find the backing tensor" ); |
238 | auto dim = samplesInput.dims(); |
239 | assert(backingTensor->dims().drop_front() == dim.drop_front() && |
240 | "Invalid slice size" ); |
241 | // Extract the n'th slice, that must be a tensor. |
242 | size_t slc = sampleCounter % dim[0]; |
243 | // Pick up one slice from the input tensors, and load it into the |
244 | // corresponding network Placeholder. |
245 | backingTensor->copyConsecutiveSlices(&samplesInput, slc); |
246 | |
247 | // Run the network. |
248 | if (name == "" ) { |
249 | EE.run(bindings); |
250 | } else { |
251 | EE.run(bindings, name); |
252 | } |
253 | |
254 | for (unsigned i = 0; i < minibatchSize; i++) { |
255 | auto sampleInputTensor = backingTensor->getHandle().extractSlice(i); |
256 | auto sampleOutputTensor = |
257 | bindings.get(outputPH)->getHandle().extractSlice(i); |
258 | // If index is out of bounds of samples/labels first dimension, it is |
259 | // wrapped around to be consistent with "copyConsecutiveSlices". |
260 | auto labelTensor = LIH.extractSlice((sampleCounter + i) % dim[0]); |
261 | |
262 | cb(sampleInputTensor, sampleOutputTensor, labelTensor, sampleCounter + i); |
263 | } |
264 | sampleCounter += minibatchSize; |
265 | } |
266 | } |
267 | |
268 | void ExecutionEngine::compile(CompilationMode mode) { |
269 | CompilationContext cctx; |
270 | cctx.compMode = mode; |
271 | if (skipModuleStrip_) { |
272 | cctx.skipModuleStrip = true; |
273 | } |
274 | compile(cctx); |
275 | } |
276 | |
277 | void ExecutionEngine::compile(CompilationContext &cctx) { |
278 | assert(module_.get() && "Compile has already been called." ); |
279 | |
280 | if (skipModuleStrip_) { |
281 | cctx.skipModuleStrip = true; |
282 | } |
283 | |
284 | if (cctx.prepartitionedConfig) { |
285 | if (cctx.prepartitionedConfig->funcs.size() > 1) { |
286 | allowMultiFunction_ = true; |
287 | } |
288 | |
289 | // If we are compiling a prepartitioned model then we should add the name of |
290 | // the prepartitioned config, which is later used as the root of the DAG, |
291 | // and also used to kick off running. |
292 | compiledFunctions_.insert(cctx.prepartitionedConfig->funcName); |
293 | } |
294 | |
295 | for (auto *F : module_->getFunctions()) { |
296 | // Check to see if this Function is part of the prepartitioned config if it |
297 | // exists, as we do not kick off execution for the partitions individually. |
298 | bool skipAdding = false; |
299 | if (cctx.prepartitionedConfig) { |
300 | auto &pFuns = cctx.prepartitionedConfig->funcs; |
301 | skipAdding = std::find(pFuns.begin(), pFuns.end(), F) != pFuns.end(); |
302 | } |
303 | if (!skipAdding) { |
304 | compiledFunctions_.insert(F->getName().str()); |
305 | } |
306 | } |
307 | |
308 | EXIT_ON_ERR(hostManager_->addNetwork(std::move(module_), cctx)); |
309 | } |
310 | |
311 | Backend &ExecutionEngine::getBackend(llvm::StringRef backendName) const { |
312 | return hostManager_->getBackend(backendName); |
313 | } |
314 | |
315 | Backend &ExecutionEngine::getBackend() const { |
316 | return hostManager_->getBackend(backendName_); |
317 | } |
318 | |