1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | #include "Base.h" |
18 | #include "GlowOnnxifiManager.h" |
19 | #include "folly/String.h" |
20 | #include "llvm/Support/CommandLine.h" |
21 | |
22 | #include "glow/Flags/Flags.h" |
23 | #include "glow/Importer/ONNXIFIModelLoader.h" |
24 | #include "glow/Runtime/RequestData.h" |
25 | |
26 | #include <cstring> |
27 | #include <glog/logging.h> |
28 | |
29 | /// Allow defining names for onnxifi implementation. |
30 | #ifndef GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER |
31 | #define GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(name) name |
32 | #endif |
33 | |
34 | #define EXTERNC extern "C" |
35 | |
36 | /** |
37 | * This file contains implementation of the onnxifi interface. |
38 | * Documentation on the functions implementing onnxifi interface in |
39 | * this file is very shallow. |
40 | * Please see more documentation on functions that need to be |
41 | * implemented: https://github.com/houseroad/foxi/blob/master/foxi/onnxifi.h. |
42 | */ |
43 | |
44 | /// Return stable IDs of available backends on the system. |
45 | /// \param backendIDs output parameter and represents pointer to the memory |
46 | /// where the backend IDs will be returned. If it's NULL, |
47 | /// numBackends will be populated with the number of backends |
48 | /// supported. |
49 | /// \param numBackends input/output parameter. |
50 | /// As an input, it specifies the capacity allocated in the |
51 | /// backendIDs. As an output, it specifies the number of |
52 | /// actual available backends. |
53 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
54 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendIDs)( |
55 | onnxBackendID *backendIDs, size_t *numBackends) { |
56 | if (!numBackends) { |
57 | return ONNXIFI_STATUS_INVALID_POINTER; |
58 | } |
59 | |
60 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
61 | |
62 | const size_t numBackendsCapacity = *numBackends; |
63 | |
64 | using namespace glow::runtime; |
65 | using namespace glow::onnxifi; |
66 | const bool withCPU = DeviceManager::numDevices("CPU" ) > 0; |
67 | const bool withHabana = DeviceManager::numDevices("Habana" ) > 0; |
68 | const bool withNNPI = DeviceManager::numDevices("NNPI" ) > 0; |
69 | #ifdef GLOW_EXTRABACKEND |
70 | #define V(name) with##name |
71 | #define makeVar(name) V(name) |
72 | |
73 | #define Q(name) #name |
74 | #define makeQuote(name) Q(name) |
75 | const bool makeVar(GLOW_EXTRABACKEND) = |
76 | DeviceManager::numDevices(makeQuote(GLOW_EXTRABACKEND)) > 0; |
77 | #endif |
78 | |
79 | // Only return quantization backend if GLOW_DUMP_PROFILE. |
80 | if (getenv("GLOW_DUMP_PROFILE" )) { |
81 | *numBackends = 1; |
82 | |
83 | // In case backendIDs is nullptr or does not have enough capacity just |
84 | // return the total number of supported backends. |
85 | if (numBackendsCapacity < *numBackends || !backendIDs) { |
86 | return ONNXIFI_STATUS_FALLBACK; |
87 | } |
88 | |
89 | auto backendName = glow::onnxifi::flags::BackendName.empty() |
90 | ? "Interpreter" |
91 | : glow::onnxifi::flags::BackendName; |
92 | LOG(INFO) << "ONNXIFI: Executing on " << backendName << " Glow backend" ; |
93 | auto *quantizationBackendC2 = |
94 | manager.createBackend(backendName, |
95 | /*useOnnx*/ false, /*forQuantization*/ true); |
96 | backendIDs[0] = quantizationBackendC2; |
97 | } else { |
98 | *numBackends = 2; |
99 | |
100 | auto backendName = glow::onnxifi::flags::BackendName; |
101 | |
102 | if (backendName.empty()) { |
103 | if (withNNPI) { |
104 | backendName = "NNPI" ; |
105 | } else if (withHabana) { |
106 | backendName = "Habana" ; |
107 | #ifdef GLOW_EXTRABACKEND |
108 | } else if (makeVar(GLOW_EXTRABACKEND)) { |
109 | backendName = makeQuote(GLOW_EXTRABACKEND); |
110 | #undef V |
111 | #undef makeVar |
112 | #undef Q |
113 | #undef makeQuote |
114 | #endif |
115 | } else if (withCPU) { |
116 | backendName = "CPU" ; |
117 | } else { |
118 | backendName = "Interpreter" ; |
119 | } |
120 | } |
121 | |
122 | // In case backendIDs is nullptr or does not have enough capacity just |
123 | // return the total number of supported backends. |
124 | if (numBackendsCapacity < *numBackends || !backendIDs) { |
125 | return ONNXIFI_STATUS_FALLBACK; |
126 | } |
127 | |
128 | LOG(INFO) << "ONNXIFI: Executing on " << backendName << " Glow backend" ; |
129 | |
130 | backendIDs[0] = manager.createBackend(backendName, |
131 | /*useOnnx*/ false); |
132 | backendIDs[1] = manager.createBackend(backendName, |
133 | /*useOnnx*/ true); |
134 | } |
135 | |
136 | return ONNXIFI_STATUS_SUCCESS; |
137 | } |
138 | |
139 | /// Deinitialize ONNXIFI backend ID and release associated resources. |
140 | /// Caller is responsible to release objects associated with the backend ID |
141 | /// (onnxBackend, onnxGraph, onnxEvent) before calling this function. |
142 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
143 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseBackendID)( |
144 | onnxBackendID backendID) { |
145 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
146 | |
147 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID); |
148 | if (!manager.isValid(glowBackend)) { |
149 | return ONNXIFI_STATUS_INVALID_ID; |
150 | } |
151 | |
152 | manager.release(glowBackend); |
153 | |
154 | return ONNXIFI_STATUS_SUCCESS; |
155 | } |
156 | |
157 | static onnxStatus setBackendInfoString(void *infoValue, size_t *infoValueSize, |
158 | const char *str) { |
159 | size_t len = strlen(str) + 1; |
160 | if (!infoValue || *infoValueSize < len) { |
161 | *infoValueSize = len; |
162 | return ONNXIFI_STATUS_FALLBACK; |
163 | } |
164 | |
165 | strncpy((char *)infoValue, str, len); |
166 | *infoValueSize = len; |
167 | return ONNXIFI_STATUS_SUCCESS; |
168 | } |
169 | |
170 | static onnxStatus setBackendInfoUInt64(void *infoValue, size_t *infoValueSize, |
171 | uint64_t value) { |
172 | if (!infoValue || *infoValueSize < sizeof(uint64_t)) { |
173 | *infoValueSize = sizeof(uint64_t); |
174 | return ONNXIFI_STATUS_FALLBACK; |
175 | } |
176 | |
177 | *(uint64_t *)(infoValue) = value; |
178 | *infoValueSize = sizeof(uint64_t); |
179 | return ONNXIFI_STATUS_SUCCESS; |
180 | } |
181 | |
182 | /// Query high-level information about the backend and its target device. |
183 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
184 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendInfo)( |
185 | onnxBackendID backendID, onnxBackendInfo infoType, void *infoValue, |
186 | size_t *infoValueSize) { |
187 | if (!infoValueSize) { |
188 | return ONNXIFI_STATUS_INVALID_POINTER; |
189 | } |
190 | |
191 | if (!infoValue) { |
192 | return ONNXIFI_STATUS_INVALID_POINTER; |
193 | } |
194 | |
195 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
196 | |
197 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID); |
198 | if (!manager.isValid(glowBackend)) { |
199 | return ONNXIFI_STATUS_INVALID_ID; |
200 | } |
201 | |
202 | // TODO: support more info type values. Here is the minimal required |
203 | // subset of info types. |
204 | switch (infoType) { |
205 | case ONNXIFI_BACKEND_NAME: |
206 | return setBackendInfoString(infoValue, infoValueSize, "Glow" ); |
207 | case ONNXIFI_BACKEND_VENDOR: |
208 | return setBackendInfoString(infoValue, infoValueSize, "PyTorch" ); |
209 | case ONNXIFI_BACKEND_VERSION: |
210 | return setBackendInfoString(infoValue, infoValueSize, "1.0.0" ); |
211 | case ONNXIFI_BACKEND_DEVICE: |
212 | return setBackendInfoString(infoValue, infoValueSize, |
213 | glowBackend->getUseOnnx() ? "Glow Onnx" |
214 | : "Glow Caffe2" ); |
215 | case ONNXIFI_BACKEND_MEMORY_TYPES: |
216 | return setBackendInfoUInt64(infoValue, infoValueSize, |
217 | ONNXIFI_MEMORY_TYPE_CPU); |
218 | case ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES: |
219 | return setBackendInfoUInt64(infoValue, infoValueSize, |
220 | ONNXIFI_SYNCHRONIZATION_EVENT); |
221 | case ONNXIFI_BACKEND_EXTENSIONS: |
222 | return setBackendInfoString( |
223 | infoValue, infoValueSize, |
224 | "onnxSetIOAndRunGraphFunction onnxWaitEventForFunction " |
225 | "onnxReleaseTraceEventsFunction onnxGetCurrentBatchSizeFunction" ); |
226 | default: |
227 | return ONNXIFI_STATUS_UNSUPPORTED_PROPERTY; |
228 | } |
229 | } |
230 | |
231 | /// Query if an ONNX model graph is compatible with the backend. |
232 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
233 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendCompatibility)( |
234 | onnxBackendID backendID, size_t onnxModelSize, const void *onnxModel) { |
235 | if (!onnxModel) { |
236 | return ONNXIFI_STATUS_INVALID_POINTER; |
237 | } |
238 | |
239 | if (!onnxModelSize) { |
240 | return ONNXIFI_STATUS_INVALID_SIZE; |
241 | } |
242 | |
243 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
244 | |
245 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID); |
246 | if (!manager.isValid(glowBackend)) { |
247 | return ONNXIFI_STATUS_INVALID_ID; |
248 | } |
249 | |
250 | return glowBackend->checkGraphCompatibility(onnxModel, onnxModelSize); |
251 | } |
252 | |
253 | /// Initialize an ONNXIFI backend. |
254 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
255 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitBackend)( |
256 | onnxBackendID backendID, const uint64_t *auxpropertiesList, |
257 | onnxBackend *backend) { |
258 | if (!backend) { |
259 | return ONNXIFI_STATUS_INVALID_POINTER; |
260 | } |
261 | |
262 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
263 | |
264 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID); |
265 | if (!manager.isValid(glowBackend)) { |
266 | return ONNXIFI_STATUS_INVALID_ID; |
267 | } |
268 | |
269 | *backend = glowBackend; |
270 | |
271 | return ONNXIFI_STATUS_SUCCESS; |
272 | } |
273 | |
274 | /// Deinitialize an ONNXIFI backend and release associated resources. |
275 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
276 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseBackend)(onnxBackend backend) { |
277 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
278 | |
279 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend); |
280 | if (!manager.isValid(glowBackend)) { |
281 | return ONNXIFI_STATUS_INVALID_BACKEND; |
282 | } |
283 | |
284 | return ONNXIFI_STATUS_SUCCESS; |
285 | } |
286 | |
287 | /// Initialize a single-shot ONNXIFI event. |
288 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
289 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitEvent)(onnxBackend backend, |
290 | onnxEvent *event) { |
291 | if (!event) { |
292 | return ONNXIFI_STATUS_INVALID_POINTER; |
293 | } |
294 | |
295 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
296 | |
297 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend); |
298 | if (!manager.isValid(glowBackend)) { |
299 | return ONNXIFI_STATUS_INVALID_BACKEND; |
300 | } |
301 | |
302 | *event = manager.createEvent(); |
303 | |
304 | return ONNXIFI_STATUS_SUCCESS; |
305 | } |
306 | |
307 | /// Change the state of the ONNXIFI event \p event to signalled. |
308 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
309 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSignalEvent)(onnxEvent event) { |
310 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
311 | |
312 | auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event); |
313 | if (!manager.isValid(glowEvent)) { |
314 | return ONNXIFI_STATUS_INVALID_EVENT; |
315 | } |
316 | |
317 | if (!glowEvent->signal(ONNXIFI_STATUS_SUCCESS)) { |
318 | return ONNXIFI_STATUS_INVALID_STATE; |
319 | } |
320 | |
321 | return ONNXIFI_STATUS_SUCCESS; |
322 | } |
323 | |
324 | /// Wait until an ONNXIFI \p event is signalled. |
325 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
326 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEvent)(onnxEvent event) { |
327 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
328 | |
329 | auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event); |
330 | if (!manager.isValid(glowEvent)) { |
331 | return ONNXIFI_STATUS_INVALID_EVENT; |
332 | } |
333 | |
334 | return glowEvent->wait(); |
335 | } |
336 | |
337 | /// Wait until an ONNXIFI \p event is signalled or until \p timeoutMs |
338 | /// milliseconds have elapsed. If \p timeoutMs is 0 then wait fallback to |
339 | /// waiting indefinitely for the event to be signalled. |
340 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
341 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEventFor)( |
342 | onnxEvent event, uint32_t timeoutMs, onnxEventState *eventState, |
343 | onnxStatus *eventStatus, char *outMessage, size_t *outMessageLength) { |
344 | size_t maxMessageLength = *outMessageLength - 1; |
345 | *outMessageLength = 0; |
346 | |
347 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
348 | |
349 | if (!eventState || !eventStatus) { |
350 | return ONNXIFI_STATUS_INVALID_POINTER; |
351 | } |
352 | |
353 | auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event); |
354 | if (!manager.isValid(glowEvent)) { |
355 | return ONNXIFI_STATUS_INVALID_EVENT; |
356 | } |
357 | |
358 | if (timeoutMs == 0) { |
359 | auto res = glowEvent->wait(); |
360 | *eventState = ONNXIFI_EVENT_STATE_SIGNALLED; |
361 | *eventStatus = res; |
362 | } else { |
363 | auto resPair = glowEvent->waitFor(timeoutMs); |
364 | if (resPair.first) { |
365 | *eventState = ONNXIFI_EVENT_STATE_SIGNALLED; |
366 | *eventStatus = resPair.second; |
367 | } else { |
368 | *eventState = ONNXIFI_EVENT_STATE_NONSIGNALLED; |
369 | } |
370 | } |
371 | |
372 | const auto &message = glowEvent->getMessage(); |
373 | if (message.size() > 0) { |
374 | std::strncpy(outMessage, message.c_str(), maxMessageLength); |
375 | // make sure the message is null termininated. strncpy will pad 0 iff |
376 | // message size is smaller than maxMessageLength. |
377 | outMessage[maxMessageLength] = '\0'; |
378 | *outMessageLength = std::min(message.size(), maxMessageLength) + 1; |
379 | } |
380 | |
381 | return ONNXIFI_STATUS_SUCCESS; |
382 | } |
383 | |
384 | /// Query ONNXIFI event state without blocking. |
385 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
386 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetEventState)( |
387 | onnxEvent event, onnxEventState *state) { |
388 | if (!state) { |
389 | return ONNXIFI_STATUS_INVALID_POINTER; |
390 | } |
391 | *state = ONNXIFI_EVENT_STATE_INVALID; |
392 | |
393 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
394 | |
395 | auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event); |
396 | if (!manager.isValid(glowEvent)) { |
397 | return ONNXIFI_STATUS_INVALID_EVENT; |
398 | } |
399 | |
400 | *state = glowEvent->isSignalled() ? ONNXIFI_EVENT_STATE_SIGNALLED |
401 | : ONNXIFI_EVENT_STATE_NONSIGNALLED; |
402 | return ONNXIFI_STATUS_SUCCESS; |
403 | } |
404 | |
405 | /// Deinitialize an ONNXIFI event and release associated resources. |
406 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
407 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseEvent)(onnxEvent event) { |
408 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
409 | |
410 | auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event); |
411 | if (!manager.isValid(glowEvent)) { |
412 | return ONNXIFI_STATUS_INVALID_EVENT; |
413 | } |
414 | |
415 | manager.release(glowEvent); |
416 | |
417 | return ONNXIFI_STATUS_SUCCESS; |
418 | } |
419 | |
420 | /// Parse an ONNXIFI graph and convert it for a particular backend. |
421 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
422 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitGraph)( |
423 | onnxBackend backend, const uint64_t *auxPropertiesList, |
424 | size_t onnxModelSize, const void *onnxModel, uint32_t weightsCount, |
425 | const onnxTensorDescriptorV1 *weightDescriptors, onnxGraph *graph, |
426 | uint32_t maxSeqLength, void *deferredBlobReader) { |
427 | if (!onnxModel || (!weightDescriptors && weightsCount) || !graph) { |
428 | return ONNXIFI_STATUS_INVALID_POINTER; |
429 | } |
430 | if (!onnxModelSize) { |
431 | return ONNXIFI_STATUS_INVALID_SIZE; |
432 | } |
433 | |
434 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
435 | |
436 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend); |
437 | if (!manager.isValid(glowBackend)) { |
438 | return ONNXIFI_STATUS_INVALID_BACKEND; |
439 | } |
440 | |
441 | glow::QuantizationMode quantizationMode; |
442 | if (getenv("GLOW_DUMP_PROFILE" )) { |
443 | quantizationMode = glow::QuantizationMode::Profile; |
444 | } else if (getenv("GLOW_LOAD_PROFILE" )) { |
445 | quantizationMode = glow::QuantizationMode::Quantize; |
446 | } else { |
447 | quantizationMode = glow::QuantizationMode::None; |
448 | } |
449 | |
450 | bool loadingGlowAOT = false; |
451 | if (auxPropertiesList) { |
452 | for (; *auxPropertiesList != ONNXIFI_GRAPH_PROPERTY_NONE; |
453 | auxPropertiesList++) { |
454 | if (*auxPropertiesList == ONNXIFI_OPTIMIZATION_AOT) { |
455 | loadingGlowAOT = true; |
456 | } else { |
457 | return ONNXIFI_STATUS_UNSUPPORTED_PROPERTY; |
458 | } |
459 | } |
460 | } |
461 | |
462 | auto *glowGraph = manager.createGraph(glowBackend, quantizationMode); |
463 | auto ret = glowGraph->initGraph(onnxModel, onnxModelSize, weightsCount, |
464 | weightDescriptors, maxSeqLength, |
465 | deferredBlobReader, loadingGlowAOT); |
466 | if (ret != ONNXIFI_STATUS_SUCCESS) { |
467 | manager.release(glowGraph); |
468 | return ret; |
469 | } |
470 | |
471 | *graph = glowGraph; |
472 | return ONNXIFI_STATUS_SUCCESS; |
473 | } |
474 | |
475 | /// Sanity check for tensor descriptors |
476 | static onnxStatus verifyDescriptors(uint32_t count, |
477 | const onnxTensorDescriptorV1 *descriptors) { |
478 | for (unsigned i = 0; i < count; i++) { |
479 | const auto &descriptor = descriptors[i]; |
480 | if (descriptor.tag != ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1) { |
481 | return ONNXIFI_STATUS_UNSUPPORTED_TAG; |
482 | } |
483 | |
484 | if (descriptor.memoryType != ONNXIFI_MEMORY_TYPE_CPU) { |
485 | return ONNXIFI_STATUS_INVALID_MEMORY_TYPE; |
486 | } |
487 | |
488 | if (!descriptor.buffer) { |
489 | bool hasZeroDims = false; |
490 | for (int i = 0; i < descriptor.dimensions; ++i) { |
491 | if (descriptor.shape[i] == 0) { |
492 | hasZeroDims = true; |
493 | break; |
494 | } |
495 | } |
496 | |
497 | if (!hasZeroDims) { |
498 | LOG(ERROR) << "Bad memory on input " << descriptor.name << " (" << i |
499 | << " out of " << count |
500 | << "). It has no memory buffer, but has " |
501 | << descriptor.dimensions << " dimensions: [" |
502 | << folly::join( |
503 | "," , llvm::ArrayRef<uint64_t>(descriptor.shape, |
504 | descriptor.dimensions)) |
505 | << "]" ; |
506 | return ONNXIFI_STATUS_INVALID_MEMORY_LOCATION; |
507 | } |
508 | } |
509 | } |
510 | |
511 | return ONNXIFI_STATUS_SUCCESS; |
512 | } |
513 | |
514 | /// Binds inputs and outputs of an ONNXIFI graph to specific addresses. |
515 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
516 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetGraphIO)( |
517 | onnxGraph graph, uint32_t inputsCount, |
518 | const onnxTensorDescriptorV1 *inputDescriptors, uint32_t outputsCount, |
519 | const onnxTensorDescriptorV1 *outputDescriptors) { |
520 | LOG(ERROR) << "Use onnxSetIOAndRunGraph instead of onnxSetGraphIO" ; |
521 | return ONNXIFI_STATUS_INTERNAL_ERROR; |
522 | } |
523 | |
524 | /// Asynchronously execute operations in an ONNXIFI graph using pre-specified |
525 | /// locations for inputs and outputs. |
526 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
527 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxRunGraph)( |
528 | onnxGraph graph, const onnxMemoryFenceV1 *inputFence, |
529 | onnxMemoryFenceV1 *outputFence) { |
530 | LOG(ERROR) << "Use onnxSetIOAndRunGraph instead of onnxRunGraph" ; |
531 | return ONNXIFI_STATUS_INTERNAL_ERROR; |
532 | } |
533 | |
534 | /// Binds inputs and outputs of an ONNXIFI graph to specific addresses then |
535 | /// asynchronously execute operations in the graph using the provided |
536 | /// addresses. |
537 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
538 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetIOAndRunGraph)( |
539 | onnxGraph graph, uint32_t inputsCount, |
540 | const onnxTensorDescriptorV1 *inputDescriptors, uint32_t outputsCount, |
541 | const onnxTensorDescriptorV1 *outputDescriptors, |
542 | onnxMemoryFenceV1 *outputFence, onnxTraceEventList *traceEvents) { |
543 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
544 | |
545 | if ((inputsCount && !inputDescriptors) || |
546 | (outputsCount && !outputDescriptors) || !outputFence) { |
547 | LOG(ERROR) << "inputsCount " << inputsCount << ", outputsCount " |
548 | << outputsCount << ", inputDescriptors: " << inputDescriptors |
549 | << ", outputDescriptors: " << outputDescriptors |
550 | << ", outputFence: " << outputFence; |
551 | return ONNXIFI_STATUS_INVALID_POINTER; |
552 | } |
553 | |
554 | // Check output fence is correct type and tag. |
555 | if (outputFence->type != ONNXIFI_SYNCHRONIZATION_EVENT || |
556 | outputFence->tag != ONNXIFI_TAG_MEMORY_FENCE_V1) { |
557 | return ONNXIFI_STATUS_UNSUPPORTED_TAG; |
558 | } |
559 | |
560 | // Check glowGraph is valid. |
561 | auto *glowGraph = static_cast<glow::onnxifi::GraphPtr>(graph); |
562 | if (!manager.isValid(glowGraph)) { |
563 | return ONNXIFI_STATUS_INVALID_GRAPH; |
564 | } |
565 | |
566 | // Initialize outputFence's event. |
567 | auto outputEventInitStatus = GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER( |
568 | onnxInitEvent)(glowGraph->backend(), &outputFence->event); |
569 | if (outputEventInitStatus != ONNXIFI_STATUS_SUCCESS) { |
570 | return outputEventInitStatus; |
571 | } |
572 | |
573 | // Verify inputs. |
574 | auto inputStatus = verifyDescriptors(inputsCount, inputDescriptors); |
575 | if (inputStatus != ONNXIFI_STATUS_SUCCESS) { |
576 | return inputStatus; |
577 | } |
578 | |
579 | // Verify outputs. |
580 | auto outputStatus = verifyDescriptors(outputsCount, outputDescriptors); |
581 | if (outputStatus != ONNXIFI_STATUS_SUCCESS) { |
582 | return outputStatus; |
583 | } |
584 | |
585 | auto *outputEvent = static_cast<glow::onnxifi::EventPtr>(outputFence->event); |
586 | |
587 | // Set graph IO and run async |
588 | return glowGraph->setIOAndRun(inputsCount, inputDescriptors, outputsCount, |
589 | outputDescriptors, outputEvent, traceEvents); |
590 | } |
591 | |
592 | /// Deinitialize an ONNXIFI graph and release associated resources. |
593 | /// It blocks until all in-flight inference operations complete. |
594 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
595 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseGraph)(onnxGraph graph) { |
596 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
597 | |
598 | auto *glowGraph = static_cast<glow::onnxifi::GraphPtr>(graph); |
599 | if (!manager.isValid(glowGraph)) { |
600 | return ONNXIFI_STATUS_INVALID_GRAPH; |
601 | } |
602 | |
603 | manager.release(glowGraph); |
604 | |
605 | return ONNXIFI_STATUS_SUCCESS; |
606 | } |
607 | |
608 | /// Release onnxTraceEvents in \p traceEvents. |
609 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
610 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseTraceEvents)( |
611 | onnxTraceEventList *traceEvents) { |
612 | if (!traceEvents) { |
613 | return ONNXIFI_STATUS_INVALID_POINTER; |
614 | } |
615 | glow::onnxifi::Graph::releaseTraceEvents(traceEvents); |
616 | return ONNXIFI_STATUS_SUCCESS; |
617 | } |
618 | |
619 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
620 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetCurrentBatchSize)( |
621 | int64_t *currentBatchSize) { |
622 | if (!currentBatchSize) { |
623 | return ONNXIFI_STATUS_INVALID_POINTER; |
624 | } |
625 | auto requestData = glow::runtime::RequestData::get(); |
626 | if (!requestData) { |
627 | return ONNXIFI_STATUS_INVALID_STATE; |
628 | } |
629 | *currentBatchSize = requestData->currentBatchSize; |
630 | return ONNXIFI_STATUS_SUCCESS; |
631 | } |
632 | |
633 | /// Set Onnxifi option |
634 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
635 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetOption)(const char *optionName, |
636 | const char *optionValue) { |
637 | if (!optionName || !optionValue) { |
638 | return ONNXIFI_STATUS_INVALID_POINTER; |
639 | } |
640 | onnxStatus ret = ONNXIFI_STATUS_SUCCESS; |
641 | int d = 0; |
642 | if (!strcmp(optionName, "glow_num_devices" )) { |
643 | if (sscanf(optionValue, "%d" , &d) == 1) { |
644 | glow::flags::NumDevices = d; |
645 | } else { |
646 | ret = ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE; |
647 | } |
648 | } else { |
649 | ret = ONNXIFI_STATUS_INVALID_NAME; |
650 | } |
651 | return ret; |
652 | } |
653 | |
654 | /// Get Onnxifi option |
655 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
656 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetOption)( |
657 | const char *optionName, char *optionValue, size_t *optionValueLength) { |
658 | if (!optionName || !optionValue || !optionValueLength) { |
659 | return ONNXIFI_STATUS_INVALID_POINTER; |
660 | } |
661 | onnxStatus ret = ONNXIFI_STATUS_SUCCESS; |
662 | if (!strcmp(optionName, "glow_num_devices" )) { |
663 | int n = snprintf(optionValue, *optionValueLength, "%d" , |
664 | glow::flags::NumDevices); |
665 | if (n < 0) { |
666 | ret = ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE; |
667 | } else if (n < *optionValueLength) { |
668 | *optionValueLength = n; |
669 | } |
670 | } else { |
671 | ret = ONNXIFI_STATUS_INVALID_NAME; |
672 | } |
673 | return ret; |
674 | } |
675 | /// Get pointer to onnxifi extension function with \p name. |
676 | EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
677 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetExtensionFunctionAddress)( |
678 | onnxBackendID backendID, const char *name, |
679 | onnxExtensionFunctionPointer *function) { |
680 | if (!name || !function) { |
681 | return ONNXIFI_STATUS_INVALID_POINTER; |
682 | } |
683 | |
684 | // We don't check backend id for set/get option functions as the options |
685 | // global to Glow. |
686 | static const std::unordered_set<std::string> bypass{"onnxSetOptionFunction" , |
687 | "onnxGetOptionFunction" }; |
688 | if (bypass.find(name) == bypass.end()) { |
689 | auto &manager = glow::onnxifi::GlowOnnxifiManager::get(); |
690 | auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID); |
691 | if (!manager.isValid(glowBackend)) { |
692 | return ONNXIFI_STATUS_INVALID_ID; |
693 | } |
694 | } |
695 | |
696 | // Map of name to onnxExtensionFunctionPointer, one entry for each implemented |
697 | // onnxifi extension. |
698 | // NOTE: when updating this map, also update the response from |
699 | // onnxGetBackendInfo for the ONNXIFI_BACKEND_EXTENSIONS query. |
700 | static const std::unordered_map<std::string, onnxExtensionFunctionPointer> |
701 | extensionMap = { |
702 | {"onnxSetIOAndRunGraphFunction" , |
703 | reinterpret_cast<onnxExtensionFunctionPointer>( |
704 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetIOAndRunGraph))}, |
705 | {"onnxWaitEventForFunction" , |
706 | reinterpret_cast<onnxExtensionFunctionPointer>( |
707 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEventFor))}, |
708 | {"onnxReleaseTraceEventsFunction" , |
709 | reinterpret_cast<onnxExtensionFunctionPointer>( |
710 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseTraceEvents))}, |
711 | {"onnxGetCurrentBatchSizeFunction" , |
712 | reinterpret_cast<onnxExtensionFunctionPointer>( |
713 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetCurrentBatchSize))}, |
714 | {"onnxSetOptionFunction" , |
715 | reinterpret_cast<onnxExtensionFunctionPointer>( |
716 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetOption))}, |
717 | {"onnxGetOptionFunction" , |
718 | reinterpret_cast<onnxExtensionFunctionPointer>( |
719 | GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetOption))}}; |
720 | |
721 | auto extensionIt = extensionMap.find(name); |
722 | |
723 | if (extensionIt == extensionMap.end()) { |
724 | // No function found for the given name. |
725 | return ONNXIFI_STATUS_UNIDENTIFIED_NAME; |
726 | } |
727 | |
728 | *function = extensionIt->second; |
729 | return ONNXIFI_STATUS_SUCCESS; |
730 | } |
731 | |