1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "Base.h"
18#include "GlowOnnxifiManager.h"
19#include "folly/String.h"
20#include "llvm/Support/CommandLine.h"
21
22#include "glow/Flags/Flags.h"
23#include "glow/Importer/ONNXIFIModelLoader.h"
24#include "glow/Runtime/RequestData.h"
25
26#include <cstring>
27#include <glog/logging.h>
28
29/// Allow defining names for onnxifi implementation.
30#ifndef GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER
31#define GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(name) name
32#endif
33
34#define EXTERNC extern "C"
35
36/**
37 * This file contains implementation of the onnxifi interface.
38 * Documentation on the functions implementing onnxifi interface in
39 * this file is very shallow.
40 * Please see more documentation on functions that need to be
41 * implemented: https://github.com/houseroad/foxi/blob/master/foxi/onnxifi.h.
42 */
43
44/// Return stable IDs of available backends on the system.
45/// \param backendIDs output parameter and represents pointer to the memory
46/// where the backend IDs will be returned. If it's NULL,
47/// numBackends will be populated with the number of backends
48/// supported.
49/// \param numBackends input/output parameter.
50/// As an input, it specifies the capacity allocated in the
51/// backendIDs. As an output, it specifies the number of
52/// actual available backends.
53EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
54GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendIDs)(
55 onnxBackendID *backendIDs, size_t *numBackends) {
56 if (!numBackends) {
57 return ONNXIFI_STATUS_INVALID_POINTER;
58 }
59
60 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
61
62 const size_t numBackendsCapacity = *numBackends;
63
64 using namespace glow::runtime;
65 using namespace glow::onnxifi;
66 const bool withCPU = DeviceManager::numDevices("CPU") > 0;
67 const bool withHabana = DeviceManager::numDevices("Habana") > 0;
68 const bool withNNPI = DeviceManager::numDevices("NNPI") > 0;
69#ifdef GLOW_EXTRABACKEND
70#define V(name) with##name
71#define makeVar(name) V(name)
72
73#define Q(name) #name
74#define makeQuote(name) Q(name)
75 const bool makeVar(GLOW_EXTRABACKEND) =
76 DeviceManager::numDevices(makeQuote(GLOW_EXTRABACKEND)) > 0;
77#endif
78
79 // Only return quantization backend if GLOW_DUMP_PROFILE.
80 if (getenv("GLOW_DUMP_PROFILE")) {
81 *numBackends = 1;
82
83 // In case backendIDs is nullptr or does not have enough capacity just
84 // return the total number of supported backends.
85 if (numBackendsCapacity < *numBackends || !backendIDs) {
86 return ONNXIFI_STATUS_FALLBACK;
87 }
88
89 auto backendName = glow::onnxifi::flags::BackendName.empty()
90 ? "Interpreter"
91 : glow::onnxifi::flags::BackendName;
92 LOG(INFO) << "ONNXIFI: Executing on " << backendName << " Glow backend";
93 auto *quantizationBackendC2 =
94 manager.createBackend(backendName,
95 /*useOnnx*/ false, /*forQuantization*/ true);
96 backendIDs[0] = quantizationBackendC2;
97 } else {
98 *numBackends = 2;
99
100 auto backendName = glow::onnxifi::flags::BackendName;
101
102 if (backendName.empty()) {
103 if (withNNPI) {
104 backendName = "NNPI";
105 } else if (withHabana) {
106 backendName = "Habana";
107#ifdef GLOW_EXTRABACKEND
108 } else if (makeVar(GLOW_EXTRABACKEND)) {
109 backendName = makeQuote(GLOW_EXTRABACKEND);
110#undef V
111#undef makeVar
112#undef Q
113#undef makeQuote
114#endif
115 } else if (withCPU) {
116 backendName = "CPU";
117 } else {
118 backendName = "Interpreter";
119 }
120 }
121
122 // In case backendIDs is nullptr or does not have enough capacity just
123 // return the total number of supported backends.
124 if (numBackendsCapacity < *numBackends || !backendIDs) {
125 return ONNXIFI_STATUS_FALLBACK;
126 }
127
128 LOG(INFO) << "ONNXIFI: Executing on " << backendName << " Glow backend";
129
130 backendIDs[0] = manager.createBackend(backendName,
131 /*useOnnx*/ false);
132 backendIDs[1] = manager.createBackend(backendName,
133 /*useOnnx*/ true);
134 }
135
136 return ONNXIFI_STATUS_SUCCESS;
137}
138
139/// Deinitialize ONNXIFI backend ID and release associated resources.
140/// Caller is responsible to release objects associated with the backend ID
141/// (onnxBackend, onnxGraph, onnxEvent) before calling this function.
142EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
143GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseBackendID)(
144 onnxBackendID backendID) {
145 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
146
147 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID);
148 if (!manager.isValid(glowBackend)) {
149 return ONNXIFI_STATUS_INVALID_ID;
150 }
151
152 manager.release(glowBackend);
153
154 return ONNXIFI_STATUS_SUCCESS;
155}
156
157static onnxStatus setBackendInfoString(void *infoValue, size_t *infoValueSize,
158 const char *str) {
159 size_t len = strlen(str) + 1;
160 if (!infoValue || *infoValueSize < len) {
161 *infoValueSize = len;
162 return ONNXIFI_STATUS_FALLBACK;
163 }
164
165 strncpy((char *)infoValue, str, len);
166 *infoValueSize = len;
167 return ONNXIFI_STATUS_SUCCESS;
168}
169
170static onnxStatus setBackendInfoUInt64(void *infoValue, size_t *infoValueSize,
171 uint64_t value) {
172 if (!infoValue || *infoValueSize < sizeof(uint64_t)) {
173 *infoValueSize = sizeof(uint64_t);
174 return ONNXIFI_STATUS_FALLBACK;
175 }
176
177 *(uint64_t *)(infoValue) = value;
178 *infoValueSize = sizeof(uint64_t);
179 return ONNXIFI_STATUS_SUCCESS;
180}
181
182/// Query high-level information about the backend and its target device.
183EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
184GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendInfo)(
185 onnxBackendID backendID, onnxBackendInfo infoType, void *infoValue,
186 size_t *infoValueSize) {
187 if (!infoValueSize) {
188 return ONNXIFI_STATUS_INVALID_POINTER;
189 }
190
191 if (!infoValue) {
192 return ONNXIFI_STATUS_INVALID_POINTER;
193 }
194
195 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
196
197 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID);
198 if (!manager.isValid(glowBackend)) {
199 return ONNXIFI_STATUS_INVALID_ID;
200 }
201
202 // TODO: support more info type values. Here is the minimal required
203 // subset of info types.
204 switch (infoType) {
205 case ONNXIFI_BACKEND_NAME:
206 return setBackendInfoString(infoValue, infoValueSize, "Glow");
207 case ONNXIFI_BACKEND_VENDOR:
208 return setBackendInfoString(infoValue, infoValueSize, "PyTorch");
209 case ONNXIFI_BACKEND_VERSION:
210 return setBackendInfoString(infoValue, infoValueSize, "1.0.0");
211 case ONNXIFI_BACKEND_DEVICE:
212 return setBackendInfoString(infoValue, infoValueSize,
213 glowBackend->getUseOnnx() ? "Glow Onnx"
214 : "Glow Caffe2");
215 case ONNXIFI_BACKEND_MEMORY_TYPES:
216 return setBackendInfoUInt64(infoValue, infoValueSize,
217 ONNXIFI_MEMORY_TYPE_CPU);
218 case ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES:
219 return setBackendInfoUInt64(infoValue, infoValueSize,
220 ONNXIFI_SYNCHRONIZATION_EVENT);
221 case ONNXIFI_BACKEND_EXTENSIONS:
222 return setBackendInfoString(
223 infoValue, infoValueSize,
224 "onnxSetIOAndRunGraphFunction onnxWaitEventForFunction "
225 "onnxReleaseTraceEventsFunction onnxGetCurrentBatchSizeFunction");
226 default:
227 return ONNXIFI_STATUS_UNSUPPORTED_PROPERTY;
228 }
229}
230
231/// Query if an ONNX model graph is compatible with the backend.
232EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
233GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetBackendCompatibility)(
234 onnxBackendID backendID, size_t onnxModelSize, const void *onnxModel) {
235 if (!onnxModel) {
236 return ONNXIFI_STATUS_INVALID_POINTER;
237 }
238
239 if (!onnxModelSize) {
240 return ONNXIFI_STATUS_INVALID_SIZE;
241 }
242
243 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
244
245 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID);
246 if (!manager.isValid(glowBackend)) {
247 return ONNXIFI_STATUS_INVALID_ID;
248 }
249
250 return glowBackend->checkGraphCompatibility(onnxModel, onnxModelSize);
251}
252
253/// Initialize an ONNXIFI backend.
254EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
255GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitBackend)(
256 onnxBackendID backendID, const uint64_t *auxpropertiesList,
257 onnxBackend *backend) {
258 if (!backend) {
259 return ONNXIFI_STATUS_INVALID_POINTER;
260 }
261
262 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
263
264 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID);
265 if (!manager.isValid(glowBackend)) {
266 return ONNXIFI_STATUS_INVALID_ID;
267 }
268
269 *backend = glowBackend;
270
271 return ONNXIFI_STATUS_SUCCESS;
272}
273
274/// Deinitialize an ONNXIFI backend and release associated resources.
275EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
276GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseBackend)(onnxBackend backend) {
277 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
278
279 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend);
280 if (!manager.isValid(glowBackend)) {
281 return ONNXIFI_STATUS_INVALID_BACKEND;
282 }
283
284 return ONNXIFI_STATUS_SUCCESS;
285}
286
287/// Initialize a single-shot ONNXIFI event.
288EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
289GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitEvent)(onnxBackend backend,
290 onnxEvent *event) {
291 if (!event) {
292 return ONNXIFI_STATUS_INVALID_POINTER;
293 }
294
295 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
296
297 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend);
298 if (!manager.isValid(glowBackend)) {
299 return ONNXIFI_STATUS_INVALID_BACKEND;
300 }
301
302 *event = manager.createEvent();
303
304 return ONNXIFI_STATUS_SUCCESS;
305}
306
307/// Change the state of the ONNXIFI event \p event to signalled.
308EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
309GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSignalEvent)(onnxEvent event) {
310 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
311
312 auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event);
313 if (!manager.isValid(glowEvent)) {
314 return ONNXIFI_STATUS_INVALID_EVENT;
315 }
316
317 if (!glowEvent->signal(ONNXIFI_STATUS_SUCCESS)) {
318 return ONNXIFI_STATUS_INVALID_STATE;
319 }
320
321 return ONNXIFI_STATUS_SUCCESS;
322}
323
324/// Wait until an ONNXIFI \p event is signalled.
325EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
326GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEvent)(onnxEvent event) {
327 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
328
329 auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event);
330 if (!manager.isValid(glowEvent)) {
331 return ONNXIFI_STATUS_INVALID_EVENT;
332 }
333
334 return glowEvent->wait();
335}
336
337/// Wait until an ONNXIFI \p event is signalled or until \p timeoutMs
338/// milliseconds have elapsed. If \p timeoutMs is 0 then wait fallback to
339/// waiting indefinitely for the event to be signalled.
340EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
341GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEventFor)(
342 onnxEvent event, uint32_t timeoutMs, onnxEventState *eventState,
343 onnxStatus *eventStatus, char *outMessage, size_t *outMessageLength) {
344 size_t maxMessageLength = *outMessageLength - 1;
345 *outMessageLength = 0;
346
347 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
348
349 if (!eventState || !eventStatus) {
350 return ONNXIFI_STATUS_INVALID_POINTER;
351 }
352
353 auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event);
354 if (!manager.isValid(glowEvent)) {
355 return ONNXIFI_STATUS_INVALID_EVENT;
356 }
357
358 if (timeoutMs == 0) {
359 auto res = glowEvent->wait();
360 *eventState = ONNXIFI_EVENT_STATE_SIGNALLED;
361 *eventStatus = res;
362 } else {
363 auto resPair = glowEvent->waitFor(timeoutMs);
364 if (resPair.first) {
365 *eventState = ONNXIFI_EVENT_STATE_SIGNALLED;
366 *eventStatus = resPair.second;
367 } else {
368 *eventState = ONNXIFI_EVENT_STATE_NONSIGNALLED;
369 }
370 }
371
372 const auto &message = glowEvent->getMessage();
373 if (message.size() > 0) {
374 std::strncpy(outMessage, message.c_str(), maxMessageLength);
375 // make sure the message is null termininated. strncpy will pad 0 iff
376 // message size is smaller than maxMessageLength.
377 outMessage[maxMessageLength] = '\0';
378 *outMessageLength = std::min(message.size(), maxMessageLength) + 1;
379 }
380
381 return ONNXIFI_STATUS_SUCCESS;
382}
383
384/// Query ONNXIFI event state without blocking.
385EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
386GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetEventState)(
387 onnxEvent event, onnxEventState *state) {
388 if (!state) {
389 return ONNXIFI_STATUS_INVALID_POINTER;
390 }
391 *state = ONNXIFI_EVENT_STATE_INVALID;
392
393 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
394
395 auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event);
396 if (!manager.isValid(glowEvent)) {
397 return ONNXIFI_STATUS_INVALID_EVENT;
398 }
399
400 *state = glowEvent->isSignalled() ? ONNXIFI_EVENT_STATE_SIGNALLED
401 : ONNXIFI_EVENT_STATE_NONSIGNALLED;
402 return ONNXIFI_STATUS_SUCCESS;
403}
404
405/// Deinitialize an ONNXIFI event and release associated resources.
406EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
407GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseEvent)(onnxEvent event) {
408 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
409
410 auto *glowEvent = static_cast<glow::onnxifi::EventPtr>(event);
411 if (!manager.isValid(glowEvent)) {
412 return ONNXIFI_STATUS_INVALID_EVENT;
413 }
414
415 manager.release(glowEvent);
416
417 return ONNXIFI_STATUS_SUCCESS;
418}
419
420/// Parse an ONNXIFI graph and convert it for a particular backend.
421EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
422GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxInitGraph)(
423 onnxBackend backend, const uint64_t *auxPropertiesList,
424 size_t onnxModelSize, const void *onnxModel, uint32_t weightsCount,
425 const onnxTensorDescriptorV1 *weightDescriptors, onnxGraph *graph,
426 uint32_t maxSeqLength, void *deferredBlobReader) {
427 if (!onnxModel || (!weightDescriptors && weightsCount) || !graph) {
428 return ONNXIFI_STATUS_INVALID_POINTER;
429 }
430 if (!onnxModelSize) {
431 return ONNXIFI_STATUS_INVALID_SIZE;
432 }
433
434 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
435
436 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backend);
437 if (!manager.isValid(glowBackend)) {
438 return ONNXIFI_STATUS_INVALID_BACKEND;
439 }
440
441 glow::QuantizationMode quantizationMode;
442 if (getenv("GLOW_DUMP_PROFILE")) {
443 quantizationMode = glow::QuantizationMode::Profile;
444 } else if (getenv("GLOW_LOAD_PROFILE")) {
445 quantizationMode = glow::QuantizationMode::Quantize;
446 } else {
447 quantizationMode = glow::QuantizationMode::None;
448 }
449
450 bool loadingGlowAOT = false;
451 if (auxPropertiesList) {
452 for (; *auxPropertiesList != ONNXIFI_GRAPH_PROPERTY_NONE;
453 auxPropertiesList++) {
454 if (*auxPropertiesList == ONNXIFI_OPTIMIZATION_AOT) {
455 loadingGlowAOT = true;
456 } else {
457 return ONNXIFI_STATUS_UNSUPPORTED_PROPERTY;
458 }
459 }
460 }
461
462 auto *glowGraph = manager.createGraph(glowBackend, quantizationMode);
463 auto ret = glowGraph->initGraph(onnxModel, onnxModelSize, weightsCount,
464 weightDescriptors, maxSeqLength,
465 deferredBlobReader, loadingGlowAOT);
466 if (ret != ONNXIFI_STATUS_SUCCESS) {
467 manager.release(glowGraph);
468 return ret;
469 }
470
471 *graph = glowGraph;
472 return ONNXIFI_STATUS_SUCCESS;
473}
474
475/// Sanity check for tensor descriptors
476static onnxStatus verifyDescriptors(uint32_t count,
477 const onnxTensorDescriptorV1 *descriptors) {
478 for (unsigned i = 0; i < count; i++) {
479 const auto &descriptor = descriptors[i];
480 if (descriptor.tag != ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1) {
481 return ONNXIFI_STATUS_UNSUPPORTED_TAG;
482 }
483
484 if (descriptor.memoryType != ONNXIFI_MEMORY_TYPE_CPU) {
485 return ONNXIFI_STATUS_INVALID_MEMORY_TYPE;
486 }
487
488 if (!descriptor.buffer) {
489 bool hasZeroDims = false;
490 for (int i = 0; i < descriptor.dimensions; ++i) {
491 if (descriptor.shape[i] == 0) {
492 hasZeroDims = true;
493 break;
494 }
495 }
496
497 if (!hasZeroDims) {
498 LOG(ERROR) << "Bad memory on input " << descriptor.name << " (" << i
499 << " out of " << count
500 << "). It has no memory buffer, but has "
501 << descriptor.dimensions << " dimensions: ["
502 << folly::join(
503 ",", llvm::ArrayRef<uint64_t>(descriptor.shape,
504 descriptor.dimensions))
505 << "]";
506 return ONNXIFI_STATUS_INVALID_MEMORY_LOCATION;
507 }
508 }
509 }
510
511 return ONNXIFI_STATUS_SUCCESS;
512}
513
514/// Binds inputs and outputs of an ONNXIFI graph to specific addresses.
515EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
516GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetGraphIO)(
517 onnxGraph graph, uint32_t inputsCount,
518 const onnxTensorDescriptorV1 *inputDescriptors, uint32_t outputsCount,
519 const onnxTensorDescriptorV1 *outputDescriptors) {
520 LOG(ERROR) << "Use onnxSetIOAndRunGraph instead of onnxSetGraphIO";
521 return ONNXIFI_STATUS_INTERNAL_ERROR;
522}
523
524/// Asynchronously execute operations in an ONNXIFI graph using pre-specified
525/// locations for inputs and outputs.
526EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
527GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxRunGraph)(
528 onnxGraph graph, const onnxMemoryFenceV1 *inputFence,
529 onnxMemoryFenceV1 *outputFence) {
530 LOG(ERROR) << "Use onnxSetIOAndRunGraph instead of onnxRunGraph";
531 return ONNXIFI_STATUS_INTERNAL_ERROR;
532}
533
534/// Binds inputs and outputs of an ONNXIFI graph to specific addresses then
535/// asynchronously execute operations in the graph using the provided
536/// addresses.
537EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
538GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetIOAndRunGraph)(
539 onnxGraph graph, uint32_t inputsCount,
540 const onnxTensorDescriptorV1 *inputDescriptors, uint32_t outputsCount,
541 const onnxTensorDescriptorV1 *outputDescriptors,
542 onnxMemoryFenceV1 *outputFence, onnxTraceEventList *traceEvents) {
543 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
544
545 if ((inputsCount && !inputDescriptors) ||
546 (outputsCount && !outputDescriptors) || !outputFence) {
547 LOG(ERROR) << "inputsCount " << inputsCount << ", outputsCount "
548 << outputsCount << ", inputDescriptors: " << inputDescriptors
549 << ", outputDescriptors: " << outputDescriptors
550 << ", outputFence: " << outputFence;
551 return ONNXIFI_STATUS_INVALID_POINTER;
552 }
553
554 // Check output fence is correct type and tag.
555 if (outputFence->type != ONNXIFI_SYNCHRONIZATION_EVENT ||
556 outputFence->tag != ONNXIFI_TAG_MEMORY_FENCE_V1) {
557 return ONNXIFI_STATUS_UNSUPPORTED_TAG;
558 }
559
560 // Check glowGraph is valid.
561 auto *glowGraph = static_cast<glow::onnxifi::GraphPtr>(graph);
562 if (!manager.isValid(glowGraph)) {
563 return ONNXIFI_STATUS_INVALID_GRAPH;
564 }
565
566 // Initialize outputFence's event.
567 auto outputEventInitStatus = GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(
568 onnxInitEvent)(glowGraph->backend(), &outputFence->event);
569 if (outputEventInitStatus != ONNXIFI_STATUS_SUCCESS) {
570 return outputEventInitStatus;
571 }
572
573 // Verify inputs.
574 auto inputStatus = verifyDescriptors(inputsCount, inputDescriptors);
575 if (inputStatus != ONNXIFI_STATUS_SUCCESS) {
576 return inputStatus;
577 }
578
579 // Verify outputs.
580 auto outputStatus = verifyDescriptors(outputsCount, outputDescriptors);
581 if (outputStatus != ONNXIFI_STATUS_SUCCESS) {
582 return outputStatus;
583 }
584
585 auto *outputEvent = static_cast<glow::onnxifi::EventPtr>(outputFence->event);
586
587 // Set graph IO and run async
588 return glowGraph->setIOAndRun(inputsCount, inputDescriptors, outputsCount,
589 outputDescriptors, outputEvent, traceEvents);
590}
591
592/// Deinitialize an ONNXIFI graph and release associated resources.
593/// It blocks until all in-flight inference operations complete.
594EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
595GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseGraph)(onnxGraph graph) {
596 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
597
598 auto *glowGraph = static_cast<glow::onnxifi::GraphPtr>(graph);
599 if (!manager.isValid(glowGraph)) {
600 return ONNXIFI_STATUS_INVALID_GRAPH;
601 }
602
603 manager.release(glowGraph);
604
605 return ONNXIFI_STATUS_SUCCESS;
606}
607
608/// Release onnxTraceEvents in \p traceEvents.
609EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
610GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseTraceEvents)(
611 onnxTraceEventList *traceEvents) {
612 if (!traceEvents) {
613 return ONNXIFI_STATUS_INVALID_POINTER;
614 }
615 glow::onnxifi::Graph::releaseTraceEvents(traceEvents);
616 return ONNXIFI_STATUS_SUCCESS;
617}
618
619EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
620GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetCurrentBatchSize)(
621 int64_t *currentBatchSize) {
622 if (!currentBatchSize) {
623 return ONNXIFI_STATUS_INVALID_POINTER;
624 }
625 auto requestData = glow::runtime::RequestData::get();
626 if (!requestData) {
627 return ONNXIFI_STATUS_INVALID_STATE;
628 }
629 *currentBatchSize = requestData->currentBatchSize;
630 return ONNXIFI_STATUS_SUCCESS;
631}
632
633/// Set Onnxifi option
634EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
635GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetOption)(const char *optionName,
636 const char *optionValue) {
637 if (!optionName || !optionValue) {
638 return ONNXIFI_STATUS_INVALID_POINTER;
639 }
640 onnxStatus ret = ONNXIFI_STATUS_SUCCESS;
641 int d = 0;
642 if (!strcmp(optionName, "glow_num_devices")) {
643 if (sscanf(optionValue, "%d", &d) == 1) {
644 glow::flags::NumDevices = d;
645 } else {
646 ret = ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE;
647 }
648 } else {
649 ret = ONNXIFI_STATUS_INVALID_NAME;
650 }
651 return ret;
652}
653
654/// Get Onnxifi option
655EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
656GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetOption)(
657 const char *optionName, char *optionValue, size_t *optionValueLength) {
658 if (!optionName || !optionValue || !optionValueLength) {
659 return ONNXIFI_STATUS_INVALID_POINTER;
660 }
661 onnxStatus ret = ONNXIFI_STATUS_SUCCESS;
662 if (!strcmp(optionName, "glow_num_devices")) {
663 int n = snprintf(optionValue, *optionValueLength, "%d",
664 glow::flags::NumDevices);
665 if (n < 0) {
666 ret = ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE;
667 } else if (n < *optionValueLength) {
668 *optionValueLength = n;
669 }
670 } else {
671 ret = ONNXIFI_STATUS_INVALID_NAME;
672 }
673 return ret;
674}
675/// Get pointer to onnxifi extension function with \p name.
676EXTERNC ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
677GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetExtensionFunctionAddress)(
678 onnxBackendID backendID, const char *name,
679 onnxExtensionFunctionPointer *function) {
680 if (!name || !function) {
681 return ONNXIFI_STATUS_INVALID_POINTER;
682 }
683
684 // We don't check backend id for set/get option functions as the options
685 // global to Glow.
686 static const std::unordered_set<std::string> bypass{"onnxSetOptionFunction",
687 "onnxGetOptionFunction"};
688 if (bypass.find(name) == bypass.end()) {
689 auto &manager = glow::onnxifi::GlowOnnxifiManager::get();
690 auto *glowBackend = static_cast<glow::onnxifi::BackendPtr>(backendID);
691 if (!manager.isValid(glowBackend)) {
692 return ONNXIFI_STATUS_INVALID_ID;
693 }
694 }
695
696 // Map of name to onnxExtensionFunctionPointer, one entry for each implemented
697 // onnxifi extension.
698 // NOTE: when updating this map, also update the response from
699 // onnxGetBackendInfo for the ONNXIFI_BACKEND_EXTENSIONS query.
700 static const std::unordered_map<std::string, onnxExtensionFunctionPointer>
701 extensionMap = {
702 {"onnxSetIOAndRunGraphFunction",
703 reinterpret_cast<onnxExtensionFunctionPointer>(
704 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetIOAndRunGraph))},
705 {"onnxWaitEventForFunction",
706 reinterpret_cast<onnxExtensionFunctionPointer>(
707 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxWaitEventFor))},
708 {"onnxReleaseTraceEventsFunction",
709 reinterpret_cast<onnxExtensionFunctionPointer>(
710 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxReleaseTraceEvents))},
711 {"onnxGetCurrentBatchSizeFunction",
712 reinterpret_cast<onnxExtensionFunctionPointer>(
713 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetCurrentBatchSize))},
714 {"onnxSetOptionFunction",
715 reinterpret_cast<onnxExtensionFunctionPointer>(
716 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxSetOption))},
717 {"onnxGetOptionFunction",
718 reinterpret_cast<onnxExtensionFunctionPointer>(
719 GLOW_ONNXIFI_LIBRARY_FUNCTION_WRAPPER(onnxGetOption))}};
720
721 auto extensionIt = extensionMap.find(name);
722
723 if (extensionIt == extensionMap.end()) {
724 // No function found for the given name.
725 return ONNXIFI_STATUS_UNIDENTIFIED_NAME;
726 }
727
728 *function = extensionIt->second;
729 return ONNXIFI_STATUS_SUCCESS;
730}
731