1#ifndef ONNXIFI_H
2#define ONNXIFI_H 1
3
4#ifdef __cplusplus
5extern "C" {
6#endif
7
8#if defined(_WIN32) && defined(_M_IX86)
9/* Windows x86 */
10#define ONNXIFI_ABI __stdcall
11#elif defined(__i386__)
12/* Linux x86 */
13#define ONNXIFI_ABI __attribute__((__cdecl__))
14#else
15#define ONNXIFI_ABI
16#endif
17
18#ifndef ONNXIFI_PUBLIC
19#if defined(__ELF__)
20#define ONNXIFI_PUBLIC __attribute__((__visibility__("default")))
21#elif defined(__MACH__)
22#define ONNXIFI_PUBLIC __attribute__((__visibility__("default")))
23#elif defined(_WIN32) && defined(__GNUC__)
24#ifdef ONNXIFI_BUILD_LIBRARY
25#define ONNXIFI_PUBLIC __attribute__((__dllexport__))
26#else
27#define ONNXIFI_PUBLIC __attribute__((__dllimport__))
28#endif
29#elif defined(_WIN32)
30#ifdef ONNXIFI_BUILD_LIBRARY
31#define ONNXIFI_PUBLIC __declspec(dllexport)
32#else
33#define ONNXIFI_PUBLIC __declspec(dllimport)
34#endif
35#else
36#define ONNXIFI_PUBLIC
37#endif
38#endif
39
40#ifndef ONNXIFI_CHECK_RESULT
41 #if defined(__GNUC__) && (__GNUC__ >= 4)
42 #define ONNXIFI_CHECK_RESULT __attribute__((__warn_unused_result__))
43 #elif defined(_MSC_VER) && (_MSC_VER >= 1700)
44 #define ONNXIFI_CHECK_RESULT _Check_return_
45 #else
46 #define ONNXIFI_CHECK_RESULT
47 #endif
48#endif
49
50#include <stddef.h>
51
52#if !defined(ONNXIFI_NO_STDINT_H)
53#if defined(_MSC_VER) && (_MSC_VER < 1600)
54typedef signed __int8 int8_t;
55typedef unsigned __int8 uint8_t;
56typedef signed __int16 int16_t;
57typedef unsigned __int16 uint16_t;
58typedef signed __int32 int32_t;
59typedef unsigned __int32 uint32_t;
60typedef signed __int64 int64_t;
61typedef unsigned __int64 uint64_t;
62#else
63#include <stdint.h>
64#endif
65#endif /* !defined(ONNXIFI_NO_STDINT_H) */
66
67/**
68 * Opaque ONNXIFI backend ID.
69 *
70 * ONNXIFI backend is a combination of software layer and hardware device used
71 * to run an ONNX graph. Backend ID uniquely identifies a backend for the life-
72 * time of the process (i.e. no two hardware devices, software layers, or
73 * combinations of both can have the same backend ID). Backend ID stays valid
74 * even if the hardware device used by the backend disconnects from the system.
75 */
76typedef void* onnxBackendID;
77/**
78 * Opaque ONNXIFI backend handle.
79 * ONNXIFI backend is a combination of software layer and hardware device used
80 * to run an ONNX graph.
81 */
82typedef void* onnxBackend;
83/** Opaque ONNXIFI graph handle. */
84typedef void* onnxGraph;
85/** Opaque ONNXIFI even handle. */
86typedef void* onnxEvent;
87
88/** Return code for ONNXIFI functions */
89typedef int32_t onnxStatus;
90/**
91 * Type for enumeration values.
92 *
93 * The low 32 bits are reserved for standardized ONNXIFI values.
94 * The high 32 bits are reserved for vendor-specific extensions. Applications
95 * must check for specific vendor extensions before interpreting these bits.
96 */
97typedef uint64_t onnxEnum;
98/**
99 * Type for bit fields.
100 *
101 * The low 32 bits are reserved for standardized ONNXIFI values.
102 * The high 32 bits are reserved for vendor-specific extensions. Applications
103 * must check for specific vendor extensions before interpreting these bits.
104 */
105typedef uint64_t onnxBitfield;
106/**
107 * Type for pointers or handles for memory buffers.
108 * This type is intended to work not only for CPU-addressable memory, but also
109 * for device memory. uint64_t ensures the API can accomodate Vulkan buffers.
110 */
111typedef uint64_t onnxPointer;
112
113#define ONNXIFI_STATUS_SUCCESS 0x0000
114#define ONNXIFI_STATUS_FALLBACK 0x0001
115#define ONNXIFI_STATUS_INVALID_ID 0x0101
116#define ONNXIFI_STATUS_INVALID_SIZE 0x0102
117#define ONNXIFI_STATUS_INVALID_POINTER 0x0103
118#define ONNXIFI_STATUS_INVALID_PROTOBUF 0x0104
119#define ONNXIFI_STATUS_INVALID_MODEL 0x0105
120#define ONNXIFI_STATUS_INVALID_BACKEND 0x0106
121#define ONNXIFI_STATUS_INVALID_GRAPH 0x0107
122#define ONNXIFI_STATUS_INVALID_EVENT 0x0108
123#define ONNXIFI_STATUS_INVALID_STATE 0x0109
124#define ONNXIFI_STATUS_INVALID_NAME 0x010A
125#define ONNXIFI_STATUS_INVALID_SHAPE 0x010B
126#define ONNXIFI_STATUS_INVALID_DATATYPE 0x010C
127#define ONNXIFI_STATUS_INVALID_MEMORY_TYPE 0x010D
128#define ONNXIFI_STATUS_INVALID_MEMORY_LOCATION 0x010E
129#define ONNXIFI_STATUS_INVALID_FENCE_TYPE 0x010F
130#define ONNXIFI_STATUS_INVALID_PROPERTY 0x0110
131#define ONNXIFI_STATUS_UNSUPPORTED_TAG 0x0201
132#define ONNXIFI_STATUS_UNSUPPORTED_VERSION 0x0202
133#define ONNXIFI_STATUS_UNSUPPORTED_OPERATOR 0x0203
134#define ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE 0x0204
135#define ONNXIFI_STATUS_UNSUPPORTED_SHAPE 0x0205
136#define ONNXIFI_STATUS_UNSUPPORTED_DATATYPE 0x0206
137#define ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE 0x0207
138#define ONNXIFI_STATUS_UNSUPPORTED_FENCE_TYPE 0x0208
139#define ONNXIFI_STATUS_UNSUPPORTED_PROPERTY 0x0209
140#define ONNXIFI_STATUS_UNIDENTIFIED_NAME 0x0301
141#define ONNXIFI_STATUS_MISMATCHING_SHAPE 0x0302
142#define ONNXIFI_STATUS_MISMATCHING_DATATYPE 0x0303
143#define ONNXIFI_STATUS_NO_SYSTEM_MEMORY 0x0401
144#define ONNXIFI_STATUS_NO_DEVICE_MEMORY 0x0402
145#define ONNXIFI_STATUS_NO_SYSTEM_RESOURCES 0x0403
146#define ONNXIFI_STATUS_NO_DEVICE_RESOURCES 0x0404
147#define ONNXIFI_STATUS_BACKEND_UNAVAILABLE 0x0405
148#define ONNXIFI_STATUS_INTERNAL_ERROR 0x0406
149#define ONNXIFI_STATUS_FATAL_ERROR 0x0407
150
151/**
152 * State of an ONNXIFI event object.
153 *
154 * Possible values:
155 * ONNXIFI_EVENT_STATE_INVALID
156 * ONNXIFI_EVENT_STATE_NONSIGNALLED
157 * ONNXIFI_EVENT_STATE_SIGNALLED
158 */
159typedef int32_t onnxEventState;
160
161/**
162 * State for an invalid onnxEvent.
163 */
164#define ONNXIFI_EVENT_STATE_INVALID 0
165/**
166 * Non-signalled onnxEvent state.
167 * onnxInitEvent creates events in non-signalled state.
168 */
169#define ONNXIFI_EVENT_STATE_NONSIGNALLED 0x16BD
170/**
171 * Signalled onnxEvent state.
172 * onnxSignalEvent changes event state to signalled.
173 */
174#define ONNXIFI_EVENT_STATE_SIGNALLED 0x3395
175
176/** Special-purpose accelerator for neural network */
177#define ONNXIFI_DEVICE_TYPE_NPU 0x01
178/** Digital signal processor */
179#define ONNXIFI_DEVICE_TYPE_DSP 0x02
180/** Graphics accelerator */
181#define ONNXIFI_DEVICE_TYPE_GPU 0x04
182/** General-purpose central processor */
183#define ONNXIFI_DEVICE_TYPE_CPU 0x08
184/** Field-programmable gate array */
185#define ONNXIFI_DEVICE_TYPE_FPGA 0x10
186/**
187 * Heterogeneous backend whichinternally arbitrates or distributes work between
188 * multiple device types.
189 */
190#define ONNXIFI_DEVICE_TYPE_HETEROGENEOUS 0x20
191
192/**
193 * The backend supports multi-threaded access to ONNXIFI backend, graph, and
194 * event objects. E.g. onnxInitGraph can be called on a different thread than
195 * onnxInitBackend.
196 *
197 * If this capability it not indicated, ONNXIFI backend, graph, and event
198 * objects that relate to the backend must always be used on the same thread
199 * where the backend object was initialized.
200 */
201#define ONNXIFI_CAPABILITY_THREAD_SAFE 0x01
202/**
203 * The backend supports ONNX graphs with symbolic variables in the outer
204 * shape dimension (batch size), using TensorShapeProto.dim_param for
205 * ModelProto.graph.input.type.shape or ModelProto.graph.output.type.shape.
206 *
207 * The exact numerical value of the of all input and output tensors must be specified
208 * in the onnxSetGraphIO call(s).
209 */
210#define ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE 0x02
211/**
212 * The backend supports ONNX graphs with symbolic variables in the all
213 * shape dimensions, using TensorShapeProto.dim_param for
214 * ModelProto.graph.input.type.shape or ModelProto.graph.output.type.shape.
215 *
216 * Backends with this capability also MUST support
217 * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE capability.
218 *
219 * The exact numerical shape of all input and output tensors must be specified
220 * in the onnxSetGraphIO call(s).
221 */
222#define ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS 0x04
223/**
224 * The backend supports ONNX graphs with data-dependent outer shape dimension
225 * (batch size) of graph outputs. The ONNX graph would specify unknown outer
226 * shape dimension (batch size) using symbolic variables, so this capability
227 * requires ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE support.
228 *
229 * For outputs with data-dependent outer shape dimension (batch size) the value
230 * specified in onnxSetGraphIO call is interpreted as the upper limit. The exact
231 * numerical batch size of the output can be retrieved by attaching a Shape
232 * operator to the tensor with data-dependent shape and reading its output
233 * through ONNXIFI.
234 */
235#define ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE 0x08
236/**
237 * The backend supports ONNX graphs with data-dependent output shapes.
238 * The ONNX graph would specify unknown output shapes using symbolic variables,
239 * so this capability requires ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS support.
240 *
241 * Backends with this capability also MUST support
242 * ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE capability.
243 *
244 * For outputs with data-dependent shapes the shape specified in onnxSetGraphIO
245 * call is interpreted as the upper limit. The exact numerical shape of the
246 * output can be retrieved by attaching a Shape operator to the tensor with
247 * data-dependent shape and reading its output through ONNXIFI.
248 */
249#define ONNXIFI_CAPABILITY_VARIABLE_SIZE_OUTPUTS 0x10
250/**
251 * The backend uses a hot-pluggable device, and can be disconnected at any time.
252 *
253 * If the underlying device disconnects from the system, subsequent operations
254 * with the backend, or objects created on the backend, will fail with
255 * ONNXIFI_STATUS_BACKEND_UNAVAILABLE status code.
256 */
257#define ONNXIFI_CAPABILITY_HOT_PLUGGABLE 0x20
258
259/**
260 * Type of the backend information.
261 *
262 * Possible values:
263 * ONNXIFI_BACKEND_ONNXIFI_VERSION
264 * ONNXIFI_BACKEND_NAME
265 * ONNXIFI_BACKEND_VENDOR
266 * ONNXIFI_BACKEND_VERSION
267 * ONNXIFI_BACKEND_EXTENSIONS
268 * ONNXIFI_BACKEND_DEVICE
269 * ONNXIFI_BACKEND_DEVICE_TYPE
270 * ONNXIFI_BACKEND_ONNX_IR_VERSION
271 * ONNXIFI_BACKEND_OPSET_VERSION
272 * ONNXIFI_BACKEND_CAPABILITIES
273 * ONNXIFI_BACKEND_INIT_PROPERTIES
274 * ONNXIFI_BACKEND_MEMORY_TYPES
275 * ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES
276 * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES
277 * ONNXIFI_BACKEND_MEMORY_SIZE
278 * ONNXIFI_BACKEND_MAX_GRAPH_SIZE
279 * ONNXIFI_BACKEND_MAX_GRAPH_COUNT
280 * ONNXIFI_BACKEND_MACS_FP32
281 * ONNXIFI_BACKEND_MACS_FP16
282 * ONNXIFI_BACKEND_MEMORY_BANDWIDTH
283 * ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH
284 * ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH
285 * ONNXIFI_BACKEND_PCI_BUS_ID
286 * ONNXIFI_BACKEND_PCI_DEVICE_ID
287 * ONNXIFI_BACKEND_PCI_DOMAIN_ID
288 * ONNXIFI_BACKEND_DIRECTX_ID
289 * ONNXIFI_BACKEND_CUDA_INDEX
290 * ONNXIFI_BACKEND_OPENCL_PLATFORM_ID
291 * ONNXIFI_BACKEND_OPENCL_DEVICE_ID
292 */
293typedef int32_t onnxBackendInfo;
294
295/**
296 * Major and minor version of ONNXIFI specification implemented by the backend.
297 *
298 * Since ONNXIFI 1.0, backends MUST support this information query.
299 *
300 * Value type: uint64_t.
301 * The high 32 bits specify the major version.
302 * The low 32 bits specify the minor version.
303 *
304 * Possible values:
305 * UINT64_C(0x0000000100000000) for ONNXIFI 1.0
306 */
307#define ONNXIFI_BACKEND_ONNXIFI_VERSION 0
308
309/**
310 * Marketing name of the backend (excluding the vendor name).
311 *
312 * Since ONNXIFI 1.0, backends MUST support this information query.
313 *
314 * This string MUST be in UTF-8 encoding and NOT locale-sensitive.
315 *
316 * Value type: char[], e.g.:
317 * "Caffe2"
318 * "Glow"
319 */
320#define ONNXIFI_BACKEND_NAME 1
321
322/**
323 * Name of the backend vendor.
324 *
325 * Since ONNXIFI 1.0, backends MUST support this information query.
326 *
327 * This string MUST be in UTF-8 encoding and NOT locale-sensitive.
328 *
329 * Value type: char[], e.g.:
330 * "Facebook"
331 * "Marat Dukhan"
332 */
333#define ONNXIFI_BACKEND_VENDOR 2
334
335/**
336 * Version of the backend software. Exact format is vendor-specific, but MUST be
337 * unique for the software release.
338 *
339 * Since ONNXIFI 1.0, backends MUST support this information query.
340 *
341 * This string MUST be in US-ASCII encoding and NOT locale-sensitive.
342 *
343 * Value type: char[], e.g.:
344 * "1.2.3"
345 * "1.2.3.0"
346 * "1.2.3-db3a4439d233276e25681fb4735b7f8e674dda65"
347 */
348#define ONNXIFI_BACKEND_VERSION 3
349
350/**
351 * Space-separated list of vendor- or device-specific extensions supported on
352 * this backend.
353 *
354 * Since ONNXIFI 1.0, backends MUST support this information query.
355 *
356 * This string MUST be in US-ASCII encoding and NOT locale-sensitive.
357 *
358 * Value type: char[], e.g.:
359 * ""
360 * "onnx_clone_graph"
361 * "onnx_clone_graph fb_maskrcnn"
362 */
363#define ONNXIFI_BACKEND_EXTENSIONS 4
364
365/**
366 * Descriptive name of the device (i.e. CPU, GPU, DSP, or NPU model).
367 *
368 * Since ONNXIFI 1.0, backends MUST support this information query.
369 *
370 * This string MUST be in UTF-8 encoding and NOT locale-sensitive.
371 *
372 * Value type: char[], e.g.:
373 * "nnDuino 123"
374 */
375#define ONNXIFI_BACKEND_DEVICE 5
376
377/**
378 * Type of the device.
379 *
380 * Since ONNXIFI 1.0, backends MUST support this information query.
381 *
382 * Value type: onnxEnum.
383 * Possible values:
384 * ONNXIFI_DEVICE_TYPE_NPU
385 * ONNXIFI_DEVICE_TYPE_DSP
386 * ONNXIFI_DEVICE_TYPE_GPU
387 * ONNXIFI_DEVICE_TYPE_CPU
388 * ONNXIFI_DEVICE_TYPE_FPGA
389 * ONNXIFI_DEVICE_TYPE_HETEROGENEOUS
390 */
391#define ONNXIFI_BACKEND_DEVICE_TYPE 6
392
393/**
394 * List of supported ONNX IR versions.
395 *
396 * Since ONNXIFI 1.0, backends MUST support this information query.
397 *
398 * Value type: char[], e.g.:
399 * "3" (IR version in ONNX 1.0)
400 *
401 * Possible values: space-separated list of supported ONNX IR versions,
402 * represented as decimal integers. ONNX IR versions must match values
403 * in ONNX Version enum.
404 */
405#define ONNXIFI_BACKEND_ONNX_IR_VERSION 7
406
407/**
408 * List of supported operator set domains and maximum supported operator set
409 * version for each domain.
410 *
411 * Since ONNXIFI 1.0, backends MUST support this information query.
412 *
413 * Value type: char[], e.g.:
414 * "ai.onnx:1" (only operators in version 1 of default ONNX operator set)
415 * "ai.onnx:7" (operators up to version 7 of default ONNX operator set)
416 * "org.pytorch:2 ai.onnx:6 ai.facebook:1"
417 *
418 * Possible values: space-separated list of domain:max_version pairs where
419 * domain corresponds to OperatorSetIdProto.domain and max_version
420 * corresponds to the maximum value of OperatorSetIdProto.version supported
421 * by the backend for this domain. The backend MUST support all previous
422 * operator set versions as well.
423 */
424#define ONNXIFI_BACKEND_OPSET_VERSION 8
425
426/**
427 * Optional features supported by the backend.
428 *
429 * Since ONNXIFI 1.0, backends MUST support this information query.
430 *
431 * Value type: onnxBitfield.
432 * Possible values: any combination of the following flags:
433 * ONNXIFI_CAPABILITY_THREAD_SAFE
434 * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE
435 * ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS
436 * ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE
437 * ONNXIFI_CAPABILITY_VARIABLE_SIZE_OUTPUTS
438 * ONNXIFI_CAPABILITY_HOT_PLUGGABLE
439 * or any vendor-specific flags in the high 32 bits of the bit field.
440 */
441#define ONNXIFI_BACKEND_CAPABILITIES 10
442
443/**
444 * Auxiliary initialization properties supported by the backend.
445 *
446 * Since ONNXIFI 1.0, backends MUST support this information query.
447 *
448 * Value type: onnxBitfield.
449 * Possible values: any combination of vendor-specific flags in high 32 bits of
450 * the bit field.
451 */
452#define ONNXIFI_BACKEND_INIT_PROPERTIES 11
453
454/**
455 * Memory types supported for graph inputs and outputs.
456 *
457 * Since ONNXIFI 1.0, backends MUST support this information query.
458 *
459 * Value type: onnxBitfield.
460 * Possible values are any combination of the following flags:
461 * ONNXIFI_MEMORY_TYPE_CPU (always supported)
462 * ONNXIFI_MEMORY_TYPE_CUDA_BUFFER
463 * ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER
464 * ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D
465 * ONNXIFI_MEMORY_TYPE_D3D_RESOURCE
466 * or any vendor-specific flags in the high 32 bits of the bit field.
467 */
468#define ONNXIFI_BACKEND_MEMORY_TYPES 12
469
470/**
471 * Auxiliary initialization properties supported by graphs on the backend.
472 *
473 * Since ONNXIFI 1.0, backends MUST support this information query.
474 *
475 * Value type: onnxBitfield.
476 * Possible values: any combination of vendor-specific flags in high 32 bits of
477 * the bit field.
478 */
479#define ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES 13
480
481/**
482 * Memory synchronization primitives supported for graph inputs and outputs.
483 *
484 * Since ONNXIFI 1.0, backends MUST support this information query.
485 *
486 * Possible values are any combination of the following flags:
487 * ONNXIFI_SYNCHRONIZATION_EVENT (onnxEvent, always supported)
488 * ONNXIFI_SYNCHRONIZATION_IMPLICIT
489 * or any vendor-specific flags in the high 32 bits of the bit field.
490 */
491#define ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES 14
492
493/**
494 * Maximum amount of memory, in bytes, available to the use by the backend.
495 *
496 * Since ONNXIFI 1.0, backends MUST support this information query.
497 *
498 * Value type: uint64_t.
499 */
500#define ONNXIFI_BACKEND_MEMORY_SIZE 20
501
502/**
503 * Maximum size of network parameters, in bytes.
504 *
505 * Since ONNXIFI 1.0, backends MUST support this information query.
506 *
507 * Value type: uint64_t.
508 */
509#define ONNXIFI_BACKEND_MAX_GRAPH_SIZE 21
510
511/**
512 * Maximum number of independent network graphs supported by the backend.
513 *
514 * Since ONNXIFI 1.0, backends MUST support this information query.
515 *
516 * Value type: uint64_t.
517 */
518#define ONNXIFI_BACKEND_MAX_GRAPH_COUNT 22
519
520/**
521 * Number of FP32 multiply-accumulate operations per second delivered by the
522 * backend.
523 *
524 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
525 * information query.
526 *
527 * Value type: uint64_t.
528 * If the backend does not support FP32 computation, the value MUST be 0.
529 */
530#define ONNXIFI_BACKEND_MACS_FP32 30
531
532/**
533 * Number of FP16 multiply-accumulate operations per second delivered by the
534 * backend.
535 *
536 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
537 * information query.
538 *
539 * Value type: uint64_t.
540 * If the backend does not support FP16 computation, the value MUST be 0.
541 */
542#define ONNXIFI_BACKEND_MACS_FP16 31
543
544/**
545 * Bandwidth, in bytes per second, of the global memory specific to the backend
546 * device.
547 *
548 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
549 * information query.
550 *
551 * Value type: uint64_t.
552 */
553#define ONNXIFI_BACKEND_MEMORY_BANDWIDTH 35
554
555/**
556 * Bandwidth, in bytes per second, of transferring data from cacheable
557 * CPU-allocated memory to the backend device.
558 *
559 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
560 * information query.
561 *
562 * Value type: uint64_t.
563 */
564#define ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH 36
565
566/**
567 * Bandwidth, in bytes per second, of transferring data to cacheable
568 * CPU-allocated memory from the backend device.
569 *
570 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
571 * information query.
572 *
573 * Value type: uint64_t.
574 */
575#define ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH 37
576
577/**
578 * PCI bus ID of the backend device.
579 *
580 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
581 * information query.
582 *
583 * Value type: uint64_t.
584 */
585#define ONNXIFI_BACKEND_PCI_BUS_ID 40
586
587/**
588 * PCI device ID of the backend device.
589 *
590 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
591 * information query.
592 *
593 * Value type: uint64_t.
594 */
595#define ONNXIFI_BACKEND_PCI_DEVICE_ID 41
596
597/**
598 * PCI domain/function ID of the backend device.
599 *
600 * Since ONNXIFI 1.0, backends are recommended, but not required to support this
601 * information query.
602 *
603 * Value type: uint64_t.
604 */
605#define ONNXIFI_BACKEND_PCI_DOMAIN_ID 42
606
607/**
608 * DirectX ID of the backend device.
609 *
610 * This is the value that would be returned by ID3D12Device::GetAdapterLuid()
611 * for the hardware device used by the backend.
612 *
613 * Since ONNXIFI 1.0, DXGI-based backends are recommended, but not required to
614 * support this information query.
615 *
616 * Value type: LUID (8 bytes).
617 */
618#define ONNXIFI_BACKEND_DIRECTX_ID 43
619
620/**
621 * CUDA index of the backend device.
622 *
623 * Since ONNXIFI 1.0, CUDA-based backends are recommended, but not required to
624 * support this information query.
625 *
626 * Value type: uint64_t.
627 */
628#define ONNXIFI_BACKEND_CUDA_INDEX 44
629
630/**
631 * OpenCL platform ID for the backend device.
632 * This platform ID is guaranteed to remain valid for the lifetime of ONNXIFI
633 * objects related to the same ONNXIFI backend (backend ID, backend, graph,
634 * event).
635 *
636 * Since ONNXIFI 1.0, OpenCL-based backends are recommended, but not required to
637 * support this information query.
638 *
639 * Value type: cl_platform_id.
640 */
641#define ONNXIFI_BACKEND_OPENCL_PLATFORM_ID 45
642
643/**
644 * OpenCL device ID for the backend device.
645 * This device ID is guaranteed to remain valid for the lifetime of ONNXIFI
646 * objects related to the same ONNXIFI backend (backend ID, backend, graph,
647 * event).
648 *
649 * Since ONNXIFI 1.0, OpenCL-based backends are recommended, but not required to
650 * support this information query.
651 *
652 * Value type: cl_device_id.
653 */
654#define ONNXIFI_BACKEND_OPENCL_DEVICE_ID 46
655
656/* Note: the data type values match ONNX TensorProto.DataType enum */
657#define ONNXIFI_DATATYPE_UNDEFINED 0
658#define ONNXIFI_DATATYPE_FLOAT16 10
659#define ONNXIFI_DATATYPE_FLOAT32 1
660#define ONNXIFI_DATATYPE_FLOAT64 11
661#define ONNXIFI_DATATYPE_INT8 3
662#define ONNXIFI_DATATYPE_INT16 5
663#define ONNXIFI_DATATYPE_INT32 6
664#define ONNXIFI_DATATYPE_INT64 7
665#define ONNXIFI_DATATYPE_UINT8 2
666#define ONNXIFI_DATATYPE_UINT16 4
667#define ONNXIFI_DATATYPE_UINT32 12
668#define ONNXIFI_DATATYPE_UINT64 13
669#define ONNXIFI_DATATYPE_COMPLEX64 14
670#define ONNXIFI_DATATYPE_COMPLEX128 15
671#define ONNXIFI_DATATYPE_BFLOAT16 16
672
673/** Cacheable CPU memory */
674#define ONNXIFI_MEMORY_TYPE_CPU 0
675/** CUDA memory buffer (allocated via cudaMalloc/cuMalloc). */
676#define ONNXIFI_MEMORY_TYPE_CUDA_BUFFER 1
677/** OpenCL cl_mem object for a buffer or sub-buffer. */
678#define ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER 2
679/** OpenGL ES 2.0+ 2D Texture. */
680#define ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D 4
681/** Direct3D resource. */
682#define ONNXIFI_MEMORY_TYPE_D3D_RESOURCE 8
683
684/**
685 * Terminates the list of auxiliary backend initialization properties passed to
686 * onnxInitBackend.
687 */
688#define ONNXIFI_BACKEND_PROPERTY_NONE 0
689/**
690 * Optimization target for graphs initialized on the backend.
691 *
692 * Possible values:
693 * ONNXIFI_OPTIMIZATION_HIGH_THROUGHPUT
694 * ONNXIFI_OPTIMIZATION_LOW_LATENCY
695 * ONNXIFI_OPTIMIZATION_LOW_POWER
696 * ONNXIFI_OPTIMIZATION_LOW_DELAY
697 * ONNXIFI_OPTIMIZATION_AOT
698 */
699#define ONNXIFI_BACKEND_PROPERTY_OPTIMIZATION 1
700/**
701 * Logging verbosity level for the backend.
702 *
703 * If this property is not specified during initialization, the backend should
704 * assume ONNXIFI_LOG_LEVEL_WARNING logging verbosity level.
705 *
706 * Possible values:
707 * ONNXIFI_LOG_LEVEL_ERROR
708 * ONNXIFI_LOG_LEVEL_WARNING
709 * ONNXIFI_LOG_LEVEL_INFO
710 * ONNXIFI_LOG_LEVEL_DEBUG
711 */
712#define ONNXIFI_BACKEND_PROPERTY_LOG_LEVEL 2
713/**
714 * CUDA stream to be used by the backend.
715 * CUDA stream must be created on the CUDA device used by the ONNXIFI backend.
716 * Users can query which CUDA device is used by the ONNXIFI backend by calling
717 * onnxGetBackendInfo with ONNXIFI_BACKEND_CUDA_INDEX info type.
718 *
719 * If this property is not specified during initialization, the backend can
720 * create a new CUDA stream for the device, or use a default CUDA stream.
721 *
722 * Possible values: cudaStream_t or CUstream object, cast to uint64_t.
723 */
724#define ONNXIFI_BACKEND_CUDA_STREAM 4
725/**
726 * OpenCL context to be used by the backend.
727 * The context must be created with the OpenCL device ID and the OpenCL platform
728 * ID used by the ONNXIFI backend. Users can query which OpenCL device ID and
729 * OpenCL platform ID are used by the ONNXIFI backend by calling
730 * onnxGetBackendInfo with ONNXIFI_BACKEND_OPENCL_PLATFORM_ID
731 * and ONNXIFI_BACKEND_OPENCL_DEVICE_ID info types.
732 *
733 * If this property is not specified during initialization, the backend will
734 * create a new OpenCL context for the device.
735 *
736 * Possible values: cl_context object, cast to uint64_t.
737 */
738#define ONNXIFI_BACKEND_OPENCL_CONTEXT 8
739
740/**
741 * Terminates the list of auxiliary graph initialization properties passed to
742 * onnxInitGraph.
743 */
744#define ONNXIFI_GRAPH_PROPERTY_NONE 0
745
746/**
747 * Optimize graph representation and compilation for highest throughput.
748 */
749#define ONNXIFI_OPTIMIZATION_HIGH_THROUGHPUT 0
750
751/**
752 * Optimize graph representation and compilation for lowest latency.
753 */
754#define ONNXIFI_OPTIMIZATION_LOW_LATENCY 1
755
756/**
757 * Optimize graph representation and compilation for lowest power consumption.
758 */
759#define ONNXIFI_OPTIMIZATION_LOW_POWER 2
760
761/**
762 * Optimize graph representation and compilation for lowest delay until first
763 * result.
764 */
765#define ONNXIFI_OPTIMIZATION_LOW_DELAY 3
766
767/**
768 * Optimize graph representation and compilation using potentially
769 * longer-running ahead-of-time optimization.
770 */
771#define ONNXIFI_OPTIMIZATION_AOT 4
772
773/**
774 * Log events which caused a failure in an ONNXIFI function call.
775 */
776#define ONNXIFI_LOG_LEVEL_ERROR 4
777
778/**
779 * Log events in ONNXIFI_LOG_LEVEL_ERROR and events which caused
780 * a performance, accuracy, or quality of service degradation in a backend.
781 * Enabling this logging level SHOULD NOT have a measurable effect on
782 * performance.
783 */
784#define ONNXIFI_LOG_LEVEL_WARNING 3
785
786/**
787 * Log events in ONNXIFI_LOG_LEVEL_WARNING and high-level status information
788 * about operation of a backend. Enabling this logging level MAY cause a small
789 * degradation in performance.
790 */
791#define ONNXIFI_LOG_LEVEL_INFO 2
792
793/**
794 * Log events in ONNXIFI_LOG_LEVEL_INFO and detailed status information about
795 * operations of a backend. Enabling this logging level MAY cause a serious
796 * degradation in performance.
797 */
798#define ONNXIFI_LOG_LEVEL_DEBUG 1
799
800/**
801 * Tag for version 1 of tensor descriptor structure (onnxTensorDescriptorV1).
802 *
803 * The tag is unique for this version. If ONNXIFI introduce a new version of
804 * the tensor descriptor structure in the future, it will get a new tag value.
805 */
806#define ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1 0x43DFBF69
807
808typedef struct onnxTensorDescriptorV1 {
809 /**
810 * 32-bit tag needed to distinguish different versions of a tensor descriptor
811 * structure. In the onnxTensorDescriptorV1 structure, the tag MUST be set to
812 * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1. If ONNXIFI introduce a new version of the
813 * tensor descriptor structure in the future, it WILL have 32-bit tag with a
814 * different value as the first member of the structure.
815 *
816 * ONNXIFI implementations MUST validate tag before accessing any other
817 * members of the structure.
818 */
819 int32_t tag;
820 /**
821 * Name of the blob corresponding to this tensor in the ONNX model. The name
822 * must exactly match the ValueInfoProto.name of one of the
823 * ModelProto.graph.input or ModelProto.graph.output
824 */
825 const char* name;
826 /**
827 * Data type of the elements in the tensor.
828 *
829 * Possible values:
830 * ONNXIFI_DATATYPE_FLOAT16
831 * ONNXIFI_DATATYPE_FLOAT32
832 * ONNXIFI_DATATYPE_FLOAT64
833 * ONNXIFI_DATATYPE_INT8
834 * ONNXIFI_DATATYPE_INT16
835 * ONNXIFI_DATATYPE_INT32
836 * ONNXIFI_DATATYPE_INT64
837 * ONNXIFI_DATATYPE_UINT8
838 * ONNXIFI_DATATYPE_UINT16
839 * ONNXIFI_DATATYPE_UINT32
840 * ONNXIFI_DATATYPE_UINT64
841 * ONNXIFI_DATATYPE_COMPLEX64
842 * ONNXIFI_DATATYPE_COMPLEX128
843 */
844 onnxEnum dataType;
845 /**
846 * Type of memory that stores the tensor.
847 *
848 * ONNXIFI_MEMORY_TYPE_CPU memory type is always supported by the backend, but
849 * other memory types are optional. The use MUST call onnxGetBackendInfo with
850 * ONNXIFI_BACKEND_MEMORY_TYPES to check if a particular memory type is
851 * supported before using it.
852 *
853 * If the memory type is different than ONNXIFI_MEMORY_TYPE_CPU, it must be
854 * allocated on the same device as the backend.
855 *
856 * Possible values:
857 * ONNXIFI_MEMORY_TYPE_CPU (always supported)
858 * ONNXIFI_MEMORY_TYPE_CUDA_BUFFER (support is optional)
859 * ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER (support is optional)
860 * ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D (support is optional)
861 * ONNXIFI_MEMORY_TYPE_D3D_RESOURCE (support is optional)
862 */
863 onnxEnum memoryType;
864 /**
865 * Number of dimensions in the tensor.
866 * For a scalar, the number of dimensions is 0.
867 */
868 uint32_t dimensions;
869 /**
870 * Dimensions of the tensor.
871 * For a scalar, this pointer can be NULL.
872 */
873 const uint64_t* shape;
874 /**
875 * Indicates the axis along which quantization parameters are shared.
876 */
877 uint32_t quantizationAxis;
878 /**
879 * This indicates how many quantization parameters this tensor has in the
880 * scales and biases fields. If this tensor is not quantized then this is 0.
881 * This must be less than or equal to the size of the tensor in the
882 * quantizationAxis dimension. Note that quantization parameters will be
883 * shared amongst equally sized groups along the quantizationAxis.
884 */
885 uint64_t quantizationParams;
886 /**
887 * Quantized tensor scale info.
888 * For non-quantized tensor, these scales are not valid and should not exist.
889 */
890 const float* scales;
891 /**
892 * Quantized tensor bias info.
893 * For non-quantized tensor, these biases are not valid and should not exist.
894 */
895 const int32_t* biases;
896
897 /**
898 * Whether this tensor is an offline tensor, which doesn't carry actual data.
899 */
900 uint8_t isOffline;
901
902 /**
903 * Pointers to tensor data.
904 *
905 * Interpretation depends on memoryType:
906 * - ONNXIFI_MEMORY_TYPE_CPU: buffer is a valid pointer to CPU memory.
907 * - ONNXIFI_MEMORY_TYPE_CUDA_BUFFER: buffer is a valid pointer to CUDA
908 * device memory, allocated via cudaMalloc or cuMalloc. CUDA device memory
909 * must be allocated on the same device as the backend.
910 * - ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER: buffer is a cl_mem handle for an
911 * OpenCL buffer or a sub-buffer. cl_mem object must be allocated on the
912 * same OpenCL context as the backend. The context must be specified via
913 * ONNXIFI_BACKEND_OPENCL_CONTEXT initialization property to
914 * onnxInitBackend.
915 * - ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D: buffer is a name of a 2D
916 * texture created by glGenTextures. The texture must be allocated on the
917 * same device as the backend.
918 * - ONNXIFI_MEMORY_TYPE_D3D_RESOURCE: TBD
919 */
920 onnxPointer buffer;
921} onnxTensorDescriptorV1;
922
923/**
924 * Synchronization using ONNXIFI event object (onnxEvent).
925 */
926#define ONNXIFI_SYNCHRONIZATION_EVENT 0
927/**
928 * Implicit synchronization of inputs and outputs access with the caller.
929 * The details are backend-specific, and may involve extra parameters passed
930 * during backend initialization.
931 *
932 * Examples:
933 * - CUDA-based backends could implicitly synchronize with the caller through
934 * the use of the same CUDA stream.
935 * - OpenCL-based backends could implicitly synchronize with the caller through
936 * the use of the same in-order OpenCL command queue.
937 */
938#define ONNXIFI_SYNCHRONIZATION_IMPLICIT 2
939
940/**
941 * Tag for version 1 of memory fence structure (onnxMemoryFenceV1).
942 *
943 * The tag is unique for this version. If ONNXIFI introduce a new version of
944 * the memory fence structure in the future, it will get a new tag value.
945 */
946#define ONNXIFI_TAG_MEMORY_FENCE_V1 0x23E08AAB
947
948typedef struct onnxMemoryFenceV1 {
949 /**
950 * 32-bit tag needed to distinguish different versions of a memory fence
951 * structure. In the onnxMemoryFenceV1 structure, the tag MUST be set to
952 * ONNXIFI_TAG_MEMORY_FENCE_V1. If ONNXIFI introduce a new version of the
953 * memory fence structure in the future, it WILL have 32-bit tag with a
954 * different value as the first member of the structure.
955 *
956 * ONNXIFI implementations MUST validate tag before accessing any other
957 * members of the structure.
958 */
959 int32_t tag;
960 /**
961 * Type of memory synchronization primitive.
962 *
963 * Possible values:
964 * ONNXIFI_SYNCHRONIZATION_EVENT (onnxEvent, always supported)
965 * ONNXIFI_SYNCHRONIZATION_IMPLICIT
966 */
967 onnxEnum type;
968 union {
969 /**
970 * Handle for a single-shot ONNXIFI event used as a synchronization
971 * primitive. Event for the input fence must be created by the caller to
972 * onnxRunGraph. Event for the output fence is created by implementation of
973 * onnxRunGraph, and stored into the output memory fence structure before
974 * onnxRunGraph returns.
975 */
976 onnxEvent event;
977 };
978} onnxMemoryFenceV1;
979
980/* Function pointer declarations for dynamic loading */
981typedef ONNXIFI_CHECK_RESULT onnxStatus
982 (ONNXIFI_ABI* onnxGetBackendIDsFunction)(
983 onnxBackendID* backendIDs,
984 size_t* numBackends);
985typedef onnxStatus
986 (ONNXIFI_ABI* onnxReleaseBackendIDFunction)(
987 onnxBackendID backendID);
988typedef ONNXIFI_CHECK_RESULT onnxStatus
989 (ONNXIFI_ABI* onnxGetBackendInfoFunction)(
990 onnxBackendID backendID,
991 onnxBackendInfo infoType,
992 void* infoValue,
993 size_t* infoValueSize);
994typedef ONNXIFI_CHECK_RESULT onnxStatus
995 (ONNXIFI_ABI* onnxGetBackendCompatibilityFunction)(
996 onnxBackendID backendID,
997 size_t onnxModelSize,
998 const void* onnxModel);
999typedef ONNXIFI_CHECK_RESULT onnxStatus
1000 (ONNXIFI_ABI* onnxInitBackendFunction)(
1001 onnxBackendID backendID,
1002 const uint64_t* auxPropertiesList,
1003 onnxBackend* backend);
1004typedef onnxStatus
1005 (ONNXIFI_ABI* onnxReleaseBackendFunction)(
1006 onnxBackend backend);
1007typedef ONNXIFI_CHECK_RESULT onnxStatus
1008 (ONNXIFI_ABI* onnxInitEventFunction)(
1009 onnxBackend backend,
1010 onnxEvent* event);
1011typedef ONNXIFI_CHECK_RESULT onnxStatus
1012 (ONNXIFI_ABI* onnxSignalEventFunction)(
1013 onnxEvent event);
1014typedef ONNXIFI_CHECK_RESULT onnxStatus
1015 (ONNXIFI_ABI* onnxGetEventStateFunction)(
1016 onnxEvent event,
1017 onnxEventState* state);
1018typedef ONNXIFI_CHECK_RESULT onnxStatus
1019 (ONNXIFI_ABI* onnxWaitEventFunction)(
1020 onnxEvent event);
1021typedef onnxStatus
1022 (ONNXIFI_ABI* onnxReleaseEventFunction)(
1023 onnxEvent event);
1024typedef ONNXIFI_CHECK_RESULT onnxStatus
1025 (ONNXIFI_ABI* onnxInitGraphFunction)(
1026 onnxBackend backend,
1027 const uint64_t* auxPropertiesList,
1028 size_t onnxModelSize,
1029 const void* onnxModel,
1030 uint32_t weightsCount,
1031 const onnxTensorDescriptorV1* weightDescriptors,
1032 onnxGraph* graph,
1033 uint32_t maxSeqLength,
1034 void* deferredWeightReader);
1035typedef ONNXIFI_CHECK_RESULT onnxStatus
1036 (ONNXIFI_ABI* onnxSetGraphIOFunction)(
1037 onnxGraph graph,
1038 uint32_t inputsCount,
1039 const onnxTensorDescriptorV1* inputDescriptors,
1040 uint32_t outputsCount,
1041 const onnxTensorDescriptorV1* outputDescriptors);
1042typedef ONNXIFI_CHECK_RESULT onnxStatus
1043 (ONNXIFI_ABI* onnxRunGraphFunction)(
1044 onnxGraph graph,
1045 const onnxMemoryFenceV1* inputFence,
1046 onnxMemoryFenceV1* outputFence);
1047typedef onnxStatus
1048 (ONNXIFI_ABI* onnxReleaseGraphFunction)(
1049 onnxGraph graph);
1050
1051/**
1052 * Get stable IDs of available backends on the system.
1053 *
1054 * ONNXIFI backend is a combination of software layer and hardware device used
1055 * to run an ONNX graph. The same software layer may expose multiple backends
1056 * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend
1057 * for GPU and another for CPU, both implemented in the same software). Backends
1058 * implemented in the same software, but targeting different devices (e.g.
1059 * "MyNN" for CPU and "MyNN" for GPU) have different backend IDs.
1060 *
1061 * Note that some (hot-pluggable) backends can be connected and disconnected at
1062 * any time, and thus subsequent calls to this function may return different
1063 * number or set of backend IDs. The returned IDs, however, stay valid even if
1064 * the hardware device used by the backend disconnects from the system.
1065 *
1066 * To avoid resource leak, the backend ID MUST be released through a call to
1067 * onnxReleaseBackendID when it is no longer needed.
1068 *
1069 * @param backendIDs[out] - pointer to the memory location where the backend IDs
1070 * will be returned. If the pointer is NULL, it is
1071 * ignored, and the function returns only the number
1072 * of backend IDs through numBackendIDs pointer.
1073 * @param numBackendIDs[in,out] - pointer to a variable specifying number of
1074 * available backends. On function entry, the
1075 * variable MUST contain the capacity, in number
1076 * of backend IDs, of the memory buffer specified
1077 * by backendIDs. For successful completion, this
1078 * capacity must be at least as large as the
1079 * number of available backends. If the function
1080 * completes with either ONNXIFI_STATUS_SUCCESS
1081 * or ONNXIFI_STATUS_FALLBACK status codes, the
1082 * number of backend IDs written into backendIDs
1083 * buffer is stored in the variable specified by
1084 * this pointer.
1085 *
1086 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded, and backend IDs
1087 * are stored in the location specified by
1088 * backendIDs, and the number of the backends
1089 * is stored in the location specified by
1090 * numBackends.
1091 * @retval ONNXIFI_STATUS_FALLBACK The function call completed, but the
1092 * backend IDs were not stored in the
1093 * location specified by backendIDs, either
1094 * because it is NULL, or because the size of
1095 * the memory buffer is insufficient to store
1096 * all available backend IDs. The number of
1097 * available backends is stored in the
1098 * location specified by numBackends.
1099 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1100 * numBackends is NULL.
1101 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
1102 * system failed to allocate memory
1103 * to store backend ID information.
1104 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1105 * implementation experienced an
1106 * unrecovered internal error.
1107 */
1108ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1109 onnxGetBackendIDs(
1110 onnxBackendID* backendIDs,
1111 size_t* numBackends);
1112
1113/**
1114 * Deinitialize ONNXIFI backend IDs and release associated resources.
1115 *
1116 * The user MUST deinitialize all objects created with this backend ID
1117 * (onnxBackend, onnxGraph, onnxEvent) before calling this function to
1118 * deinitialize the backend ID.
1119 *
1120 * @param backendID - backend ID returned by onnxGetBackendIDs.
1121 *
1122 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the resources
1123 * associated to the backend ID were released to
1124 * the operating system.
1125 * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID
1126 * is not an ONNXIFI backend ID.
1127 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1128 * implementation experienced an
1129 * unrecovered internal error.
1130 */
1131ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI
1132 onnxReleaseBackendID(
1133 onnxBackendID backendID);
1134
1135/**
1136 * Query high-level information about the backend and its target device.
1137 *
1138 * ONNXIFI backend is a combination of software layer and hardware device used
1139 * to run an ONNX graph. The same software layer may expose multiple backends
1140 * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend
1141 * for GPU and another for CPU, both implemented in the same software).
1142 *
1143 * The content, data type, and availability of information provided by this
1144 * function depends on infoType value as specified below:
1145 *
1146 * infoType value data type support
1147 * ONNXIFI_BACKEND_ONNXIFI_VERSION uint64_t required
1148 * ONNXIFI_BACKEND_NAME char[] required
1149 * ONNXIFI_BACKEND_VENDOR char[] required
1150 * ONNXIFI_BACKEND_VERSION char[] required
1151 * ONNXIFI_BACKEND_EXTENSIONS char[] required
1152 * ONNXIFI_BACKEND_DEVICE char[] required
1153 * ONNXIFI_BACKEND_DEVICE_TYPE onnxEnum required
1154 * ONNXIFI_BACKEND_ONNX_IR_VERSION char[] required
1155 * ONNXIFI_BACKEND_OPSET_VERSION char[] required
1156 * ONNXIFI_BACKEND_CAPABILITIES onnxBitfield required
1157 * ONNXIFI_BACKEND_INIT_PROPERTIES onnxBitfield required
1158 * ONNXIFI_BACKEND_MEMORY_TYPES onnxBitfield required
1159 * ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES onnxBitfield required
1160 * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES onnxBitfield required
1161 * ONNXIFI_BACKEND_MEMORY_SIZE uint64_t required
1162 * ONNXIFI_BACKEND_MAX_GRAPH_SIZE uint64_t required
1163 * ONNXIFI_BACKEND_MAX_GRAPH_COUNT uint64_t required
1164 * ONNXIFI_BACKEND_MACS_FP32 uint64_t optional
1165 * ONNXIFI_BACKEND_MACS_FP16 uint64_t optional
1166 * ONNXIFI_BACKEND_MEMORY_BANDWIDTH uint64_t optional
1167 * ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH uint64_t optional
1168 * ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH uint64_t optional
1169 * ONNXIFI_BACKEND_PCI_BUS_ID uint64_t optional
1170 * ONNXIFI_BACKEND_PCI_DEVICE_ID uint64_t optional
1171 * ONNXIFI_BACKEND_PCI_DOMAIN_ID uint64_t optional
1172 * ONNXIFI_BACKEND_DIRECTX_ID LUID optional
1173 * ONNXIFI_BACKEND_CUDA_INDEX uint64_t optional
1174 * ONNXIFI_BACKEND_OPENCL_PLATFORM_ID cl_platform_id optional
1175 * ONNXIFI_BACKEND_OPENCL_DEVICE_ID cl_device_id optional
1176 *
1177 * @param backendID - ID of the backend to query.
1178 * @param infoType - type of the backend information to query. Must be one of
1179 * the ONNXIFI_BACKEND_* constants. If this value is not
1180 * supported by the backend, the function will fail with
1181 * ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE.
1182 * @param infoValue[out] - pointer to the memory location where the backend
1183 * information value will be returned. If the pointer is
1184 * NULL, is it ignored.
1185 * @param infoValueSize[in,out] - pointer to a variable specifying size, in
1186 * bytes, of the information value. On function
1187 * entry, the variable MUST contain the size of
1188 * the memory buffer specified by infoValue.
1189 * For successful completion, this size must be
1190 * at least as large as the queried value. If the
1191 * function completes with either
1192 * ONNXIFI_STATUS_SUCCESS or
1193 * ONNXIFI_STATUS_FALLBACK status codes, the
1194 * actual size of the value queried in the call
1195 * is stored in the variable specified by this
1196 * pointer.
1197 *
1198 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded, and requested
1199 * value is stored in the location specified by
1200 * infoValue, and the actual size of the
1201 * requested value is stored in the location
1202 * specified by infoValueSize.
1203 * @retval ONNXIFI_STATUS_FALLBACK The function call completed, but the
1204 * requested value was not stored in the
1205 * location specified by infoValue, either
1206 * because it is NULL, or because the size of
1207 * the memory buffer is insufficient for the
1208 * value. The actual size of the requested value
1209 * is stored in the location specified by
1210 * infoValueSize.
1211 * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID
1212 * is not an ONNXIFI backend ID.
1213 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1214 * infoValueSize is NULL.
1215 * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because
1216 * the value of infoType is not
1217 * supported by the backend.
1218 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1219 * the backend was disconnected or
1220 * uninstalled from the system.
1221 */
1222ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1223 onnxGetBackendInfo(
1224 onnxBackendID backendID,
1225 onnxBackendInfo infoType,
1226 void* infoValue,
1227 size_t* infoValueSize);
1228
1229/**
1230 * Query if an ONNX model graph is compatible with the backend.
1231 *
1232 * Model graph is passed as a serialized ModelProto message, where types and
1233 * dimensions of all inputs (including static weights) and outputs are specified
1234 * through ModelProto.graph.input and ModelProto.graph.output messages. If the
1235 * backend supports ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS, some of the shape
1236 * dimensions can be symbolic. If the backend supports
1237 * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE, the outer shape dimension can be
1238 * symbolic. In these cases, the validation of symbolic dimension should be
1239 * deferred until graph inputs and outputs are specified in onnxSetGraphIO.
1240 *
1241 * Commonly, the serialized ModelProto message passed to this function would
1242 * not include the static weights (ModelProto.graph.initializer is empty), and
1243 * the backend implementation MUST NOT rely on the weights to determine if the
1244 * graph is supported.
1245 *
1246 * An important use-case is a ModelProto containing only a single NodeProto in
1247 * ModelProto.graph.node, which happens when a high-level framework checks
1248 * operators one-by-one to find a connected subgraph that can be offloaded to
1249 * the backend. Backend implementations SHOULD optimize performance for this
1250 * use-case.
1251 *
1252 * @param backend - ID of the backend to query.
1253 * @param onnxModelSize - size of the serialized ONNX ModelProto message,
1254 * in bytes.
1255 * @param[in] onnxModel - pointer to serialized ONNX ModelProto message
1256 * representing the model graph.
1257 *
1258 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the model
1259 * graph can efficiently run on the backend.
1260 * @retval ONNXIFI_STATUS_FALLBACK The function call succeeded and the model
1261 * graph can run on the backend through some
1262 * emulation layer with some efficiency loss. If
1263 * a backend decomposes this operator into
1264 * multiple sub-operators, it should return this
1265 * code. E.g. if a backend does not natively
1266 * support grouped or depthwise convolution, but
1267 * can execute it as multiple unit-group
1268 * convolution operators, it must returns this
1269 * code.
1270 * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID
1271 * is not an ONNXIFI backend ID.
1272 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1273 * onnxModel is NULL.
1274 * @retval ONNXIFI_STATUS_INVALID_SIZE The function call failed because
1275 * onnxModelSize is 0.
1276 * @retval ONNXIFI_STATUS_INVALID_PROTOBUF The function call failed because it
1277 * couldn't parse the serialized
1278 * protobuf as an ONNX ModelProto
1279 * message.
1280 * @retval ONNXIFI_STATUS_INVALID_MODEL The function call failed because the
1281 * parsed ModelProto message does not
1282 * satisfy ONNX requirements and
1283 * constraints.
1284 * @retval ONNXIFI_STATUS_UNSUPPORTED_VERSION The function call failed because
1285 * the ONNX IR version or operator
1286 * version is not supported by the
1287 * backend.
1288 * @retval ONNXIFI_STATUS_UNSUPPORTED_OPERATOR The function call failed because
1289 * one of the operators in the model
1290 * graph is not supported by the
1291 * backend.
1292 * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because
1293 * the backend does not support the
1294 * particular AttributeProto
1295 * values in one of the operators.
1296 * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the
1297 * backend does not support the
1298 * tensor shapes in an input or output
1299 * of one of the operators. The
1300 * problematic tensor shapes could be
1301 * directly specified through
1302 * ValueInfoProto in GraphProto.input,
1303 * GraphProto.output, or
1304 * GraphProto.value_info, through
1305 * TensorProto in
1306 * GraphProto.initializer, or inferred
1307 * from the inputs by the backend.
1308 * @retval ONNXIFI_STATUS_UNSUPPORTED_DATATYPE The function call failed because
1309 * the backend does not support the
1310 * data types in an input or output
1311 * of one of the operators. The
1312 * problematic data types could be
1313 * directly specified through
1314 * ValueInfoProto in
1315 * GraphProto.input,
1316 * GraphProto.output, or
1317 * GraphProto.value_info, through
1318 * TensorProto in
1319 * GraphProto.initializer, or
1320 * inferred from the inputs by the
1321 * backend.
1322 * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because
1323 * output or intermediate shapes
1324 * specified in the ONNX model graph do
1325 * not match the shapes inferred from
1326 * input shapes.
1327 * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because
1328 * output or intermediate data types
1329 * specified in the ONNX model graph
1330 * do not match the data types
1331 * inferred from graph inputs.
1332 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
1333 * backend could not allocate enough
1334 * system memory to parse and analyze
1335 * the model graph.
1336 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1337 * the backend was disconnected or
1338 * uninstalled from the system.
1339 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1340 * backend experienced an unrecovered
1341 * internal error.
1342 */
1343ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1344 onnxGetBackendCompatibility(
1345 onnxBackendID backendID,
1346 size_t onnxModelSize,
1347 const void* onnxModel);
1348
1349/**
1350 * Initialize an ONNXIFI backend.
1351 *
1352 * ONNXIFI backend is a combination of software layer and hardware device used
1353 * to run an ONNXIFI graph. The same software layer may expose multiple backends
1354 * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend
1355 * for GPU and another for CPU, both implemented in the same software).
1356 *
1357 * @param backendID - ID of the backend to initialize.
1358 * @param[in] auxPropertiesList - optional list of backend initialization
1359 * properties, terminated by
1360 * ONNXIFI_BACKEND_PROPERTY_NONE entry. Can be
1361 * NULL or empty.
1362 * @param[out] backend - pointer to an opaque handle for the initialized ONNXIFI
1363 * backend. If the function fails, the handle is
1364 * initialized to NULL.
1365 *
1366 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the backend
1367 * was successfully initialized.
1368 * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID
1369 * is not an ONNXIFI backend ID.
1370 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1371 * backend pointer is NULL.
1372 * @retval ONNXIFI_STATUS_INVALID_PROPERTY The function call failed because one
1373 * of the backend initialization
1374 * property values is invalid.
1375 * @retval ONNXIFI_STATUS_UNSUPPORTED_PROPERTY The function call failed because
1376 * backend does not recognize one
1377 * of the initialization
1378 * property IDs.
1379 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed due to
1380 * insufficient system memory to
1381 * initialize backend.
1382 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
1383 * insufficient non-memory system
1384 * resources (e.g. file handles) to
1385 * initialize the backend.
1386 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
1387 * insufficient backend-specific memory
1388 * to initialize the backend.
1389 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
1390 * insufficient non-memory
1391 * backend-specific resources (e.g.
1392 * command queues) to initialize the
1393 * backend.
1394 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1395 * the backend was disconnected or
1396 * uninstalled from the system.
1397 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1398 * backend experienced an unrecovered
1399 * internal error.
1400 */
1401ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1402 onnxInitBackend(
1403 onnxBackendID backendID,
1404 const uint64_t* auxPropertiesList,
1405 onnxBackend* backend);
1406
1407/**
1408 * Deinitialize an ONNXIFI backend and release associated resources.
1409 *
1410 * The user MUST deinitialize all objects created on this backend (onnxGraph,
1411 * onnxEvent) before calling this function to deinitialize the backend.
1412 *
1413 * @param backend - ONNXIFI backend handle created by onnxInitBackend.
1414 *
1415 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the backend
1416 * resources were released to the operating
1417 * system.
1418 * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because
1419 * backend is not an ONNXIFI backend
1420 * handle.
1421 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1422 * backend experienced an unrecovered
1423 * internal error.
1424 */
1425ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI
1426 onnxReleaseBackend(
1427 onnxBackend backend);
1428
1429/**
1430 * Initialize a single-shot ONNXIFI event.
1431 *
1432 * The newly created event is in non-signalled state.
1433 *
1434 * @param backend - backend handle created by onnxInitBackend. This backend
1435 * would be used to initialize the event.
1436 * @param[out] event - pointer to the opaque handle for the created ONNXIFI
1437 * event. If the function fails, the handle is initialized
1438 * to NULL.
1439 *
1440 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event
1441 * was successfully initialized.
1442 * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because
1443 * backend is not an ONNXIFI backend
1444 * handle.
1445 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1446 * event pointer is NULL.
1447 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed due to
1448 * insufficient system memory to
1449 * initialize the event.
1450 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
1451 * insufficient non-memory system
1452 * resources (e.g. file handles) to
1453 * initialize the event.
1454 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
1455 * insufficient backend-specific memory
1456 * to initialize the event.
1457 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
1458 * insufficient non-memory
1459 * backend-specific resources (e.g.
1460 * command queues) to initialize the
1461 * event.
1462 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1463 * the backend was disconnected or
1464 * uninstalled from the system.
1465 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1466 * backend experienced an unrecovered
1467 * internal error.
1468 */
1469ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1470 onnxInitEvent(
1471 onnxBackend backend,
1472 onnxEvent* event);
1473
1474/**
1475 * Change the state of an ONNXIFI event to signalled.
1476 *
1477 * @param event - event handle created by onnxInitEvent. While it is technically
1478 * possible to use this function for output memory fence event
1479 * created by onnxRunGraph, users SHOULD NOT do that.
1480 *
1481 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event
1482 * was changed to signalled state.
1483 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event
1484 * is not an ONNXIFI event handle.
1485 * @retval ONNXIFI_STATUS_INVALID_STATE The function call failed because event
1486 * is already in the signalled state.
1487 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1488 * the backend was disconnected or
1489 * uninstalled from the system.
1490 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1491 * implementation experienced an
1492 * unrecovered internal error.
1493 */
1494ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1495 onnxSignalEvent(
1496 onnxEvent event);
1497
1498/**
1499 * Query ONNXIFI event state without blocking.
1500 *
1501 * @param event - event handle created by onnxRunGraph. While it is technically
1502 * possible to use this function to events created by
1503 * onnxInitEvent, this is not the intended use-case.
1504 * @param[out] state - pointer to the variable that will store the state of the
1505 * event. If the function fails, the variable is initialized
1506 * to ONNXIFI_EVENT_STATE_INVALID.
1507 *
1508 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the state
1509 * variable was initialized to either
1510 * ONNXIFI_EVENT_STATE_SIGNALLED or
1511 * ONNXIFI_EVENT_STATE_NONSIGNALLED according
1512 * to the state of the event.
1513 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event
1514 * is not an ONNXIFI event handle.
1515 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because state
1516 * pointer is NULL.
1517 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1518 * the backend was disconnected or
1519 * uninstalled from the system.
1520 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1521 * implementation experienced an
1522 * unrecovered internal error.
1523 */
1524ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1525 onnxGetEventState(
1526 onnxEvent event,
1527 onnxEventState* state);
1528
1529/**
1530 * Wait until an ONNXIFI event transitions to signalled state.
1531 *
1532 * @param event - event handle created by onnxRunGraph. While it is technically
1533 * possible to use this function to events created by
1534 * onnxInitEvent, this is not the intended use-case.
1535 *
1536 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function
1537 * returned because event transitioned to
1538 * signalled state.
1539 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event
1540 * is not an ONNXIFI event handle.
1541 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1542 * the backend was disconnected or
1543 * uninstalled from the system.
1544 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1545 * implementation experienced an
1546 * unrecovered internal error.
1547 */
1548ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1549 onnxWaitEvent(
1550 onnxEvent event);
1551
1552/**
1553 * Deinitialize an ONNXIFI event and release associated resources.
1554 *
1555 * @param event - event handle created by either onnxInitEvent or onnxRunGraph.
1556 *
1557 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event
1558 * resources were released to the operating
1559 * system.
1560 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event
1561 * is not an ONNXIFI event handle.
1562 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1563 * implementation experienced an
1564 * unrecovered internal error.
1565 */
1566ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI
1567 onnxReleaseEvent(
1568 onnxEvent event);
1569
1570/**
1571 * Parse an ONNXIFI graph and convert it for a particular backend.
1572 *
1573 * Model graph is passed as a serialized ModelProto message, where types and
1574 * dimensions of all inputs (including static weights) and outputs are specified
1575 * through ModelProto.graph.input and ModelProto.graph.output messages. If the
1576 * backend supports ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS, some of the shape
1577 * dimensions can be symbolic. If the backend supports
1578 * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE, the outer shape dimension can be
1579 * symbolic. In these cases, their validation should be deferred until a later
1580 * call to onnxSetGraphIO.
1581 *
1582 * Values of all static weights of the graph must be specified either in
1583 * ModelProto.graph.initializer, or through the weightDescriptors parameters,
1584 * but not through any combination of the two methods. If the caller creates the
1585 * graph on the fly, it SHOULD pass weights through weightDescriptors as it
1586 * involves less overhead.
1587 *
1588 * Blobs and operators in this graph are independent of the blobs and operators
1589 * of other graphs on the same backend.
1590 *
1591 * @param backend - backend handle created by onnxInitBackend. This backend
1592 * would be used to setup and run the model graph.
1593 * @param[in] auxPropertiesList - optional list of graph initialization
1594 * properties, terminated by
1595 * ONNXIFI_GRAPH_PROPERTY_NONE entry. Can be
1596 * NULL or empty.
1597 * @param onnxModelSize - size of the serialized ONNX ModelProto message,
1598 * in bytes.
1599 * @param[in] onnxModel - pointer to serialized ONNX ModelProto message
1600 * representing the model graph. The backend MUST not
1601 * assume that the serialized ModelProto message is
1602 * present at this address after the function returns.
1603 * @param weightsCount - number of weights specified in this function call
1604 * through tensor descriptors. Alternatively, the weights
1605 * can be specified in ModelProto.graph.initializer.
1606 * If weightsCount is non-zero, weightDescriptors must be
1607 * non-NULL.
1608 * @param[in] weightDescriptors - descriptors of static input tensors for the
1609 * graph. Elements of this array provide location
1610 * for blobs identified by ValueInfoProto.name
1611 * listed in ModelProto.graph.input of the ONNX
1612 * graph. If this parameter is non-NULL,
1613 * all static inputs must be specified through
1614 * the tensor descriptors, and the
1615 * ModelProto.graph.initilizer list must be
1616 * empty. The tensor descriptors
1617 * must use ONNXIFI_MEMORY_TYPE_CPU memory type,
1618 * and the backend must copy the values of the
1619 * tensors and all metadata, including shape,
1620 * into its own memory before the function
1621 * returns.
1622 * @param[out] graph - pointer to the opaque handle for the created ONNXIFI
1623 * graph. If the function fails, and this pointer is
1624 * non-NULL, the handle is initialized to NULL.
1625 * @param[inout] deferredWeightReader - pointer to an opaque handle to populate
1626 * the offline tensors during backend
1627 * compilation time. Default is NULL.
1628 *
1629 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the model
1630 * graph was successfully initialized on the
1631 * backend.
1632 * @retval ONNXIFI_STATUS_FALLBACK The function call succeeded and the model
1633 * graph was initialized for the backend through
1634 * an emulation layer with substantial
1635 * efficiency loss. If a backend decomposes an
1636 * operator into multiple sub-operators, it
1637 * MUST return this code. E.g. if a backend
1638 * does not natively support grouped or
1639 * depthwise convolution, but can execute it as
1640 * multiple unit-group convolution operators, it
1641 * should return this code.
1642 * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because
1643 * backend is not an ONNXIFI backend
1644 * handle.
1645 * @retval ONNXIFI_STATUS_INVALID_PROPERTY The function call failed because one
1646 * of the graph initialization property
1647 * values is invalid.
1648 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1649 * onnxModel or graph pointer is NULL, or
1650 * weightDescriptors pointer is NULL
1651 * while weightsCount is non-zero.
1652 * @retval ONNXIFI_STATUS_INVALID_SIZE The function call failed because
1653 * onnxModelSize is 0.
1654 * @retval ONNXIFI_STATUS_INVALID_PROTOBUF The function call failed because it
1655 * couldn't parse the serialized
1656 * protobuf as an ONNX ModelProto
1657 * message.
1658 * @retval ONNXIFI_STATUS_INVALID_MODEL The function call failed because the
1659 * parsed ModelProto message does not
1660 * satisfy ONNX requirements and
1661 * constraints.
1662 * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of
1663 * the shape dimensions in
1664 * weightDescriptors is 0.
1665 * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because
1666 * one of the data types in
1667 * weightDescriptors is unknown to the
1668 * backend.
1669 * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because
1670 * one of the memory types in
1671 * weightDescriptors is unknown to
1672 * the backend.
1673 * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed
1674 * because one of the memory
1675 * locations in weightDescriptors
1676 * is invalid (NULL pointer).
1677 * @retval ONNXIFI_STATUS_UNSUPPORTED_PROPERTY The function call failed because
1678 * backend does not recognize one
1679 * of the graph initialization
1680 * property IDs.
1681 * @retval ONNXIFI_STATUS_UNSUPPORTED_VERSION The function call failed because
1682 * the ONNX IR version or operator
1683 * version is not supported by the
1684 * backend.
1685 * @retval ONNXIFI_STATUS_UNSUPPORTED_OPERATOR The function call failed because
1686 * one of the operators in the model
1687 * graph is not supported by the
1688 * backend.
1689 * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because
1690 * the backend does not support the
1691 * particular AttributeProto
1692 * values in one of the operators.
1693 * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the
1694 * backend does not support the
1695 * tensor shapes in an input or
1696 * output of one of the operators.
1697 * The problematic tensor shapes could
1698 * be directly specified through
1699 * ValueInfoProto in GraphProto.input,
1700 * GraphProto.output, or
1701 & GraphProto.value_info, through
1702 * TensorProto in
1703 * GraphProto.initializer, through
1704 * weightDescriptors argument,
1705 * or inferred from the inputs by the
1706 * backend.
1707 * @retval ONNXIFI_STATUS_UNSUPPORTED_DATATYPE The function call failed because
1708 * the backend does not support the
1709 * data types in an input or output
1710 * of one of the operators. The
1711 * problematic data types could be
1712 * directly specified through
1713 * ValueInfoProto in
1714 * GraphProto.input,
1715 * GraphProto.output, or
1716 * GraphProto.value_info, through
1717 * TensorProto in
1718 * GraphProto.initializer, through
1719 * weightDescriptors argument,
1720 * or inferred from the inputs by
1721 * the backend.
1722 * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed
1723 * because one of the memory
1724 * types in weightDescriptors is
1725 * different from
1726 * ONNXIFI_MEMORY_TYPE_CPU.
1727 * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because
1728 * the shapes specified in weight
1729 * descriptors do not match the shapes
1730 * specified in the ONNX model graph,
1731 * or output or intermediate shapes
1732 * specified in the ONNX model graph do
1733 * not match the shapes inferred from
1734 * input shapes.
1735 * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because
1736 * data types specified in weight
1737 * descriptors do not match the data
1738 * types specified in ONNX model
1739 * graph, or output or intermediate
1740 * data types specified in the ONNX
1741 * model graph do not match the data
1742 * types inferred from graph inputs.
1743 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
1744 * backend could not allocate enough
1745 * system memory to parse, analyze, and
1746 * initialize the model graph.
1747 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
1748 * insufficient non-memory system
1749 * resources (e.g. file handles) to
1750 * initialize the graph.
1751 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
1752 * insufficient backend-specific memory
1753 * to initialize the graph.
1754 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
1755 * insufficient non-memory
1756 * backend-specific resources (e.g.
1757 * command queues) to initialize the
1758 * graph.
1759 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1760 * the backend was disconnected or
1761 * uninstalled from the system.
1762 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1763 * implementation experienced an
1764 * unrecovered internal error.
1765 */
1766ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1767 onnxInitGraph(
1768 onnxBackend backend,
1769 const uint64_t* auxPropertiesList,
1770 size_t onnxModelSize,
1771 const void* onnxModel,
1772 uint32_t weightsCount,
1773 const onnxTensorDescriptorV1* weightDescriptors,
1774 onnxGraph* graph,
1775 uint32_t maxSeqLength,
1776 void* deferredWeightReader);
1777
1778/**
1779 * Set locations for inputs and outputs of an ONNXIFI graph.
1780 *
1781 * The caller MUST ensure that the memory buffers specified for input and output
1782 * tensors remain accessible until all in-flight graph executions which use
1783 * specified buffer locations complete AND
1784 * - Either a next call to onnxSetGraphIO specifies different buffer locations
1785 * - Or the graph is deinitialized via onnxReleaseGraph
1786 * The caller can invalidate other data in tensor descriptors, including shape,
1787 * once the function returns.
1788 *
1789 * Calls to onnxRunGraph WILL use input and output locations specified in the
1790 * preceeding onnxSetGraphIO on the same graph. Asynchronous graph executions
1791 * that were in-flight before onnxSetGraphIO call will continue to use buffer
1792 * locations that were current when these graph executions started. An ONNXIFI
1793 * implementation MAY block inside onnxSetGraphIO until all in-flight graph
1794 * executions that started before the call complete.
1795 *
1796 * If a call to onnxSetGraphIO fails, it invalidates input and output locations
1797 * for the graph, and a subsequent call to onnxRunGraph will fail with
1798 * ONNXIFI_STATUS_UNIDENTIFIED_NAME.
1799 *
1800 * @param graph - graph handle created by onnxInitGraph.
1801 * @param inputsCount - number of elements in the inputDescriptors array.
1802 * @param[in] inputDescriptors - descriptors of input tensors for the graph.
1803 * Elements of this array must provide a location
1804 * for each ValueInfoProto.name listed in
1805 * ModelProto.graph.input of the ONNX graph.
1806 * If inputsCount is non-zero, inputDescriptors
1807 * pointer must be non-NULL.
1808 * @param outputsCount - number of elements in the outputDescriptors array.
1809 * Must be greater than zero.
1810 * @param[in] outputDescriptors - descriptors of output tensors for the graph.
1811 * outputDescriptors pointer must be non-NULL.
1812 * Elements of this array must provide a location
1813 * for each ValueInfoProto.name listed in
1814 * ModelProto.graph.output of the ONNX graph.
1815 *
1816 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph
1817 * inputs and outputs were matched to a memory
1818 * location.
1819 * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because
1820 * graph is not an ONNXIFI graph handle.
1821 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1822 * outputDescriptors pointer is NULL or
1823 * inputDescriptors pointer is NULL while
1824 * inputsCount is non-zero.
1825 * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because one of
1826 * the names in tensor descriptors doesn't
1827 * match blob name in ModelProto.graph.input
1828 * or ModelProto.graph.output, or the same
1829 * name appears in more than one tensor
1830 * descriptor.
1831 * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of
1832 * the shape dimensions is 0.
1833 * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because
1834 * one of the data types in
1835 * inputDescriptors or outputDescriptors
1836 * is unknown to the backend.
1837 * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because
1838 * one of the memory types in
1839 * inputDescriptors or
1840 * outputDescriptors is unknown to
1841 * the backend.
1842 * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed
1843 * because one of the memory
1844 * locations in inputDescriptors
1845 * or outputDescriptors is not
1846 * valid for the specified
1847 * memory type (e.g. NULL pointer
1848 * for ONNXIFI_MEMORY_TYPE_CPU).
1849 * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because one
1850 * of the tags in inputDescriptors or
1851 * outputDescriptors is unknown to the
1852 * backend (tag does not match
1853 * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1).
1854 * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the
1855 * backend does not support the
1856 * tensor shapes in an input or output
1857 * of one of the operators. The
1858 * problematic tensor shapes could be
1859 * directly specified through
1860 * inputDescriptors or
1861 * outputDescriptors argument,
1862 * or inferred from the inputs by the
1863 * backend. This error code can be
1864 * returned when the backend supports
1865 * variable-size inputs and outputs,
1866 * and the problematic tensor shape was
1867 * provided in the ValueInfoProto as a
1868 * symbolic variable.
1869 * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed
1870 * because the backend does not
1871 * support one of the memory
1872 * types in inputDescriptors or
1873 * outputDescriptors.
1874 * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because one
1875 * of the ValueInfoProto.name value in
1876 * ModelProto.graph.input or
1877 * ModelProto.graph.output doesn't have
1878 * a match in the inputDescriptors or
1879 * outputDescriptors.
1880 * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because
1881 * the shapes specified through
1882 * inputDescriptors or
1883 * outputDescriptors argument are
1884 * inconsistent with the shapes
1885 * specified in the ONNX model graph.
1886 * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because
1887 * data types specified through
1888 * inputDescriptors or
1889 * outputDescriptors argument are
1890 * inconsistent with the data types
1891 * specified in the ONNX model
1892 * graph.
1893 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
1894 * backend could not allocate enough
1895 * system memory to parse, analyze, and
1896 * initialize the tensor locations.
1897 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
1898 * insufficient non-memory system
1899 * resources (e.g. file handles) to
1900 * initialize the tensor locations.
1901 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
1902 * insufficient backend-specific memory
1903 * to initialize the tensor locations.
1904 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
1905 * insufficient non-memory
1906 * backend-specific resources (e.g.
1907 * command queues) to initialize the
1908 * tensor locations.
1909 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
1910 * the backend was disconnected or
1911 * uninstalled from the system.
1912 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
1913 * backend experienced an unrecovered
1914 * internal error.
1915 */
1916ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
1917 onnxSetGraphIO(
1918 onnxGraph graph,
1919 uint32_t inputsCount,
1920 const onnxTensorDescriptorV1* inputDescriptors,
1921 uint32_t outputsCount,
1922 const onnxTensorDescriptorV1* outputDescriptors);
1923
1924/**
1925 * Asynchronously execute operations in an ONNXIFI graph using pre-specified
1926 * locations for inputs and outputs.
1927 *
1928 * This function operates asynchronously: it doesn't require that the locations
1929 * for graph inputs graph inputs hold valid values before the function is
1930 * called, and doesn't guarantee that the locations for graph outputs hold
1931 * valid values when the function returns. Instead, two synchronization
1932 * primitives are used to signal to the backend when inputs are ready to use,
1933 * and to signal to the caller when outputs are ready to use. The only
1934 * synchronization primitive that is always available is onnxEvent
1935 * (ONNXIFI_SYNCHRONIZATION_EVENT memory fence type). If a backend supports
1936 * additional types of synchronization primitives, it must indicate them in
1937 * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES information query.
1938 *
1939 * The caller must successfully specify locations of input and output tensors
1940 * for the graph through onnxSetGraphIO before calling this function.
1941 *
1942 * @param graph - graph handle created by onnxInitGraph.
1943 * @param[in] inputFence - synchronization primitive that signals when graph
1944 * inputs are ready to use by the backend. The
1945 * synchronization primitive always must be initialized
1946 * by the caller.
1947 * @param[out] outputFence - synchronization primitive that signals when graph
1948 * outputs are ready to use by the caller. The type
1949 * of the synchronization primitive always must be
1950 * initialized by the caller. The type of the
1951 * synchronization primitive determines whether it
1952 * is initialized by the user before the call or by
1953 * the backend as a result of this call. Single-shot
1954 * synchronizatiom objects are initialized as a result
1955 * of the call. Reusable synchronization objects are
1956 * generally initialized by the user prior to the
1957 * call.
1958 *
1959 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph
1960 * inputs and outputs were matched to a memory
1961 * location.
1962 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
1963 * inputFence or outputFence pointer is
1964 * NULL.
1965 * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because
1966 * graph is not an ONNXIFI graph handle.
1967 * @retval ONNXIFI_STATUS_INVALID_FENCE_TYPE The function call failed because
1968 * the type of synchronization
1969 * primitive specified in inputFence
1970 * or outputFence is unknown to the
1971 * backend.
1972 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because
1973 * the memory synchronization primitive
1974 * specified in inputFence or outputFence
1975 * is not valid (e.g. NULL onnxEvent).
1976 * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because a tag
1977 * in inputFence or outputFence is
1978 * unknown to the backend (tag does not
1979 * match ONNXIFI_TAG_MEMORY_FENCE_V1).
1980 * @retval ONNXIFI_STATUS_UNSUPPORTED_FENCE_TYPE The function call failed
1981 * because the backend does not
1982 * support the type of
1983 * synchronization primitive
1984 * specified in inputFence or
1985 * outputFence.
1986 * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because
1987 * some of the ValueInfoProto.name
1988 * value in ModelProto.graph.input or
1989 * ModelProto.graph.output were not
1990 * specified in a call to
1991 * onnxSetGraphIO.
1992 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
1993 * backend could not allocate enough
1994 * system memory to execute the model
1995 * graph.
1996 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
1997 * insufficient non-memory system
1998 * resources (e.g. file handles) to
1999 * execute the model graph.
2000 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
2001 * insufficient backend-specific memory
2002 * to execute the graph.
2003 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
2004 * insufficient non-memory
2005 * backend-specific resources (e.g.
2006 * command queues) to execute the
2007 * graph.
2008 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
2009 * the backend was disconnected or
2010 * uninstalled from the system.
2011 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
2012 * backend experienced an unrecovered
2013 * internal error.
2014 */
2015ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
2016 onnxRunGraph(
2017 onnxGraph graph,
2018 const onnxMemoryFenceV1* inputFence,
2019 onnxMemoryFenceV1* outputFence);
2020
2021/**
2022 * Deinitialize an ONNXIFI graph and release associated resources.
2023 *
2024 * If there are in-flight asynchronous inference operations on this graph,
2025 * the function MUST block until all outstanding operations complete.
2026 *
2027 * @param graph - graph handle created by onnxInitGraph.
2028 *
2029 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the graph
2030 * resources were released to the operating
2031 * system.
2032 * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because graph
2033 * is not an ONNXIFI graph handle.
2034 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
2035 * graph backend experienced an
2036 * unrecovered internal error.
2037 */
2038ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI
2039 onnxReleaseGraph(
2040 onnxGraph graph);
2041
2042#ifdef __cplusplus
2043} /* extern "C" */
2044#endif
2045
2046#endif /* !defined(ONNXIFI_H) */
2047