1 | #ifndef ONNXIFI_H |
2 | #define ONNXIFI_H 1 |
3 | |
4 | #ifdef __cplusplus |
5 | extern "C" { |
6 | #endif |
7 | |
8 | #if defined(_WIN32) && defined(_M_IX86) |
9 | /* Windows x86 */ |
10 | #define ONNXIFI_ABI __stdcall |
11 | #elif defined(__i386__) |
12 | /* Linux x86 */ |
13 | #define ONNXIFI_ABI __attribute__((__cdecl__)) |
14 | #else |
15 | #define ONNXIFI_ABI |
16 | #endif |
17 | |
18 | #ifndef ONNXIFI_PUBLIC |
19 | #if defined(__ELF__) |
20 | #define ONNXIFI_PUBLIC __attribute__((__visibility__("default"))) |
21 | #elif defined(__MACH__) |
22 | #define ONNXIFI_PUBLIC __attribute__((__visibility__("default"))) |
23 | #elif defined(_WIN32) && defined(__GNUC__) |
24 | #ifdef ONNXIFI_BUILD_LIBRARY |
25 | #define ONNXIFI_PUBLIC __attribute__((__dllexport__)) |
26 | #else |
27 | #define ONNXIFI_PUBLIC __attribute__((__dllimport__)) |
28 | #endif |
29 | #elif defined(_WIN32) |
30 | #ifdef ONNXIFI_BUILD_LIBRARY |
31 | #define ONNXIFI_PUBLIC __declspec(dllexport) |
32 | #else |
33 | #define ONNXIFI_PUBLIC __declspec(dllimport) |
34 | #endif |
35 | #else |
36 | #define ONNXIFI_PUBLIC |
37 | #endif |
38 | #endif |
39 | |
40 | #ifndef ONNXIFI_CHECK_RESULT |
41 | #if defined(__GNUC__) && (__GNUC__ >= 4) |
42 | #define ONNXIFI_CHECK_RESULT __attribute__((__warn_unused_result__)) |
43 | #elif defined(_MSC_VER) && (_MSC_VER >= 1700) |
44 | #define ONNXIFI_CHECK_RESULT _Check_return_ |
45 | #else |
46 | #define ONNXIFI_CHECK_RESULT |
47 | #endif |
48 | #endif |
49 | |
50 | #include <stddef.h> |
51 | |
52 | #if !defined(ONNXIFI_NO_STDINT_H) |
53 | #if defined(_MSC_VER) && (_MSC_VER < 1600) |
54 | typedef signed __int8 int8_t; |
55 | typedef unsigned __int8 uint8_t; |
56 | typedef signed __int16 int16_t; |
57 | typedef unsigned __int16 uint16_t; |
58 | typedef signed __int32 int32_t; |
59 | typedef unsigned __int32 uint32_t; |
60 | typedef signed __int64 int64_t; |
61 | typedef unsigned __int64 uint64_t; |
62 | #else |
63 | #include <stdint.h> |
64 | #endif |
65 | #endif /* !defined(ONNXIFI_NO_STDINT_H) */ |
66 | |
67 | /** |
68 | * Opaque ONNXIFI backend ID. |
69 | * |
70 | * ONNXIFI backend is a combination of software layer and hardware device used |
71 | * to run an ONNX graph. Backend ID uniquely identifies a backend for the life- |
72 | * time of the process (i.e. no two hardware devices, software layers, or |
73 | * combinations of both can have the same backend ID). Backend ID stays valid |
74 | * even if the hardware device used by the backend disconnects from the system. |
75 | */ |
76 | typedef void* onnxBackendID; |
77 | /** |
78 | * Opaque ONNXIFI backend handle. |
79 | * ONNXIFI backend is a combination of software layer and hardware device used |
80 | * to run an ONNX graph. |
81 | */ |
82 | typedef void* onnxBackend; |
83 | /** Opaque ONNXIFI graph handle. */ |
84 | typedef void* onnxGraph; |
85 | /** Opaque ONNXIFI even handle. */ |
86 | typedef void* onnxEvent; |
87 | |
88 | /** Return code for ONNXIFI functions */ |
89 | typedef int32_t onnxStatus; |
90 | /** |
91 | * Type for enumeration values. |
92 | * |
93 | * The low 32 bits are reserved for standardized ONNXIFI values. |
94 | * The high 32 bits are reserved for vendor-specific extensions. Applications |
95 | * must check for specific vendor extensions before interpreting these bits. |
96 | */ |
97 | typedef uint64_t onnxEnum; |
98 | /** |
99 | * Type for bit fields. |
100 | * |
101 | * The low 32 bits are reserved for standardized ONNXIFI values. |
102 | * The high 32 bits are reserved for vendor-specific extensions. Applications |
103 | * must check for specific vendor extensions before interpreting these bits. |
104 | */ |
105 | typedef uint64_t onnxBitfield; |
106 | /** |
107 | * Type for pointers or handles for memory buffers. |
108 | * This type is intended to work not only for CPU-addressable memory, but also |
109 | * for device memory. uint64_t ensures the API can accomodate Vulkan buffers. |
110 | */ |
111 | typedef uint64_t onnxPointer; |
112 | |
113 | #define ONNXIFI_STATUS_SUCCESS 0x0000 |
114 | #define ONNXIFI_STATUS_FALLBACK 0x0001 |
115 | #define ONNXIFI_STATUS_INVALID_ID 0x0101 |
116 | #define ONNXIFI_STATUS_INVALID_SIZE 0x0102 |
117 | #define ONNXIFI_STATUS_INVALID_POINTER 0x0103 |
118 | #define ONNXIFI_STATUS_INVALID_PROTOBUF 0x0104 |
119 | #define ONNXIFI_STATUS_INVALID_MODEL 0x0105 |
120 | #define ONNXIFI_STATUS_INVALID_BACKEND 0x0106 |
121 | #define ONNXIFI_STATUS_INVALID_GRAPH 0x0107 |
122 | #define ONNXIFI_STATUS_INVALID_EVENT 0x0108 |
123 | #define ONNXIFI_STATUS_INVALID_STATE 0x0109 |
124 | #define ONNXIFI_STATUS_INVALID_NAME 0x010A |
125 | #define ONNXIFI_STATUS_INVALID_SHAPE 0x010B |
126 | #define ONNXIFI_STATUS_INVALID_DATATYPE 0x010C |
127 | #define ONNXIFI_STATUS_INVALID_MEMORY_TYPE 0x010D |
128 | #define ONNXIFI_STATUS_INVALID_MEMORY_LOCATION 0x010E |
129 | #define ONNXIFI_STATUS_INVALID_FENCE_TYPE 0x010F |
130 | #define ONNXIFI_STATUS_INVALID_PROPERTY 0x0110 |
131 | #define ONNXIFI_STATUS_UNSUPPORTED_TAG 0x0201 |
132 | #define ONNXIFI_STATUS_UNSUPPORTED_VERSION 0x0202 |
133 | #define ONNXIFI_STATUS_UNSUPPORTED_OPERATOR 0x0203 |
134 | #define ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE 0x0204 |
135 | #define ONNXIFI_STATUS_UNSUPPORTED_SHAPE 0x0205 |
136 | #define ONNXIFI_STATUS_UNSUPPORTED_DATATYPE 0x0206 |
137 | #define ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE 0x0207 |
138 | #define ONNXIFI_STATUS_UNSUPPORTED_FENCE_TYPE 0x0208 |
139 | #define ONNXIFI_STATUS_UNSUPPORTED_PROPERTY 0x0209 |
140 | #define ONNXIFI_STATUS_UNIDENTIFIED_NAME 0x0301 |
141 | #define ONNXIFI_STATUS_MISMATCHING_SHAPE 0x0302 |
142 | #define ONNXIFI_STATUS_MISMATCHING_DATATYPE 0x0303 |
143 | #define ONNXIFI_STATUS_NO_SYSTEM_MEMORY 0x0401 |
144 | #define ONNXIFI_STATUS_NO_DEVICE_MEMORY 0x0402 |
145 | #define ONNXIFI_STATUS_NO_SYSTEM_RESOURCES 0x0403 |
146 | #define ONNXIFI_STATUS_NO_DEVICE_RESOURCES 0x0404 |
147 | #define ONNXIFI_STATUS_BACKEND_UNAVAILABLE 0x0405 |
148 | #define ONNXIFI_STATUS_INTERNAL_ERROR 0x0406 |
149 | #define ONNXIFI_STATUS_FATAL_ERROR 0x0407 |
150 | |
151 | /** |
152 | * State of an ONNXIFI event object. |
153 | * |
154 | * Possible values: |
155 | * ONNXIFI_EVENT_STATE_INVALID |
156 | * ONNXIFI_EVENT_STATE_NONSIGNALLED |
157 | * ONNXIFI_EVENT_STATE_SIGNALLED |
158 | */ |
159 | typedef int32_t onnxEventState; |
160 | |
161 | /** |
162 | * State for an invalid onnxEvent. |
163 | */ |
164 | #define ONNXIFI_EVENT_STATE_INVALID 0 |
165 | /** |
166 | * Non-signalled onnxEvent state. |
167 | * onnxInitEvent creates events in non-signalled state. |
168 | */ |
169 | #define ONNXIFI_EVENT_STATE_NONSIGNALLED 0x16BD |
170 | /** |
171 | * Signalled onnxEvent state. |
172 | * onnxSignalEvent changes event state to signalled. |
173 | */ |
174 | #define ONNXIFI_EVENT_STATE_SIGNALLED 0x3395 |
175 | |
176 | /** Special-purpose accelerator for neural network */ |
177 | #define ONNXIFI_DEVICE_TYPE_NPU 0x01 |
178 | /** Digital signal processor */ |
179 | #define ONNXIFI_DEVICE_TYPE_DSP 0x02 |
180 | /** Graphics accelerator */ |
181 | #define ONNXIFI_DEVICE_TYPE_GPU 0x04 |
182 | /** General-purpose central processor */ |
183 | #define ONNXIFI_DEVICE_TYPE_CPU 0x08 |
184 | /** Field-programmable gate array */ |
185 | #define ONNXIFI_DEVICE_TYPE_FPGA 0x10 |
186 | /** |
187 | * Heterogeneous backend whichinternally arbitrates or distributes work between |
188 | * multiple device types. |
189 | */ |
190 | #define ONNXIFI_DEVICE_TYPE_HETEROGENEOUS 0x20 |
191 | |
192 | /** |
193 | * The backend supports multi-threaded access to ONNXIFI backend, graph, and |
194 | * event objects. E.g. onnxInitGraph can be called on a different thread than |
195 | * onnxInitBackend. |
196 | * |
197 | * If this capability it not indicated, ONNXIFI backend, graph, and event |
198 | * objects that relate to the backend must always be used on the same thread |
199 | * where the backend object was initialized. |
200 | */ |
201 | #define ONNXIFI_CAPABILITY_THREAD_SAFE 0x01 |
202 | /** |
203 | * The backend supports ONNX graphs with symbolic variables in the outer |
204 | * shape dimension (batch size), using TensorShapeProto.dim_param for |
205 | * ModelProto.graph.input.type.shape or ModelProto.graph.output.type.shape. |
206 | * |
207 | * The exact numerical value of the of all input and output tensors must be specified |
208 | * in the onnxSetGraphIO call(s). |
209 | */ |
210 | #define ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE 0x02 |
211 | /** |
212 | * The backend supports ONNX graphs with symbolic variables in the all |
213 | * shape dimensions, using TensorShapeProto.dim_param for |
214 | * ModelProto.graph.input.type.shape or ModelProto.graph.output.type.shape. |
215 | * |
216 | * Backends with this capability also MUST support |
217 | * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE capability. |
218 | * |
219 | * The exact numerical shape of all input and output tensors must be specified |
220 | * in the onnxSetGraphIO call(s). |
221 | */ |
222 | #define ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS 0x04 |
223 | /** |
224 | * The backend supports ONNX graphs with data-dependent outer shape dimension |
225 | * (batch size) of graph outputs. The ONNX graph would specify unknown outer |
226 | * shape dimension (batch size) using symbolic variables, so this capability |
227 | * requires ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE support. |
228 | * |
229 | * For outputs with data-dependent outer shape dimension (batch size) the value |
230 | * specified in onnxSetGraphIO call is interpreted as the upper limit. The exact |
231 | * numerical batch size of the output can be retrieved by attaching a Shape |
232 | * operator to the tensor with data-dependent shape and reading its output |
233 | * through ONNXIFI. |
234 | */ |
235 | #define ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE 0x08 |
236 | /** |
237 | * The backend supports ONNX graphs with data-dependent output shapes. |
238 | * The ONNX graph would specify unknown output shapes using symbolic variables, |
239 | * so this capability requires ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS support. |
240 | * |
241 | * Backends with this capability also MUST support |
242 | * ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE capability. |
243 | * |
244 | * For outputs with data-dependent shapes the shape specified in onnxSetGraphIO |
245 | * call is interpreted as the upper limit. The exact numerical shape of the |
246 | * output can be retrieved by attaching a Shape operator to the tensor with |
247 | * data-dependent shape and reading its output through ONNXIFI. |
248 | */ |
249 | #define ONNXIFI_CAPABILITY_VARIABLE_SIZE_OUTPUTS 0x10 |
250 | /** |
251 | * The backend uses a hot-pluggable device, and can be disconnected at any time. |
252 | * |
253 | * If the underlying device disconnects from the system, subsequent operations |
254 | * with the backend, or objects created on the backend, will fail with |
255 | * ONNXIFI_STATUS_BACKEND_UNAVAILABLE status code. |
256 | */ |
257 | #define ONNXIFI_CAPABILITY_HOT_PLUGGABLE 0x20 |
258 | |
259 | /** |
260 | * Type of the backend information. |
261 | * |
262 | * Possible values: |
263 | * ONNXIFI_BACKEND_ONNXIFI_VERSION |
264 | * ONNXIFI_BACKEND_NAME |
265 | * ONNXIFI_BACKEND_VENDOR |
266 | * ONNXIFI_BACKEND_VERSION |
267 | * ONNXIFI_BACKEND_EXTENSIONS |
268 | * ONNXIFI_BACKEND_DEVICE |
269 | * ONNXIFI_BACKEND_DEVICE_TYPE |
270 | * ONNXIFI_BACKEND_ONNX_IR_VERSION |
271 | * ONNXIFI_BACKEND_OPSET_VERSION |
272 | * ONNXIFI_BACKEND_CAPABILITIES |
273 | * ONNXIFI_BACKEND_INIT_PROPERTIES |
274 | * ONNXIFI_BACKEND_MEMORY_TYPES |
275 | * ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES |
276 | * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES |
277 | * ONNXIFI_BACKEND_MEMORY_SIZE |
278 | * ONNXIFI_BACKEND_MAX_GRAPH_SIZE |
279 | * ONNXIFI_BACKEND_MAX_GRAPH_COUNT |
280 | * ONNXIFI_BACKEND_MACS_FP32 |
281 | * ONNXIFI_BACKEND_MACS_FP16 |
282 | * ONNXIFI_BACKEND_MEMORY_BANDWIDTH |
283 | * ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH |
284 | * ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH |
285 | * ONNXIFI_BACKEND_PCI_BUS_ID |
286 | * ONNXIFI_BACKEND_PCI_DEVICE_ID |
287 | * ONNXIFI_BACKEND_PCI_DOMAIN_ID |
288 | * ONNXIFI_BACKEND_DIRECTX_ID |
289 | * ONNXIFI_BACKEND_CUDA_INDEX |
290 | * ONNXIFI_BACKEND_OPENCL_PLATFORM_ID |
291 | * ONNXIFI_BACKEND_OPENCL_DEVICE_ID |
292 | */ |
293 | typedef int32_t onnxBackendInfo; |
294 | |
295 | /** |
296 | * Major and minor version of ONNXIFI specification implemented by the backend. |
297 | * |
298 | * Since ONNXIFI 1.0, backends MUST support this information query. |
299 | * |
300 | * Value type: uint64_t. |
301 | * The high 32 bits specify the major version. |
302 | * The low 32 bits specify the minor version. |
303 | * |
304 | * Possible values: |
305 | * UINT64_C(0x0000000100000000) for ONNXIFI 1.0 |
306 | */ |
307 | #define ONNXIFI_BACKEND_ONNXIFI_VERSION 0 |
308 | |
309 | /** |
310 | * Marketing name of the backend (excluding the vendor name). |
311 | * |
312 | * Since ONNXIFI 1.0, backends MUST support this information query. |
313 | * |
314 | * This string MUST be in UTF-8 encoding and NOT locale-sensitive. |
315 | * |
316 | * Value type: char[], e.g.: |
317 | * "Caffe2" |
318 | * "Glow" |
319 | */ |
320 | #define ONNXIFI_BACKEND_NAME 1 |
321 | |
322 | /** |
323 | * Name of the backend vendor. |
324 | * |
325 | * Since ONNXIFI 1.0, backends MUST support this information query. |
326 | * |
327 | * This string MUST be in UTF-8 encoding and NOT locale-sensitive. |
328 | * |
329 | * Value type: char[], e.g.: |
330 | * "Facebook" |
331 | * "Marat Dukhan" |
332 | */ |
333 | #define ONNXIFI_BACKEND_VENDOR 2 |
334 | |
335 | /** |
336 | * Version of the backend software. Exact format is vendor-specific, but MUST be |
337 | * unique for the software release. |
338 | * |
339 | * Since ONNXIFI 1.0, backends MUST support this information query. |
340 | * |
341 | * This string MUST be in US-ASCII encoding and NOT locale-sensitive. |
342 | * |
343 | * Value type: char[], e.g.: |
344 | * "1.2.3" |
345 | * "1.2.3.0" |
346 | * "1.2.3-db3a4439d233276e25681fb4735b7f8e674dda65" |
347 | */ |
348 | #define ONNXIFI_BACKEND_VERSION 3 |
349 | |
350 | /** |
351 | * Space-separated list of vendor- or device-specific extensions supported on |
352 | * this backend. |
353 | * |
354 | * Since ONNXIFI 1.0, backends MUST support this information query. |
355 | * |
356 | * This string MUST be in US-ASCII encoding and NOT locale-sensitive. |
357 | * |
358 | * Value type: char[], e.g.: |
359 | * "" |
360 | * "onnx_clone_graph" |
361 | * "onnx_clone_graph fb_maskrcnn" |
362 | */ |
363 | #define ONNXIFI_BACKEND_EXTENSIONS 4 |
364 | |
365 | /** |
366 | * Descriptive name of the device (i.e. CPU, GPU, DSP, or NPU model). |
367 | * |
368 | * Since ONNXIFI 1.0, backends MUST support this information query. |
369 | * |
370 | * This string MUST be in UTF-8 encoding and NOT locale-sensitive. |
371 | * |
372 | * Value type: char[], e.g.: |
373 | * "nnDuino 123" |
374 | */ |
375 | #define ONNXIFI_BACKEND_DEVICE 5 |
376 | |
377 | /** |
378 | * Type of the device. |
379 | * |
380 | * Since ONNXIFI 1.0, backends MUST support this information query. |
381 | * |
382 | * Value type: onnxEnum. |
383 | * Possible values: |
384 | * ONNXIFI_DEVICE_TYPE_NPU |
385 | * ONNXIFI_DEVICE_TYPE_DSP |
386 | * ONNXIFI_DEVICE_TYPE_GPU |
387 | * ONNXIFI_DEVICE_TYPE_CPU |
388 | * ONNXIFI_DEVICE_TYPE_FPGA |
389 | * ONNXIFI_DEVICE_TYPE_HETEROGENEOUS |
390 | */ |
391 | #define ONNXIFI_BACKEND_DEVICE_TYPE 6 |
392 | |
393 | /** |
394 | * List of supported ONNX IR versions. |
395 | * |
396 | * Since ONNXIFI 1.0, backends MUST support this information query. |
397 | * |
398 | * Value type: char[], e.g.: |
399 | * "3" (IR version in ONNX 1.0) |
400 | * |
401 | * Possible values: space-separated list of supported ONNX IR versions, |
402 | * represented as decimal integers. ONNX IR versions must match values |
403 | * in ONNX Version enum. |
404 | */ |
405 | #define ONNXIFI_BACKEND_ONNX_IR_VERSION 7 |
406 | |
407 | /** |
408 | * List of supported operator set domains and maximum supported operator set |
409 | * version for each domain. |
410 | * |
411 | * Since ONNXIFI 1.0, backends MUST support this information query. |
412 | * |
413 | * Value type: char[], e.g.: |
414 | * "ai.onnx:1" (only operators in version 1 of default ONNX operator set) |
415 | * "ai.onnx:7" (operators up to version 7 of default ONNX operator set) |
416 | * "org.pytorch:2 ai.onnx:6 ai.facebook:1" |
417 | * |
418 | * Possible values: space-separated list of domain:max_version pairs where |
419 | * domain corresponds to OperatorSetIdProto.domain and max_version |
420 | * corresponds to the maximum value of OperatorSetIdProto.version supported |
421 | * by the backend for this domain. The backend MUST support all previous |
422 | * operator set versions as well. |
423 | */ |
424 | #define ONNXIFI_BACKEND_OPSET_VERSION 8 |
425 | |
426 | /** |
427 | * Optional features supported by the backend. |
428 | * |
429 | * Since ONNXIFI 1.0, backends MUST support this information query. |
430 | * |
431 | * Value type: onnxBitfield. |
432 | * Possible values: any combination of the following flags: |
433 | * ONNXIFI_CAPABILITY_THREAD_SAFE |
434 | * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE |
435 | * ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS |
436 | * ONNXIFI_CAPABILITY_VARIABLE_BATCH_SIZE |
437 | * ONNXIFI_CAPABILITY_VARIABLE_SIZE_OUTPUTS |
438 | * ONNXIFI_CAPABILITY_HOT_PLUGGABLE |
439 | * or any vendor-specific flags in the high 32 bits of the bit field. |
440 | */ |
441 | #define ONNXIFI_BACKEND_CAPABILITIES 10 |
442 | |
443 | /** |
444 | * Auxiliary initialization properties supported by the backend. |
445 | * |
446 | * Since ONNXIFI 1.0, backends MUST support this information query. |
447 | * |
448 | * Value type: onnxBitfield. |
449 | * Possible values: any combination of vendor-specific flags in high 32 bits of |
450 | * the bit field. |
451 | */ |
452 | #define ONNXIFI_BACKEND_INIT_PROPERTIES 11 |
453 | |
454 | /** |
455 | * Memory types supported for graph inputs and outputs. |
456 | * |
457 | * Since ONNXIFI 1.0, backends MUST support this information query. |
458 | * |
459 | * Value type: onnxBitfield. |
460 | * Possible values are any combination of the following flags: |
461 | * ONNXIFI_MEMORY_TYPE_CPU (always supported) |
462 | * ONNXIFI_MEMORY_TYPE_CUDA_BUFFER |
463 | * ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER |
464 | * ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D |
465 | * ONNXIFI_MEMORY_TYPE_D3D_RESOURCE |
466 | * or any vendor-specific flags in the high 32 bits of the bit field. |
467 | */ |
468 | #define ONNXIFI_BACKEND_MEMORY_TYPES 12 |
469 | |
470 | /** |
471 | * Auxiliary initialization properties supported by graphs on the backend. |
472 | * |
473 | * Since ONNXIFI 1.0, backends MUST support this information query. |
474 | * |
475 | * Value type: onnxBitfield. |
476 | * Possible values: any combination of vendor-specific flags in high 32 bits of |
477 | * the bit field. |
478 | */ |
479 | #define ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES 13 |
480 | |
481 | /** |
482 | * Memory synchronization primitives supported for graph inputs and outputs. |
483 | * |
484 | * Since ONNXIFI 1.0, backends MUST support this information query. |
485 | * |
486 | * Possible values are any combination of the following flags: |
487 | * ONNXIFI_SYNCHRONIZATION_EVENT (onnxEvent, always supported) |
488 | * ONNXIFI_SYNCHRONIZATION_IMPLICIT |
489 | * or any vendor-specific flags in the high 32 bits of the bit field. |
490 | */ |
491 | #define ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES 14 |
492 | |
493 | /** |
494 | * Maximum amount of memory, in bytes, available to the use by the backend. |
495 | * |
496 | * Since ONNXIFI 1.0, backends MUST support this information query. |
497 | * |
498 | * Value type: uint64_t. |
499 | */ |
500 | #define ONNXIFI_BACKEND_MEMORY_SIZE 20 |
501 | |
502 | /** |
503 | * Maximum size of network parameters, in bytes. |
504 | * |
505 | * Since ONNXIFI 1.0, backends MUST support this information query. |
506 | * |
507 | * Value type: uint64_t. |
508 | */ |
509 | #define ONNXIFI_BACKEND_MAX_GRAPH_SIZE 21 |
510 | |
511 | /** |
512 | * Maximum number of independent network graphs supported by the backend. |
513 | * |
514 | * Since ONNXIFI 1.0, backends MUST support this information query. |
515 | * |
516 | * Value type: uint64_t. |
517 | */ |
518 | #define ONNXIFI_BACKEND_MAX_GRAPH_COUNT 22 |
519 | |
520 | /** |
521 | * Number of FP32 multiply-accumulate operations per second delivered by the |
522 | * backend. |
523 | * |
524 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
525 | * information query. |
526 | * |
527 | * Value type: uint64_t. |
528 | * If the backend does not support FP32 computation, the value MUST be 0. |
529 | */ |
530 | #define ONNXIFI_BACKEND_MACS_FP32 30 |
531 | |
532 | /** |
533 | * Number of FP16 multiply-accumulate operations per second delivered by the |
534 | * backend. |
535 | * |
536 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
537 | * information query. |
538 | * |
539 | * Value type: uint64_t. |
540 | * If the backend does not support FP16 computation, the value MUST be 0. |
541 | */ |
542 | #define ONNXIFI_BACKEND_MACS_FP16 31 |
543 | |
544 | /** |
545 | * Bandwidth, in bytes per second, of the global memory specific to the backend |
546 | * device. |
547 | * |
548 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
549 | * information query. |
550 | * |
551 | * Value type: uint64_t. |
552 | */ |
553 | #define ONNXIFI_BACKEND_MEMORY_BANDWIDTH 35 |
554 | |
555 | /** |
556 | * Bandwidth, in bytes per second, of transferring data from cacheable |
557 | * CPU-allocated memory to the backend device. |
558 | * |
559 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
560 | * information query. |
561 | * |
562 | * Value type: uint64_t. |
563 | */ |
564 | #define ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH 36 |
565 | |
566 | /** |
567 | * Bandwidth, in bytes per second, of transferring data to cacheable |
568 | * CPU-allocated memory from the backend device. |
569 | * |
570 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
571 | * information query. |
572 | * |
573 | * Value type: uint64_t. |
574 | */ |
575 | #define ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH 37 |
576 | |
577 | /** |
578 | * PCI bus ID of the backend device. |
579 | * |
580 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
581 | * information query. |
582 | * |
583 | * Value type: uint64_t. |
584 | */ |
585 | #define ONNXIFI_BACKEND_PCI_BUS_ID 40 |
586 | |
587 | /** |
588 | * PCI device ID of the backend device. |
589 | * |
590 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
591 | * information query. |
592 | * |
593 | * Value type: uint64_t. |
594 | */ |
595 | #define ONNXIFI_BACKEND_PCI_DEVICE_ID 41 |
596 | |
597 | /** |
598 | * PCI domain/function ID of the backend device. |
599 | * |
600 | * Since ONNXIFI 1.0, backends are recommended, but not required to support this |
601 | * information query. |
602 | * |
603 | * Value type: uint64_t. |
604 | */ |
605 | #define ONNXIFI_BACKEND_PCI_DOMAIN_ID 42 |
606 | |
607 | /** |
608 | * DirectX ID of the backend device. |
609 | * |
610 | * This is the value that would be returned by ID3D12Device::GetAdapterLuid() |
611 | * for the hardware device used by the backend. |
612 | * |
613 | * Since ONNXIFI 1.0, DXGI-based backends are recommended, but not required to |
614 | * support this information query. |
615 | * |
616 | * Value type: LUID (8 bytes). |
617 | */ |
618 | #define ONNXIFI_BACKEND_DIRECTX_ID 43 |
619 | |
620 | /** |
621 | * CUDA index of the backend device. |
622 | * |
623 | * Since ONNXIFI 1.0, CUDA-based backends are recommended, but not required to |
624 | * support this information query. |
625 | * |
626 | * Value type: uint64_t. |
627 | */ |
628 | #define ONNXIFI_BACKEND_CUDA_INDEX 44 |
629 | |
630 | /** |
631 | * OpenCL platform ID for the backend device. |
632 | * This platform ID is guaranteed to remain valid for the lifetime of ONNXIFI |
633 | * objects related to the same ONNXIFI backend (backend ID, backend, graph, |
634 | * event). |
635 | * |
636 | * Since ONNXIFI 1.0, OpenCL-based backends are recommended, but not required to |
637 | * support this information query. |
638 | * |
639 | * Value type: cl_platform_id. |
640 | */ |
641 | #define ONNXIFI_BACKEND_OPENCL_PLATFORM_ID 45 |
642 | |
643 | /** |
644 | * OpenCL device ID for the backend device. |
645 | * This device ID is guaranteed to remain valid for the lifetime of ONNXIFI |
646 | * objects related to the same ONNXIFI backend (backend ID, backend, graph, |
647 | * event). |
648 | * |
649 | * Since ONNXIFI 1.0, OpenCL-based backends are recommended, but not required to |
650 | * support this information query. |
651 | * |
652 | * Value type: cl_device_id. |
653 | */ |
654 | #define ONNXIFI_BACKEND_OPENCL_DEVICE_ID 46 |
655 | |
656 | /* Note: the data type values match ONNX TensorProto.DataType enum */ |
657 | #define ONNXIFI_DATATYPE_UNDEFINED 0 |
658 | #define ONNXIFI_DATATYPE_FLOAT16 10 |
659 | #define ONNXIFI_DATATYPE_FLOAT32 1 |
660 | #define ONNXIFI_DATATYPE_FLOAT64 11 |
661 | #define ONNXIFI_DATATYPE_INT8 3 |
662 | #define ONNXIFI_DATATYPE_INT16 5 |
663 | #define ONNXIFI_DATATYPE_INT32 6 |
664 | #define ONNXIFI_DATATYPE_INT64 7 |
665 | #define ONNXIFI_DATATYPE_UINT8 2 |
666 | #define ONNXIFI_DATATYPE_UINT16 4 |
667 | #define ONNXIFI_DATATYPE_UINT32 12 |
668 | #define ONNXIFI_DATATYPE_UINT64 13 |
669 | #define ONNXIFI_DATATYPE_COMPLEX64 14 |
670 | #define ONNXIFI_DATATYPE_COMPLEX128 15 |
671 | #define ONNXIFI_DATATYPE_BFLOAT16 16 |
672 | |
673 | /** Cacheable CPU memory */ |
674 | #define ONNXIFI_MEMORY_TYPE_CPU 0 |
675 | /** CUDA memory buffer (allocated via cudaMalloc/cuMalloc). */ |
676 | #define ONNXIFI_MEMORY_TYPE_CUDA_BUFFER 1 |
677 | /** OpenCL cl_mem object for a buffer or sub-buffer. */ |
678 | #define ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER 2 |
679 | /** OpenGL ES 2.0+ 2D Texture. */ |
680 | #define ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D 4 |
681 | /** Direct3D resource. */ |
682 | #define ONNXIFI_MEMORY_TYPE_D3D_RESOURCE 8 |
683 | |
684 | /** |
685 | * Terminates the list of auxiliary backend initialization properties passed to |
686 | * onnxInitBackend. |
687 | */ |
688 | #define ONNXIFI_BACKEND_PROPERTY_NONE 0 |
689 | /** |
690 | * Optimization target for graphs initialized on the backend. |
691 | * |
692 | * Possible values: |
693 | * ONNXIFI_OPTIMIZATION_HIGH_THROUGHPUT |
694 | * ONNXIFI_OPTIMIZATION_LOW_LATENCY |
695 | * ONNXIFI_OPTIMIZATION_LOW_POWER |
696 | * ONNXIFI_OPTIMIZATION_LOW_DELAY |
697 | * ONNXIFI_OPTIMIZATION_AOT |
698 | */ |
699 | #define ONNXIFI_BACKEND_PROPERTY_OPTIMIZATION 1 |
700 | /** |
701 | * Logging verbosity level for the backend. |
702 | * |
703 | * If this property is not specified during initialization, the backend should |
704 | * assume ONNXIFI_LOG_LEVEL_WARNING logging verbosity level. |
705 | * |
706 | * Possible values: |
707 | * ONNXIFI_LOG_LEVEL_ERROR |
708 | * ONNXIFI_LOG_LEVEL_WARNING |
709 | * ONNXIFI_LOG_LEVEL_INFO |
710 | * ONNXIFI_LOG_LEVEL_DEBUG |
711 | */ |
712 | #define ONNXIFI_BACKEND_PROPERTY_LOG_LEVEL 2 |
713 | /** |
714 | * CUDA stream to be used by the backend. |
715 | * CUDA stream must be created on the CUDA device used by the ONNXIFI backend. |
716 | * Users can query which CUDA device is used by the ONNXIFI backend by calling |
717 | * onnxGetBackendInfo with ONNXIFI_BACKEND_CUDA_INDEX info type. |
718 | * |
719 | * If this property is not specified during initialization, the backend can |
720 | * create a new CUDA stream for the device, or use a default CUDA stream. |
721 | * |
722 | * Possible values: cudaStream_t or CUstream object, cast to uint64_t. |
723 | */ |
724 | #define ONNXIFI_BACKEND_CUDA_STREAM 4 |
725 | /** |
726 | * OpenCL context to be used by the backend. |
727 | * The context must be created with the OpenCL device ID and the OpenCL platform |
728 | * ID used by the ONNXIFI backend. Users can query which OpenCL device ID and |
729 | * OpenCL platform ID are used by the ONNXIFI backend by calling |
730 | * onnxGetBackendInfo with ONNXIFI_BACKEND_OPENCL_PLATFORM_ID |
731 | * and ONNXIFI_BACKEND_OPENCL_DEVICE_ID info types. |
732 | * |
733 | * If this property is not specified during initialization, the backend will |
734 | * create a new OpenCL context for the device. |
735 | * |
736 | * Possible values: cl_context object, cast to uint64_t. |
737 | */ |
738 | #define ONNXIFI_BACKEND_OPENCL_CONTEXT 8 |
739 | |
740 | /** |
741 | * Terminates the list of auxiliary graph initialization properties passed to |
742 | * onnxInitGraph. |
743 | */ |
744 | #define ONNXIFI_GRAPH_PROPERTY_NONE 0 |
745 | |
746 | /** |
747 | * Optimize graph representation and compilation for highest throughput. |
748 | */ |
749 | #define ONNXIFI_OPTIMIZATION_HIGH_THROUGHPUT 0 |
750 | |
751 | /** |
752 | * Optimize graph representation and compilation for lowest latency. |
753 | */ |
754 | #define ONNXIFI_OPTIMIZATION_LOW_LATENCY 1 |
755 | |
756 | /** |
757 | * Optimize graph representation and compilation for lowest power consumption. |
758 | */ |
759 | #define ONNXIFI_OPTIMIZATION_LOW_POWER 2 |
760 | |
761 | /** |
762 | * Optimize graph representation and compilation for lowest delay until first |
763 | * result. |
764 | */ |
765 | #define ONNXIFI_OPTIMIZATION_LOW_DELAY 3 |
766 | |
767 | /** |
768 | * Optimize graph representation and compilation using potentially |
769 | * longer-running ahead-of-time optimization. |
770 | */ |
771 | #define ONNXIFI_OPTIMIZATION_AOT 4 |
772 | |
773 | /** |
774 | * Log events which caused a failure in an ONNXIFI function call. |
775 | */ |
776 | #define ONNXIFI_LOG_LEVEL_ERROR 4 |
777 | |
778 | /** |
779 | * Log events in ONNXIFI_LOG_LEVEL_ERROR and events which caused |
780 | * a performance, accuracy, or quality of service degradation in a backend. |
781 | * Enabling this logging level SHOULD NOT have a measurable effect on |
782 | * performance. |
783 | */ |
784 | #define ONNXIFI_LOG_LEVEL_WARNING 3 |
785 | |
786 | /** |
787 | * Log events in ONNXIFI_LOG_LEVEL_WARNING and high-level status information |
788 | * about operation of a backend. Enabling this logging level MAY cause a small |
789 | * degradation in performance. |
790 | */ |
791 | #define ONNXIFI_LOG_LEVEL_INFO 2 |
792 | |
793 | /** |
794 | * Log events in ONNXIFI_LOG_LEVEL_INFO and detailed status information about |
795 | * operations of a backend. Enabling this logging level MAY cause a serious |
796 | * degradation in performance. |
797 | */ |
798 | #define ONNXIFI_LOG_LEVEL_DEBUG 1 |
799 | |
800 | /** |
801 | * Tag for version 1 of tensor descriptor structure (onnxTensorDescriptorV1). |
802 | * |
803 | * The tag is unique for this version. If ONNXIFI introduce a new version of |
804 | * the tensor descriptor structure in the future, it will get a new tag value. |
805 | */ |
806 | #define ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1 0x43DFBF69 |
807 | |
808 | typedef struct onnxTensorDescriptorV1 { |
809 | /** |
810 | * 32-bit tag needed to distinguish different versions of a tensor descriptor |
811 | * structure. In the onnxTensorDescriptorV1 structure, the tag MUST be set to |
812 | * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1. If ONNXIFI introduce a new version of the |
813 | * tensor descriptor structure in the future, it WILL have 32-bit tag with a |
814 | * different value as the first member of the structure. |
815 | * |
816 | * ONNXIFI implementations MUST validate tag before accessing any other |
817 | * members of the structure. |
818 | */ |
819 | int32_t tag; |
820 | /** |
821 | * Name of the blob corresponding to this tensor in the ONNX model. The name |
822 | * must exactly match the ValueInfoProto.name of one of the |
823 | * ModelProto.graph.input or ModelProto.graph.output |
824 | */ |
825 | const char* name; |
826 | /** |
827 | * Data type of the elements in the tensor. |
828 | * |
829 | * Possible values: |
830 | * ONNXIFI_DATATYPE_FLOAT16 |
831 | * ONNXIFI_DATATYPE_FLOAT32 |
832 | * ONNXIFI_DATATYPE_FLOAT64 |
833 | * ONNXIFI_DATATYPE_INT8 |
834 | * ONNXIFI_DATATYPE_INT16 |
835 | * ONNXIFI_DATATYPE_INT32 |
836 | * ONNXIFI_DATATYPE_INT64 |
837 | * ONNXIFI_DATATYPE_UINT8 |
838 | * ONNXIFI_DATATYPE_UINT16 |
839 | * ONNXIFI_DATATYPE_UINT32 |
840 | * ONNXIFI_DATATYPE_UINT64 |
841 | * ONNXIFI_DATATYPE_COMPLEX64 |
842 | * ONNXIFI_DATATYPE_COMPLEX128 |
843 | */ |
844 | onnxEnum dataType; |
845 | /** |
846 | * Type of memory that stores the tensor. |
847 | * |
848 | * ONNXIFI_MEMORY_TYPE_CPU memory type is always supported by the backend, but |
849 | * other memory types are optional. The use MUST call onnxGetBackendInfo with |
850 | * ONNXIFI_BACKEND_MEMORY_TYPES to check if a particular memory type is |
851 | * supported before using it. |
852 | * |
853 | * If the memory type is different than ONNXIFI_MEMORY_TYPE_CPU, it must be |
854 | * allocated on the same device as the backend. |
855 | * |
856 | * Possible values: |
857 | * ONNXIFI_MEMORY_TYPE_CPU (always supported) |
858 | * ONNXIFI_MEMORY_TYPE_CUDA_BUFFER (support is optional) |
859 | * ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER (support is optional) |
860 | * ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D (support is optional) |
861 | * ONNXIFI_MEMORY_TYPE_D3D_RESOURCE (support is optional) |
862 | */ |
863 | onnxEnum memoryType; |
864 | /** |
865 | * Number of dimensions in the tensor. |
866 | * For a scalar, the number of dimensions is 0. |
867 | */ |
868 | uint32_t dimensions; |
869 | /** |
870 | * Dimensions of the tensor. |
871 | * For a scalar, this pointer can be NULL. |
872 | */ |
873 | const uint64_t* shape; |
874 | /** |
875 | * Indicates the axis along which quantization parameters are shared. |
876 | */ |
877 | uint32_t quantizationAxis; |
878 | /** |
879 | * This indicates how many quantization parameters this tensor has in the |
880 | * scales and biases fields. If this tensor is not quantized then this is 0. |
881 | * This must be less than or equal to the size of the tensor in the |
882 | * quantizationAxis dimension. Note that quantization parameters will be |
883 | * shared amongst equally sized groups along the quantizationAxis. |
884 | */ |
885 | uint64_t quantizationParams; |
886 | /** |
887 | * Quantized tensor scale info. |
888 | * For non-quantized tensor, these scales are not valid and should not exist. |
889 | */ |
890 | const float* scales; |
891 | /** |
892 | * Quantized tensor bias info. |
893 | * For non-quantized tensor, these biases are not valid and should not exist. |
894 | */ |
895 | const int32_t* biases; |
896 | |
897 | /** |
898 | * Whether this tensor is an offline tensor, which doesn't carry actual data. |
899 | */ |
900 | uint8_t isOffline; |
901 | |
902 | /** |
903 | * Pointers to tensor data. |
904 | * |
905 | * Interpretation depends on memoryType: |
906 | * - ONNXIFI_MEMORY_TYPE_CPU: buffer is a valid pointer to CPU memory. |
907 | * - ONNXIFI_MEMORY_TYPE_CUDA_BUFFER: buffer is a valid pointer to CUDA |
908 | * device memory, allocated via cudaMalloc or cuMalloc. CUDA device memory |
909 | * must be allocated on the same device as the backend. |
910 | * - ONNXIFI_MEMORY_TYPE_OPENCL_BUFFER: buffer is a cl_mem handle for an |
911 | * OpenCL buffer or a sub-buffer. cl_mem object must be allocated on the |
912 | * same OpenCL context as the backend. The context must be specified via |
913 | * ONNXIFI_BACKEND_OPENCL_CONTEXT initialization property to |
914 | * onnxInitBackend. |
915 | * - ONNXIFI_MEMORY_TYPE_OPENGLES_TEXTURE_2D: buffer is a name of a 2D |
916 | * texture created by glGenTextures. The texture must be allocated on the |
917 | * same device as the backend. |
918 | * - ONNXIFI_MEMORY_TYPE_D3D_RESOURCE: TBD |
919 | */ |
920 | onnxPointer buffer; |
921 | } onnxTensorDescriptorV1; |
922 | |
923 | /** |
924 | * Synchronization using ONNXIFI event object (onnxEvent). |
925 | */ |
926 | #define ONNXIFI_SYNCHRONIZATION_EVENT 0 |
927 | /** |
928 | * Implicit synchronization of inputs and outputs access with the caller. |
929 | * The details are backend-specific, and may involve extra parameters passed |
930 | * during backend initialization. |
931 | * |
932 | * Examples: |
933 | * - CUDA-based backends could implicitly synchronize with the caller through |
934 | * the use of the same CUDA stream. |
935 | * - OpenCL-based backends could implicitly synchronize with the caller through |
936 | * the use of the same in-order OpenCL command queue. |
937 | */ |
938 | #define ONNXIFI_SYNCHRONIZATION_IMPLICIT 2 |
939 | |
940 | /** |
941 | * Tag for version 1 of memory fence structure (onnxMemoryFenceV1). |
942 | * |
943 | * The tag is unique for this version. If ONNXIFI introduce a new version of |
944 | * the memory fence structure in the future, it will get a new tag value. |
945 | */ |
946 | #define ONNXIFI_TAG_MEMORY_FENCE_V1 0x23E08AAB |
947 | |
948 | typedef struct onnxMemoryFenceV1 { |
949 | /** |
950 | * 32-bit tag needed to distinguish different versions of a memory fence |
951 | * structure. In the onnxMemoryFenceV1 structure, the tag MUST be set to |
952 | * ONNXIFI_TAG_MEMORY_FENCE_V1. If ONNXIFI introduce a new version of the |
953 | * memory fence structure in the future, it WILL have 32-bit tag with a |
954 | * different value as the first member of the structure. |
955 | * |
956 | * ONNXIFI implementations MUST validate tag before accessing any other |
957 | * members of the structure. |
958 | */ |
959 | int32_t tag; |
960 | /** |
961 | * Type of memory synchronization primitive. |
962 | * |
963 | * Possible values: |
964 | * ONNXIFI_SYNCHRONIZATION_EVENT (onnxEvent, always supported) |
965 | * ONNXIFI_SYNCHRONIZATION_IMPLICIT |
966 | */ |
967 | onnxEnum type; |
968 | union { |
969 | /** |
970 | * Handle for a single-shot ONNXIFI event used as a synchronization |
971 | * primitive. Event for the input fence must be created by the caller to |
972 | * onnxRunGraph. Event for the output fence is created by implementation of |
973 | * onnxRunGraph, and stored into the output memory fence structure before |
974 | * onnxRunGraph returns. |
975 | */ |
976 | onnxEvent event; |
977 | }; |
978 | } onnxMemoryFenceV1; |
979 | |
980 | /* Function pointer declarations for dynamic loading */ |
981 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
982 | (ONNXIFI_ABI* onnxGetBackendIDsFunction)( |
983 | onnxBackendID* backendIDs, |
984 | size_t* numBackends); |
985 | typedef onnxStatus |
986 | (ONNXIFI_ABI* onnxReleaseBackendIDFunction)( |
987 | onnxBackendID backendID); |
988 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
989 | (ONNXIFI_ABI* onnxGetBackendInfoFunction)( |
990 | onnxBackendID backendID, |
991 | onnxBackendInfo infoType, |
992 | void* infoValue, |
993 | size_t* infoValueSize); |
994 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
995 | (ONNXIFI_ABI* onnxGetBackendCompatibilityFunction)( |
996 | onnxBackendID backendID, |
997 | size_t onnxModelSize, |
998 | const void* onnxModel); |
999 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1000 | (ONNXIFI_ABI* onnxInitBackendFunction)( |
1001 | onnxBackendID backendID, |
1002 | const uint64_t* auxPropertiesList, |
1003 | onnxBackend* backend); |
1004 | typedef onnxStatus |
1005 | (ONNXIFI_ABI* onnxReleaseBackendFunction)( |
1006 | onnxBackend backend); |
1007 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1008 | (ONNXIFI_ABI* onnxInitEventFunction)( |
1009 | onnxBackend backend, |
1010 | onnxEvent* event); |
1011 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1012 | (ONNXIFI_ABI* onnxSignalEventFunction)( |
1013 | onnxEvent event); |
1014 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1015 | (ONNXIFI_ABI* onnxGetEventStateFunction)( |
1016 | onnxEvent event, |
1017 | onnxEventState* state); |
1018 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1019 | (ONNXIFI_ABI* onnxWaitEventFunction)( |
1020 | onnxEvent event); |
1021 | typedef onnxStatus |
1022 | (ONNXIFI_ABI* onnxReleaseEventFunction)( |
1023 | onnxEvent event); |
1024 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1025 | (ONNXIFI_ABI* onnxInitGraphFunction)( |
1026 | onnxBackend backend, |
1027 | const uint64_t* auxPropertiesList, |
1028 | size_t onnxModelSize, |
1029 | const void* onnxModel, |
1030 | uint32_t weightsCount, |
1031 | const onnxTensorDescriptorV1* weightDescriptors, |
1032 | onnxGraph* graph, |
1033 | uint32_t maxSeqLength, |
1034 | void* deferredWeightReader); |
1035 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1036 | (ONNXIFI_ABI* onnxSetGraphIOFunction)( |
1037 | onnxGraph graph, |
1038 | uint32_t inputsCount, |
1039 | const onnxTensorDescriptorV1* inputDescriptors, |
1040 | uint32_t outputsCount, |
1041 | const onnxTensorDescriptorV1* outputDescriptors); |
1042 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
1043 | (ONNXIFI_ABI* onnxRunGraphFunction)( |
1044 | onnxGraph graph, |
1045 | const onnxMemoryFenceV1* inputFence, |
1046 | onnxMemoryFenceV1* outputFence); |
1047 | typedef onnxStatus |
1048 | (ONNXIFI_ABI* onnxReleaseGraphFunction)( |
1049 | onnxGraph graph); |
1050 | |
1051 | /** |
1052 | * Get stable IDs of available backends on the system. |
1053 | * |
1054 | * ONNXIFI backend is a combination of software layer and hardware device used |
1055 | * to run an ONNX graph. The same software layer may expose multiple backends |
1056 | * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend |
1057 | * for GPU and another for CPU, both implemented in the same software). Backends |
1058 | * implemented in the same software, but targeting different devices (e.g. |
1059 | * "MyNN" for CPU and "MyNN" for GPU) have different backend IDs. |
1060 | * |
1061 | * Note that some (hot-pluggable) backends can be connected and disconnected at |
1062 | * any time, and thus subsequent calls to this function may return different |
1063 | * number or set of backend IDs. The returned IDs, however, stay valid even if |
1064 | * the hardware device used by the backend disconnects from the system. |
1065 | * |
1066 | * To avoid resource leak, the backend ID MUST be released through a call to |
1067 | * onnxReleaseBackendID when it is no longer needed. |
1068 | * |
1069 | * @param backendIDs[out] - pointer to the memory location where the backend IDs |
1070 | * will be returned. If the pointer is NULL, it is |
1071 | * ignored, and the function returns only the number |
1072 | * of backend IDs through numBackendIDs pointer. |
1073 | * @param numBackendIDs[in,out] - pointer to a variable specifying number of |
1074 | * available backends. On function entry, the |
1075 | * variable MUST contain the capacity, in number |
1076 | * of backend IDs, of the memory buffer specified |
1077 | * by backendIDs. For successful completion, this |
1078 | * capacity must be at least as large as the |
1079 | * number of available backends. If the function |
1080 | * completes with either ONNXIFI_STATUS_SUCCESS |
1081 | * or ONNXIFI_STATUS_FALLBACK status codes, the |
1082 | * number of backend IDs written into backendIDs |
1083 | * buffer is stored in the variable specified by |
1084 | * this pointer. |
1085 | * |
1086 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded, and backend IDs |
1087 | * are stored in the location specified by |
1088 | * backendIDs, and the number of the backends |
1089 | * is stored in the location specified by |
1090 | * numBackends. |
1091 | * @retval ONNXIFI_STATUS_FALLBACK The function call completed, but the |
1092 | * backend IDs were not stored in the |
1093 | * location specified by backendIDs, either |
1094 | * because it is NULL, or because the size of |
1095 | * the memory buffer is insufficient to store |
1096 | * all available backend IDs. The number of |
1097 | * available backends is stored in the |
1098 | * location specified by numBackends. |
1099 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1100 | * numBackends is NULL. |
1101 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
1102 | * system failed to allocate memory |
1103 | * to store backend ID information. |
1104 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1105 | * implementation experienced an |
1106 | * unrecovered internal error. |
1107 | */ |
1108 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1109 | onnxGetBackendIDs( |
1110 | onnxBackendID* backendIDs, |
1111 | size_t* numBackends); |
1112 | |
1113 | /** |
1114 | * Deinitialize ONNXIFI backend IDs and release associated resources. |
1115 | * |
1116 | * The user MUST deinitialize all objects created with this backend ID |
1117 | * (onnxBackend, onnxGraph, onnxEvent) before calling this function to |
1118 | * deinitialize the backend ID. |
1119 | * |
1120 | * @param backendID - backend ID returned by onnxGetBackendIDs. |
1121 | * |
1122 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the resources |
1123 | * associated to the backend ID were released to |
1124 | * the operating system. |
1125 | * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID |
1126 | * is not an ONNXIFI backend ID. |
1127 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1128 | * implementation experienced an |
1129 | * unrecovered internal error. |
1130 | */ |
1131 | ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI |
1132 | onnxReleaseBackendID( |
1133 | onnxBackendID backendID); |
1134 | |
1135 | /** |
1136 | * Query high-level information about the backend and its target device. |
1137 | * |
1138 | * ONNXIFI backend is a combination of software layer and hardware device used |
1139 | * to run an ONNX graph. The same software layer may expose multiple backends |
1140 | * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend |
1141 | * for GPU and another for CPU, both implemented in the same software). |
1142 | * |
1143 | * The content, data type, and availability of information provided by this |
1144 | * function depends on infoType value as specified below: |
1145 | * |
1146 | * infoType value data type support |
1147 | * ONNXIFI_BACKEND_ONNXIFI_VERSION uint64_t required |
1148 | * ONNXIFI_BACKEND_NAME char[] required |
1149 | * ONNXIFI_BACKEND_VENDOR char[] required |
1150 | * ONNXIFI_BACKEND_VERSION char[] required |
1151 | * ONNXIFI_BACKEND_EXTENSIONS char[] required |
1152 | * ONNXIFI_BACKEND_DEVICE char[] required |
1153 | * ONNXIFI_BACKEND_DEVICE_TYPE onnxEnum required |
1154 | * ONNXIFI_BACKEND_ONNX_IR_VERSION char[] required |
1155 | * ONNXIFI_BACKEND_OPSET_VERSION char[] required |
1156 | * ONNXIFI_BACKEND_CAPABILITIES onnxBitfield required |
1157 | * ONNXIFI_BACKEND_INIT_PROPERTIES onnxBitfield required |
1158 | * ONNXIFI_BACKEND_MEMORY_TYPES onnxBitfield required |
1159 | * ONNXIFI_BACKEND_GRAPH_INIT_PROPERTIES onnxBitfield required |
1160 | * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES onnxBitfield required |
1161 | * ONNXIFI_BACKEND_MEMORY_SIZE uint64_t required |
1162 | * ONNXIFI_BACKEND_MAX_GRAPH_SIZE uint64_t required |
1163 | * ONNXIFI_BACKEND_MAX_GRAPH_COUNT uint64_t required |
1164 | * ONNXIFI_BACKEND_MACS_FP32 uint64_t optional |
1165 | * ONNXIFI_BACKEND_MACS_FP16 uint64_t optional |
1166 | * ONNXIFI_BACKEND_MEMORY_BANDWIDTH uint64_t optional |
1167 | * ONNXIFI_BACKEND_CPU_MEMORY_READ_BANDWIDTH uint64_t optional |
1168 | * ONNXIFI_BACKEND_CPU_MEMORY_WRITE_BANDWIDTH uint64_t optional |
1169 | * ONNXIFI_BACKEND_PCI_BUS_ID uint64_t optional |
1170 | * ONNXIFI_BACKEND_PCI_DEVICE_ID uint64_t optional |
1171 | * ONNXIFI_BACKEND_PCI_DOMAIN_ID uint64_t optional |
1172 | * ONNXIFI_BACKEND_DIRECTX_ID LUID optional |
1173 | * ONNXIFI_BACKEND_CUDA_INDEX uint64_t optional |
1174 | * ONNXIFI_BACKEND_OPENCL_PLATFORM_ID cl_platform_id optional |
1175 | * ONNXIFI_BACKEND_OPENCL_DEVICE_ID cl_device_id optional |
1176 | * |
1177 | * @param backendID - ID of the backend to query. |
1178 | * @param infoType - type of the backend information to query. Must be one of |
1179 | * the ONNXIFI_BACKEND_* constants. If this value is not |
1180 | * supported by the backend, the function will fail with |
1181 | * ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE. |
1182 | * @param infoValue[out] - pointer to the memory location where the backend |
1183 | * information value will be returned. If the pointer is |
1184 | * NULL, is it ignored. |
1185 | * @param infoValueSize[in,out] - pointer to a variable specifying size, in |
1186 | * bytes, of the information value. On function |
1187 | * entry, the variable MUST contain the size of |
1188 | * the memory buffer specified by infoValue. |
1189 | * For successful completion, this size must be |
1190 | * at least as large as the queried value. If the |
1191 | * function completes with either |
1192 | * ONNXIFI_STATUS_SUCCESS or |
1193 | * ONNXIFI_STATUS_FALLBACK status codes, the |
1194 | * actual size of the value queried in the call |
1195 | * is stored in the variable specified by this |
1196 | * pointer. |
1197 | * |
1198 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded, and requested |
1199 | * value is stored in the location specified by |
1200 | * infoValue, and the actual size of the |
1201 | * requested value is stored in the location |
1202 | * specified by infoValueSize. |
1203 | * @retval ONNXIFI_STATUS_FALLBACK The function call completed, but the |
1204 | * requested value was not stored in the |
1205 | * location specified by infoValue, either |
1206 | * because it is NULL, or because the size of |
1207 | * the memory buffer is insufficient for the |
1208 | * value. The actual size of the requested value |
1209 | * is stored in the location specified by |
1210 | * infoValueSize. |
1211 | * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID |
1212 | * is not an ONNXIFI backend ID. |
1213 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1214 | * infoValueSize is NULL. |
1215 | * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because |
1216 | * the value of infoType is not |
1217 | * supported by the backend. |
1218 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1219 | * the backend was disconnected or |
1220 | * uninstalled from the system. |
1221 | */ |
1222 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1223 | onnxGetBackendInfo( |
1224 | onnxBackendID backendID, |
1225 | onnxBackendInfo infoType, |
1226 | void* infoValue, |
1227 | size_t* infoValueSize); |
1228 | |
1229 | /** |
1230 | * Query if an ONNX model graph is compatible with the backend. |
1231 | * |
1232 | * Model graph is passed as a serialized ModelProto message, where types and |
1233 | * dimensions of all inputs (including static weights) and outputs are specified |
1234 | * through ModelProto.graph.input and ModelProto.graph.output messages. If the |
1235 | * backend supports ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS, some of the shape |
1236 | * dimensions can be symbolic. If the backend supports |
1237 | * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE, the outer shape dimension can be |
1238 | * symbolic. In these cases, the validation of symbolic dimension should be |
1239 | * deferred until graph inputs and outputs are specified in onnxSetGraphIO. |
1240 | * |
1241 | * Commonly, the serialized ModelProto message passed to this function would |
1242 | * not include the static weights (ModelProto.graph.initializer is empty), and |
1243 | * the backend implementation MUST NOT rely on the weights to determine if the |
1244 | * graph is supported. |
1245 | * |
1246 | * An important use-case is a ModelProto containing only a single NodeProto in |
1247 | * ModelProto.graph.node, which happens when a high-level framework checks |
1248 | * operators one-by-one to find a connected subgraph that can be offloaded to |
1249 | * the backend. Backend implementations SHOULD optimize performance for this |
1250 | * use-case. |
1251 | * |
1252 | * @param backend - ID of the backend to query. |
1253 | * @param onnxModelSize - size of the serialized ONNX ModelProto message, |
1254 | * in bytes. |
1255 | * @param[in] onnxModel - pointer to serialized ONNX ModelProto message |
1256 | * representing the model graph. |
1257 | * |
1258 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the model |
1259 | * graph can efficiently run on the backend. |
1260 | * @retval ONNXIFI_STATUS_FALLBACK The function call succeeded and the model |
1261 | * graph can run on the backend through some |
1262 | * emulation layer with some efficiency loss. If |
1263 | * a backend decomposes this operator into |
1264 | * multiple sub-operators, it should return this |
1265 | * code. E.g. if a backend does not natively |
1266 | * support grouped or depthwise convolution, but |
1267 | * can execute it as multiple unit-group |
1268 | * convolution operators, it must returns this |
1269 | * code. |
1270 | * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID |
1271 | * is not an ONNXIFI backend ID. |
1272 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1273 | * onnxModel is NULL. |
1274 | * @retval ONNXIFI_STATUS_INVALID_SIZE The function call failed because |
1275 | * onnxModelSize is 0. |
1276 | * @retval ONNXIFI_STATUS_INVALID_PROTOBUF The function call failed because it |
1277 | * couldn't parse the serialized |
1278 | * protobuf as an ONNX ModelProto |
1279 | * message. |
1280 | * @retval ONNXIFI_STATUS_INVALID_MODEL The function call failed because the |
1281 | * parsed ModelProto message does not |
1282 | * satisfy ONNX requirements and |
1283 | * constraints. |
1284 | * @retval ONNXIFI_STATUS_UNSUPPORTED_VERSION The function call failed because |
1285 | * the ONNX IR version or operator |
1286 | * version is not supported by the |
1287 | * backend. |
1288 | * @retval ONNXIFI_STATUS_UNSUPPORTED_OPERATOR The function call failed because |
1289 | * one of the operators in the model |
1290 | * graph is not supported by the |
1291 | * backend. |
1292 | * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because |
1293 | * the backend does not support the |
1294 | * particular AttributeProto |
1295 | * values in one of the operators. |
1296 | * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the |
1297 | * backend does not support the |
1298 | * tensor shapes in an input or output |
1299 | * of one of the operators. The |
1300 | * problematic tensor shapes could be |
1301 | * directly specified through |
1302 | * ValueInfoProto in GraphProto.input, |
1303 | * GraphProto.output, or |
1304 | * GraphProto.value_info, through |
1305 | * TensorProto in |
1306 | * GraphProto.initializer, or inferred |
1307 | * from the inputs by the backend. |
1308 | * @retval ONNXIFI_STATUS_UNSUPPORTED_DATATYPE The function call failed because |
1309 | * the backend does not support the |
1310 | * data types in an input or output |
1311 | * of one of the operators. The |
1312 | * problematic data types could be |
1313 | * directly specified through |
1314 | * ValueInfoProto in |
1315 | * GraphProto.input, |
1316 | * GraphProto.output, or |
1317 | * GraphProto.value_info, through |
1318 | * TensorProto in |
1319 | * GraphProto.initializer, or |
1320 | * inferred from the inputs by the |
1321 | * backend. |
1322 | * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because |
1323 | * output or intermediate shapes |
1324 | * specified in the ONNX model graph do |
1325 | * not match the shapes inferred from |
1326 | * input shapes. |
1327 | * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because |
1328 | * output or intermediate data types |
1329 | * specified in the ONNX model graph |
1330 | * do not match the data types |
1331 | * inferred from graph inputs. |
1332 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
1333 | * backend could not allocate enough |
1334 | * system memory to parse and analyze |
1335 | * the model graph. |
1336 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1337 | * the backend was disconnected or |
1338 | * uninstalled from the system. |
1339 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1340 | * backend experienced an unrecovered |
1341 | * internal error. |
1342 | */ |
1343 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1344 | onnxGetBackendCompatibility( |
1345 | onnxBackendID backendID, |
1346 | size_t onnxModelSize, |
1347 | const void* onnxModel); |
1348 | |
1349 | /** |
1350 | * Initialize an ONNXIFI backend. |
1351 | * |
1352 | * ONNXIFI backend is a combination of software layer and hardware device used |
1353 | * to run an ONNXIFI graph. The same software layer may expose multiple backends |
1354 | * (e.g. one ONNXIFI backend for each GPU in the system, or one ONNXIFI backend |
1355 | * for GPU and another for CPU, both implemented in the same software). |
1356 | * |
1357 | * @param backendID - ID of the backend to initialize. |
1358 | * @param[in] auxPropertiesList - optional list of backend initialization |
1359 | * properties, terminated by |
1360 | * ONNXIFI_BACKEND_PROPERTY_NONE entry. Can be |
1361 | * NULL or empty. |
1362 | * @param[out] backend - pointer to an opaque handle for the initialized ONNXIFI |
1363 | * backend. If the function fails, the handle is |
1364 | * initialized to NULL. |
1365 | * |
1366 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the backend |
1367 | * was successfully initialized. |
1368 | * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID |
1369 | * is not an ONNXIFI backend ID. |
1370 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1371 | * backend pointer is NULL. |
1372 | * @retval ONNXIFI_STATUS_INVALID_PROPERTY The function call failed because one |
1373 | * of the backend initialization |
1374 | * property values is invalid. |
1375 | * @retval ONNXIFI_STATUS_UNSUPPORTED_PROPERTY The function call failed because |
1376 | * backend does not recognize one |
1377 | * of the initialization |
1378 | * property IDs. |
1379 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed due to |
1380 | * insufficient system memory to |
1381 | * initialize backend. |
1382 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
1383 | * insufficient non-memory system |
1384 | * resources (e.g. file handles) to |
1385 | * initialize the backend. |
1386 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
1387 | * insufficient backend-specific memory |
1388 | * to initialize the backend. |
1389 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
1390 | * insufficient non-memory |
1391 | * backend-specific resources (e.g. |
1392 | * command queues) to initialize the |
1393 | * backend. |
1394 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1395 | * the backend was disconnected or |
1396 | * uninstalled from the system. |
1397 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1398 | * backend experienced an unrecovered |
1399 | * internal error. |
1400 | */ |
1401 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1402 | onnxInitBackend( |
1403 | onnxBackendID backendID, |
1404 | const uint64_t* auxPropertiesList, |
1405 | onnxBackend* backend); |
1406 | |
1407 | /** |
1408 | * Deinitialize an ONNXIFI backend and release associated resources. |
1409 | * |
1410 | * The user MUST deinitialize all objects created on this backend (onnxGraph, |
1411 | * onnxEvent) before calling this function to deinitialize the backend. |
1412 | * |
1413 | * @param backend - ONNXIFI backend handle created by onnxInitBackend. |
1414 | * |
1415 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the backend |
1416 | * resources were released to the operating |
1417 | * system. |
1418 | * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because |
1419 | * backend is not an ONNXIFI backend |
1420 | * handle. |
1421 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1422 | * backend experienced an unrecovered |
1423 | * internal error. |
1424 | */ |
1425 | ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI |
1426 | onnxReleaseBackend( |
1427 | onnxBackend backend); |
1428 | |
1429 | /** |
1430 | * Initialize a single-shot ONNXIFI event. |
1431 | * |
1432 | * The newly created event is in non-signalled state. |
1433 | * |
1434 | * @param backend - backend handle created by onnxInitBackend. This backend |
1435 | * would be used to initialize the event. |
1436 | * @param[out] event - pointer to the opaque handle for the created ONNXIFI |
1437 | * event. If the function fails, the handle is initialized |
1438 | * to NULL. |
1439 | * |
1440 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event |
1441 | * was successfully initialized. |
1442 | * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because |
1443 | * backend is not an ONNXIFI backend |
1444 | * handle. |
1445 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1446 | * event pointer is NULL. |
1447 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed due to |
1448 | * insufficient system memory to |
1449 | * initialize the event. |
1450 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
1451 | * insufficient non-memory system |
1452 | * resources (e.g. file handles) to |
1453 | * initialize the event. |
1454 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
1455 | * insufficient backend-specific memory |
1456 | * to initialize the event. |
1457 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
1458 | * insufficient non-memory |
1459 | * backend-specific resources (e.g. |
1460 | * command queues) to initialize the |
1461 | * event. |
1462 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1463 | * the backend was disconnected or |
1464 | * uninstalled from the system. |
1465 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1466 | * backend experienced an unrecovered |
1467 | * internal error. |
1468 | */ |
1469 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1470 | onnxInitEvent( |
1471 | onnxBackend backend, |
1472 | onnxEvent* event); |
1473 | |
1474 | /** |
1475 | * Change the state of an ONNXIFI event to signalled. |
1476 | * |
1477 | * @param event - event handle created by onnxInitEvent. While it is technically |
1478 | * possible to use this function for output memory fence event |
1479 | * created by onnxRunGraph, users SHOULD NOT do that. |
1480 | * |
1481 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event |
1482 | * was changed to signalled state. |
1483 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event |
1484 | * is not an ONNXIFI event handle. |
1485 | * @retval ONNXIFI_STATUS_INVALID_STATE The function call failed because event |
1486 | * is already in the signalled state. |
1487 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1488 | * the backend was disconnected or |
1489 | * uninstalled from the system. |
1490 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1491 | * implementation experienced an |
1492 | * unrecovered internal error. |
1493 | */ |
1494 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1495 | onnxSignalEvent( |
1496 | onnxEvent event); |
1497 | |
1498 | /** |
1499 | * Query ONNXIFI event state without blocking. |
1500 | * |
1501 | * @param event - event handle created by onnxRunGraph. While it is technically |
1502 | * possible to use this function to events created by |
1503 | * onnxInitEvent, this is not the intended use-case. |
1504 | * @param[out] state - pointer to the variable that will store the state of the |
1505 | * event. If the function fails, the variable is initialized |
1506 | * to ONNXIFI_EVENT_STATE_INVALID. |
1507 | * |
1508 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the state |
1509 | * variable was initialized to either |
1510 | * ONNXIFI_EVENT_STATE_SIGNALLED or |
1511 | * ONNXIFI_EVENT_STATE_NONSIGNALLED according |
1512 | * to the state of the event. |
1513 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event |
1514 | * is not an ONNXIFI event handle. |
1515 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because state |
1516 | * pointer is NULL. |
1517 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1518 | * the backend was disconnected or |
1519 | * uninstalled from the system. |
1520 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1521 | * implementation experienced an |
1522 | * unrecovered internal error. |
1523 | */ |
1524 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1525 | onnxGetEventState( |
1526 | onnxEvent event, |
1527 | onnxEventState* state); |
1528 | |
1529 | /** |
1530 | * Wait until an ONNXIFI event transitions to signalled state. |
1531 | * |
1532 | * @param event - event handle created by onnxRunGraph. While it is technically |
1533 | * possible to use this function to events created by |
1534 | * onnxInitEvent, this is not the intended use-case. |
1535 | * |
1536 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function |
1537 | * returned because event transitioned to |
1538 | * signalled state. |
1539 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event |
1540 | * is not an ONNXIFI event handle. |
1541 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1542 | * the backend was disconnected or |
1543 | * uninstalled from the system. |
1544 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1545 | * implementation experienced an |
1546 | * unrecovered internal error. |
1547 | */ |
1548 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1549 | onnxWaitEvent( |
1550 | onnxEvent event); |
1551 | |
1552 | /** |
1553 | * Deinitialize an ONNXIFI event and release associated resources. |
1554 | * |
1555 | * @param event - event handle created by either onnxInitEvent or onnxRunGraph. |
1556 | * |
1557 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the event |
1558 | * resources were released to the operating |
1559 | * system. |
1560 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event |
1561 | * is not an ONNXIFI event handle. |
1562 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1563 | * implementation experienced an |
1564 | * unrecovered internal error. |
1565 | */ |
1566 | ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI |
1567 | onnxReleaseEvent( |
1568 | onnxEvent event); |
1569 | |
1570 | /** |
1571 | * Parse an ONNXIFI graph and convert it for a particular backend. |
1572 | * |
1573 | * Model graph is passed as a serialized ModelProto message, where types and |
1574 | * dimensions of all inputs (including static weights) and outputs are specified |
1575 | * through ModelProto.graph.input and ModelProto.graph.output messages. If the |
1576 | * backend supports ONNXIFI_CAPABILITY_SYMBOLIC_SIZE_TENSORS, some of the shape |
1577 | * dimensions can be symbolic. If the backend supports |
1578 | * ONNXIFI_CAPABILITY_SYMBOLIC_BATCH_SIZE, the outer shape dimension can be |
1579 | * symbolic. In these cases, their validation should be deferred until a later |
1580 | * call to onnxSetGraphIO. |
1581 | * |
1582 | * Values of all static weights of the graph must be specified either in |
1583 | * ModelProto.graph.initializer, or through the weightDescriptors parameters, |
1584 | * but not through any combination of the two methods. If the caller creates the |
1585 | * graph on the fly, it SHOULD pass weights through weightDescriptors as it |
1586 | * involves less overhead. |
1587 | * |
1588 | * Blobs and operators in this graph are independent of the blobs and operators |
1589 | * of other graphs on the same backend. |
1590 | * |
1591 | * @param backend - backend handle created by onnxInitBackend. This backend |
1592 | * would be used to setup and run the model graph. |
1593 | * @param[in] auxPropertiesList - optional list of graph initialization |
1594 | * properties, terminated by |
1595 | * ONNXIFI_GRAPH_PROPERTY_NONE entry. Can be |
1596 | * NULL or empty. |
1597 | * @param onnxModelSize - size of the serialized ONNX ModelProto message, |
1598 | * in bytes. |
1599 | * @param[in] onnxModel - pointer to serialized ONNX ModelProto message |
1600 | * representing the model graph. The backend MUST not |
1601 | * assume that the serialized ModelProto message is |
1602 | * present at this address after the function returns. |
1603 | * @param weightsCount - number of weights specified in this function call |
1604 | * through tensor descriptors. Alternatively, the weights |
1605 | * can be specified in ModelProto.graph.initializer. |
1606 | * If weightsCount is non-zero, weightDescriptors must be |
1607 | * non-NULL. |
1608 | * @param[in] weightDescriptors - descriptors of static input tensors for the |
1609 | * graph. Elements of this array provide location |
1610 | * for blobs identified by ValueInfoProto.name |
1611 | * listed in ModelProto.graph.input of the ONNX |
1612 | * graph. If this parameter is non-NULL, |
1613 | * all static inputs must be specified through |
1614 | * the tensor descriptors, and the |
1615 | * ModelProto.graph.initilizer list must be |
1616 | * empty. The tensor descriptors |
1617 | * must use ONNXIFI_MEMORY_TYPE_CPU memory type, |
1618 | * and the backend must copy the values of the |
1619 | * tensors and all metadata, including shape, |
1620 | * into its own memory before the function |
1621 | * returns. |
1622 | * @param[out] graph - pointer to the opaque handle for the created ONNXIFI |
1623 | * graph. If the function fails, and this pointer is |
1624 | * non-NULL, the handle is initialized to NULL. |
1625 | * @param[inout] deferredWeightReader - pointer to an opaque handle to populate |
1626 | * the offline tensors during backend |
1627 | * compilation time. Default is NULL. |
1628 | * |
1629 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the model |
1630 | * graph was successfully initialized on the |
1631 | * backend. |
1632 | * @retval ONNXIFI_STATUS_FALLBACK The function call succeeded and the model |
1633 | * graph was initialized for the backend through |
1634 | * an emulation layer with substantial |
1635 | * efficiency loss. If a backend decomposes an |
1636 | * operator into multiple sub-operators, it |
1637 | * MUST return this code. E.g. if a backend |
1638 | * does not natively support grouped or |
1639 | * depthwise convolution, but can execute it as |
1640 | * multiple unit-group convolution operators, it |
1641 | * should return this code. |
1642 | * @retval ONNXIFI_STATUS_INVALID_BACKEND The function call failed because |
1643 | * backend is not an ONNXIFI backend |
1644 | * handle. |
1645 | * @retval ONNXIFI_STATUS_INVALID_PROPERTY The function call failed because one |
1646 | * of the graph initialization property |
1647 | * values is invalid. |
1648 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1649 | * onnxModel or graph pointer is NULL, or |
1650 | * weightDescriptors pointer is NULL |
1651 | * while weightsCount is non-zero. |
1652 | * @retval ONNXIFI_STATUS_INVALID_SIZE The function call failed because |
1653 | * onnxModelSize is 0. |
1654 | * @retval ONNXIFI_STATUS_INVALID_PROTOBUF The function call failed because it |
1655 | * couldn't parse the serialized |
1656 | * protobuf as an ONNX ModelProto |
1657 | * message. |
1658 | * @retval ONNXIFI_STATUS_INVALID_MODEL The function call failed because the |
1659 | * parsed ModelProto message does not |
1660 | * satisfy ONNX requirements and |
1661 | * constraints. |
1662 | * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of |
1663 | * the shape dimensions in |
1664 | * weightDescriptors is 0. |
1665 | * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because |
1666 | * one of the data types in |
1667 | * weightDescriptors is unknown to the |
1668 | * backend. |
1669 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because |
1670 | * one of the memory types in |
1671 | * weightDescriptors is unknown to |
1672 | * the backend. |
1673 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed |
1674 | * because one of the memory |
1675 | * locations in weightDescriptors |
1676 | * is invalid (NULL pointer). |
1677 | * @retval ONNXIFI_STATUS_UNSUPPORTED_PROPERTY The function call failed because |
1678 | * backend does not recognize one |
1679 | * of the graph initialization |
1680 | * property IDs. |
1681 | * @retval ONNXIFI_STATUS_UNSUPPORTED_VERSION The function call failed because |
1682 | * the ONNX IR version or operator |
1683 | * version is not supported by the |
1684 | * backend. |
1685 | * @retval ONNXIFI_STATUS_UNSUPPORTED_OPERATOR The function call failed because |
1686 | * one of the operators in the model |
1687 | * graph is not supported by the |
1688 | * backend. |
1689 | * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed because |
1690 | * the backend does not support the |
1691 | * particular AttributeProto |
1692 | * values in one of the operators. |
1693 | * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the |
1694 | * backend does not support the |
1695 | * tensor shapes in an input or |
1696 | * output of one of the operators. |
1697 | * The problematic tensor shapes could |
1698 | * be directly specified through |
1699 | * ValueInfoProto in GraphProto.input, |
1700 | * GraphProto.output, or |
1701 | & GraphProto.value_info, through |
1702 | * TensorProto in |
1703 | * GraphProto.initializer, through |
1704 | * weightDescriptors argument, |
1705 | * or inferred from the inputs by the |
1706 | * backend. |
1707 | * @retval ONNXIFI_STATUS_UNSUPPORTED_DATATYPE The function call failed because |
1708 | * the backend does not support the |
1709 | * data types in an input or output |
1710 | * of one of the operators. The |
1711 | * problematic data types could be |
1712 | * directly specified through |
1713 | * ValueInfoProto in |
1714 | * GraphProto.input, |
1715 | * GraphProto.output, or |
1716 | * GraphProto.value_info, through |
1717 | * TensorProto in |
1718 | * GraphProto.initializer, through |
1719 | * weightDescriptors argument, |
1720 | * or inferred from the inputs by |
1721 | * the backend. |
1722 | * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed |
1723 | * because one of the memory |
1724 | * types in weightDescriptors is |
1725 | * different from |
1726 | * ONNXIFI_MEMORY_TYPE_CPU. |
1727 | * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because |
1728 | * the shapes specified in weight |
1729 | * descriptors do not match the shapes |
1730 | * specified in the ONNX model graph, |
1731 | * or output or intermediate shapes |
1732 | * specified in the ONNX model graph do |
1733 | * not match the shapes inferred from |
1734 | * input shapes. |
1735 | * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because |
1736 | * data types specified in weight |
1737 | * descriptors do not match the data |
1738 | * types specified in ONNX model |
1739 | * graph, or output or intermediate |
1740 | * data types specified in the ONNX |
1741 | * model graph do not match the data |
1742 | * types inferred from graph inputs. |
1743 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
1744 | * backend could not allocate enough |
1745 | * system memory to parse, analyze, and |
1746 | * initialize the model graph. |
1747 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
1748 | * insufficient non-memory system |
1749 | * resources (e.g. file handles) to |
1750 | * initialize the graph. |
1751 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
1752 | * insufficient backend-specific memory |
1753 | * to initialize the graph. |
1754 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
1755 | * insufficient non-memory |
1756 | * backend-specific resources (e.g. |
1757 | * command queues) to initialize the |
1758 | * graph. |
1759 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1760 | * the backend was disconnected or |
1761 | * uninstalled from the system. |
1762 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1763 | * implementation experienced an |
1764 | * unrecovered internal error. |
1765 | */ |
1766 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1767 | onnxInitGraph( |
1768 | onnxBackend backend, |
1769 | const uint64_t* auxPropertiesList, |
1770 | size_t onnxModelSize, |
1771 | const void* onnxModel, |
1772 | uint32_t weightsCount, |
1773 | const onnxTensorDescriptorV1* weightDescriptors, |
1774 | onnxGraph* graph, |
1775 | uint32_t maxSeqLength, |
1776 | void* deferredWeightReader); |
1777 | |
1778 | /** |
1779 | * Set locations for inputs and outputs of an ONNXIFI graph. |
1780 | * |
1781 | * The caller MUST ensure that the memory buffers specified for input and output |
1782 | * tensors remain accessible until all in-flight graph executions which use |
1783 | * specified buffer locations complete AND |
1784 | * - Either a next call to onnxSetGraphIO specifies different buffer locations |
1785 | * - Or the graph is deinitialized via onnxReleaseGraph |
1786 | * The caller can invalidate other data in tensor descriptors, including shape, |
1787 | * once the function returns. |
1788 | * |
1789 | * Calls to onnxRunGraph WILL use input and output locations specified in the |
1790 | * preceeding onnxSetGraphIO on the same graph. Asynchronous graph executions |
1791 | * that were in-flight before onnxSetGraphIO call will continue to use buffer |
1792 | * locations that were current when these graph executions started. An ONNXIFI |
1793 | * implementation MAY block inside onnxSetGraphIO until all in-flight graph |
1794 | * executions that started before the call complete. |
1795 | * |
1796 | * If a call to onnxSetGraphIO fails, it invalidates input and output locations |
1797 | * for the graph, and a subsequent call to onnxRunGraph will fail with |
1798 | * ONNXIFI_STATUS_UNIDENTIFIED_NAME. |
1799 | * |
1800 | * @param graph - graph handle created by onnxInitGraph. |
1801 | * @param inputsCount - number of elements in the inputDescriptors array. |
1802 | * @param[in] inputDescriptors - descriptors of input tensors for the graph. |
1803 | * Elements of this array must provide a location |
1804 | * for each ValueInfoProto.name listed in |
1805 | * ModelProto.graph.input of the ONNX graph. |
1806 | * If inputsCount is non-zero, inputDescriptors |
1807 | * pointer must be non-NULL. |
1808 | * @param outputsCount - number of elements in the outputDescriptors array. |
1809 | * Must be greater than zero. |
1810 | * @param[in] outputDescriptors - descriptors of output tensors for the graph. |
1811 | * outputDescriptors pointer must be non-NULL. |
1812 | * Elements of this array must provide a location |
1813 | * for each ValueInfoProto.name listed in |
1814 | * ModelProto.graph.output of the ONNX graph. |
1815 | * |
1816 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph |
1817 | * inputs and outputs were matched to a memory |
1818 | * location. |
1819 | * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because |
1820 | * graph is not an ONNXIFI graph handle. |
1821 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1822 | * outputDescriptors pointer is NULL or |
1823 | * inputDescriptors pointer is NULL while |
1824 | * inputsCount is non-zero. |
1825 | * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because one of |
1826 | * the names in tensor descriptors doesn't |
1827 | * match blob name in ModelProto.graph.input |
1828 | * or ModelProto.graph.output, or the same |
1829 | * name appears in more than one tensor |
1830 | * descriptor. |
1831 | * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of |
1832 | * the shape dimensions is 0. |
1833 | * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because |
1834 | * one of the data types in |
1835 | * inputDescriptors or outputDescriptors |
1836 | * is unknown to the backend. |
1837 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because |
1838 | * one of the memory types in |
1839 | * inputDescriptors or |
1840 | * outputDescriptors is unknown to |
1841 | * the backend. |
1842 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed |
1843 | * because one of the memory |
1844 | * locations in inputDescriptors |
1845 | * or outputDescriptors is not |
1846 | * valid for the specified |
1847 | * memory type (e.g. NULL pointer |
1848 | * for ONNXIFI_MEMORY_TYPE_CPU). |
1849 | * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because one |
1850 | * of the tags in inputDescriptors or |
1851 | * outputDescriptors is unknown to the |
1852 | * backend (tag does not match |
1853 | * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1). |
1854 | * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the |
1855 | * backend does not support the |
1856 | * tensor shapes in an input or output |
1857 | * of one of the operators. The |
1858 | * problematic tensor shapes could be |
1859 | * directly specified through |
1860 | * inputDescriptors or |
1861 | * outputDescriptors argument, |
1862 | * or inferred from the inputs by the |
1863 | * backend. This error code can be |
1864 | * returned when the backend supports |
1865 | * variable-size inputs and outputs, |
1866 | * and the problematic tensor shape was |
1867 | * provided in the ValueInfoProto as a |
1868 | * symbolic variable. |
1869 | * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed |
1870 | * because the backend does not |
1871 | * support one of the memory |
1872 | * types in inputDescriptors or |
1873 | * outputDescriptors. |
1874 | * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because one |
1875 | * of the ValueInfoProto.name value in |
1876 | * ModelProto.graph.input or |
1877 | * ModelProto.graph.output doesn't have |
1878 | * a match in the inputDescriptors or |
1879 | * outputDescriptors. |
1880 | * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because |
1881 | * the shapes specified through |
1882 | * inputDescriptors or |
1883 | * outputDescriptors argument are |
1884 | * inconsistent with the shapes |
1885 | * specified in the ONNX model graph. |
1886 | * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because |
1887 | * data types specified through |
1888 | * inputDescriptors or |
1889 | * outputDescriptors argument are |
1890 | * inconsistent with the data types |
1891 | * specified in the ONNX model |
1892 | * graph. |
1893 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
1894 | * backend could not allocate enough |
1895 | * system memory to parse, analyze, and |
1896 | * initialize the tensor locations. |
1897 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
1898 | * insufficient non-memory system |
1899 | * resources (e.g. file handles) to |
1900 | * initialize the tensor locations. |
1901 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
1902 | * insufficient backend-specific memory |
1903 | * to initialize the tensor locations. |
1904 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
1905 | * insufficient non-memory |
1906 | * backend-specific resources (e.g. |
1907 | * command queues) to initialize the |
1908 | * tensor locations. |
1909 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
1910 | * the backend was disconnected or |
1911 | * uninstalled from the system. |
1912 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
1913 | * backend experienced an unrecovered |
1914 | * internal error. |
1915 | */ |
1916 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
1917 | onnxSetGraphIO( |
1918 | onnxGraph graph, |
1919 | uint32_t inputsCount, |
1920 | const onnxTensorDescriptorV1* inputDescriptors, |
1921 | uint32_t outputsCount, |
1922 | const onnxTensorDescriptorV1* outputDescriptors); |
1923 | |
1924 | /** |
1925 | * Asynchronously execute operations in an ONNXIFI graph using pre-specified |
1926 | * locations for inputs and outputs. |
1927 | * |
1928 | * This function operates asynchronously: it doesn't require that the locations |
1929 | * for graph inputs graph inputs hold valid values before the function is |
1930 | * called, and doesn't guarantee that the locations for graph outputs hold |
1931 | * valid values when the function returns. Instead, two synchronization |
1932 | * primitives are used to signal to the backend when inputs are ready to use, |
1933 | * and to signal to the caller when outputs are ready to use. The only |
1934 | * synchronization primitive that is always available is onnxEvent |
1935 | * (ONNXIFI_SYNCHRONIZATION_EVENT memory fence type). If a backend supports |
1936 | * additional types of synchronization primitives, it must indicate them in |
1937 | * ONNXIFI_BACKEND_SYNCHRONIZATION_TYPES information query. |
1938 | * |
1939 | * The caller must successfully specify locations of input and output tensors |
1940 | * for the graph through onnxSetGraphIO before calling this function. |
1941 | * |
1942 | * @param graph - graph handle created by onnxInitGraph. |
1943 | * @param[in] inputFence - synchronization primitive that signals when graph |
1944 | * inputs are ready to use by the backend. The |
1945 | * synchronization primitive always must be initialized |
1946 | * by the caller. |
1947 | * @param[out] outputFence - synchronization primitive that signals when graph |
1948 | * outputs are ready to use by the caller. The type |
1949 | * of the synchronization primitive always must be |
1950 | * initialized by the caller. The type of the |
1951 | * synchronization primitive determines whether it |
1952 | * is initialized by the user before the call or by |
1953 | * the backend as a result of this call. Single-shot |
1954 | * synchronizatiom objects are initialized as a result |
1955 | * of the call. Reusable synchronization objects are |
1956 | * generally initialized by the user prior to the |
1957 | * call. |
1958 | * |
1959 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph |
1960 | * inputs and outputs were matched to a memory |
1961 | * location. |
1962 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
1963 | * inputFence or outputFence pointer is |
1964 | * NULL. |
1965 | * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because |
1966 | * graph is not an ONNXIFI graph handle. |
1967 | * @retval ONNXIFI_STATUS_INVALID_FENCE_TYPE The function call failed because |
1968 | * the type of synchronization |
1969 | * primitive specified in inputFence |
1970 | * or outputFence is unknown to the |
1971 | * backend. |
1972 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because |
1973 | * the memory synchronization primitive |
1974 | * specified in inputFence or outputFence |
1975 | * is not valid (e.g. NULL onnxEvent). |
1976 | * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because a tag |
1977 | * in inputFence or outputFence is |
1978 | * unknown to the backend (tag does not |
1979 | * match ONNXIFI_TAG_MEMORY_FENCE_V1). |
1980 | * @retval ONNXIFI_STATUS_UNSUPPORTED_FENCE_TYPE The function call failed |
1981 | * because the backend does not |
1982 | * support the type of |
1983 | * synchronization primitive |
1984 | * specified in inputFence or |
1985 | * outputFence. |
1986 | * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because |
1987 | * some of the ValueInfoProto.name |
1988 | * value in ModelProto.graph.input or |
1989 | * ModelProto.graph.output were not |
1990 | * specified in a call to |
1991 | * onnxSetGraphIO. |
1992 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
1993 | * backend could not allocate enough |
1994 | * system memory to execute the model |
1995 | * graph. |
1996 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
1997 | * insufficient non-memory system |
1998 | * resources (e.g. file handles) to |
1999 | * execute the model graph. |
2000 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
2001 | * insufficient backend-specific memory |
2002 | * to execute the graph. |
2003 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
2004 | * insufficient non-memory |
2005 | * backend-specific resources (e.g. |
2006 | * command queues) to execute the |
2007 | * graph. |
2008 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
2009 | * the backend was disconnected or |
2010 | * uninstalled from the system. |
2011 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
2012 | * backend experienced an unrecovered |
2013 | * internal error. |
2014 | */ |
2015 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
2016 | onnxRunGraph( |
2017 | onnxGraph graph, |
2018 | const onnxMemoryFenceV1* inputFence, |
2019 | onnxMemoryFenceV1* outputFence); |
2020 | |
2021 | /** |
2022 | * Deinitialize an ONNXIFI graph and release associated resources. |
2023 | * |
2024 | * If there are in-flight asynchronous inference operations on this graph, |
2025 | * the function MUST block until all outstanding operations complete. |
2026 | * |
2027 | * @param graph - graph handle created by onnxInitGraph. |
2028 | * |
2029 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the graph |
2030 | * resources were released to the operating |
2031 | * system. |
2032 | * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because graph |
2033 | * is not an ONNXIFI graph handle. |
2034 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
2035 | * graph backend experienced an |
2036 | * unrecovered internal error. |
2037 | */ |
2038 | ONNXIFI_PUBLIC onnxStatus ONNXIFI_ABI |
2039 | onnxReleaseGraph( |
2040 | onnxGraph graph); |
2041 | |
2042 | #ifdef __cplusplus |
2043 | } /* extern "C" */ |
2044 | #endif |
2045 | |
2046 | #endif /* !defined(ONNXIFI_H) */ |
2047 | |