1 | #ifndef ONNXIFI_EXT_H |
2 | #define ONNXIFI_EXT_H 1 |
3 | |
4 | #include "foxi/onnxifi.h" |
5 | |
6 | #ifdef __cplusplus |
7 | extern "C" { |
8 | #endif |
9 | |
10 | /** |
11 | * Compile the graph with tracing events before and after all nodes enabled. |
12 | */ |
13 | #define ONNXIFI_GRAPH_PROPERTY_AUTO_INSTRUMENT_NODES 1 |
14 | |
15 | /** |
16 | * Size in characters of the name field of onnxTraceEvent. |
17 | */ |
18 | #define ONNXIFI_TRACE_EVENT_NAME_SIZE 32 |
19 | |
20 | typedef struct onnxTraceEvent { |
21 | /** |
22 | * Human readable name for the event, will be used to match up begin and end |
23 | * of an event duration. |
24 | */ |
25 | char eventName[ONNXIFI_TRACE_EVENT_NAME_SIZE + 1]; |
26 | |
27 | /** |
28 | * Type of the event, can be one of the following: |
29 | * 'B': Beginning of event |
30 | * 'E': End of event |
31 | * 'I': Instantaneous event (no duration). |
32 | * 'X': Complete event (start + duration). |
33 | */ |
34 | char eventType; |
35 | |
36 | /** |
37 | * Time of the event, in microsecond since epoch. |
38 | */ |
39 | uint64_t timestamp; |
40 | |
41 | /** |
42 | * Thread Id for this event. All events with the same tid will be grouped |
43 | * together in the trace. |
44 | */ |
45 | uint32_t tid; |
46 | |
47 | /** |
48 | * For complete events, the duration of the event, in microseconds. |
49 | */ |
50 | uint64_t duration; |
51 | } onnxTraceEvent; |
52 | |
53 | typedef struct onnxTraceEventList { |
54 | /** |
55 | * The number of events in traceEvents. |
56 | */ |
57 | uint64_t numEvents; |
58 | |
59 | /** |
60 | * A pointer to an array of pointers to onnxTraceEvents allocated by the onnx |
61 | * backend, the length of which is indicated by numEvents. |
62 | */ |
63 | onnxTraceEvent **traceEvents; |
64 | } onnxTraceEventList; |
65 | |
66 | /** |
67 | * Generic ONNXIFI extension function pointer. |
68 | * |
69 | * The caller should convert this generic function pointer to the function |
70 | * pointer specific for an extension function type. |
71 | */ |
72 | typedef onnxStatus (ONNXIFI_ABI* onnxExtensionFunctionPointer)(void); |
73 | |
74 | /* Function pointer declarations for dynamic loading */ |
75 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
76 | (ONNXIFI_ABI* onnxGetExtensionFunctionAddressFunction)( |
77 | onnxBackendID backendID, |
78 | const char* name, |
79 | onnxExtensionFunctionPointer* function); |
80 | |
81 | /** |
82 | * Query function pointer for an ONNXIFI extension function. |
83 | * |
84 | * The returned function pointer is specific to the provided backend ID, and |
85 | * MUST NOT be used with objects created for other backend IDs. |
86 | * |
87 | * This function is a part of onnx_extension_function extension. Backends which |
88 | * implement this function MUST list "onnx_extension_function" in the result of |
89 | * onnxGetBackendInfo with ONNXIFI_BACKEND_EXTENSIONS information type. |
90 | * |
91 | * @param backendID - ID of the backend to query for extension function. |
92 | * @param[in] name - name of the extension function to query. |
93 | * @param[out] function - pointer to a generic function pointer for an ONNXIFI |
94 | * extension function. If the function fails, the |
95 | * function pointer is initialized to NULL. The caller |
96 | * MUST cast this function pointer to the type specific |
97 | * for the extension function before use. |
98 | * |
99 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the extension |
100 | * function pointer is stored in the location |
101 | * specified by function argument. |
102 | * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID |
103 | * is not an ONNXIFI backend ID. |
104 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
105 | * name or function argument is NULL. |
106 | * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because |
107 | * the backend does not implement |
108 | * the function identified by the name. |
109 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
110 | * backend experienced an unrecovered |
111 | * internal error. |
112 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
113 | * the backend was disconnected or |
114 | * uninstalled from the system. |
115 | */ |
116 | |
117 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
118 | onnxGetExtensionFunctionAddress( |
119 | onnxBackendID backendID, |
120 | const char* name, |
121 | onnxExtensionFunctionPointer* function); |
122 | |
123 | /* Extension function pointer declarations for dynamic loading */ |
124 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
125 | (ONNXIFI_ABI* onnxSetIOAndRunGraphFunction)( |
126 | onnxGraph graph, |
127 | uint32_t inputsCount, |
128 | const onnxTensorDescriptorV1* inputDescriptors, |
129 | uint32_t outputsCount, |
130 | const onnxTensorDescriptorV1* outputDescriptors, |
131 | onnxMemoryFenceV1* outputFence, |
132 | onnxTraceEventList* traceEvents); |
133 | |
134 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
135 | (ONNXIFI_ABI* onnxReleaseTraceEventsFunction)( |
136 | onnxTraceEventList* traceEvents); |
137 | |
138 | typedef ONNXIFI_CHECK_RESULT onnxStatus |
139 | (ONNXIFI_ABI* onnxGetCurrentBatchSizeFunction)( |
140 | int64_t* currentBatchSize); |
141 | |
142 | /** |
143 | * A combination of onnxSetIO and onnxRunGraph, functionally equals to first run |
144 | * onnxSetIO(graph, inputsCount, inputDescriptors, outputsCount, |
145 | * outputDescriptors), then run onnxRunGraph(graph, inputFence, outputFence) |
146 | * with an internal inputFence. |
147 | * |
148 | * As two separate functions, it is difficult to do atomic evaluation. |
149 | * Therefore, we would like to unify this process and make it evaluable. |
150 | * |
151 | * @param graph - graph handle created by onnxInitGraph. |
152 | * @param inputsCount - number of elements in the inputDescriptors array. |
153 | * @param[in] inputDescriptors - descriptors of input tensors for the graph. |
154 | * Elements of this array must provide a location |
155 | * for each ValueInfoProto.name listed in |
156 | * ModelProto.graph.input of the ONNX graph. |
157 | * If inputsCount is non-zero, inputDescriptors |
158 | * pointer must be non-NULL. |
159 | * @param outputsCount - number of elements in the outputDescriptors array. |
160 | * Must be greater than zero. |
161 | * @param[in] outputDescriptors - descriptors of output tensors for the graph. |
162 | * outputDescriptors pointer must be non-NULL. |
163 | * Elements of this array must provide a location |
164 | * for each ValueInfoProto.name listed in |
165 | * ModelProto.graph.output of the ONNX graph. |
166 | * @param[out] outputFence - synchronization primitive that signals when graph |
167 | * outputs are ready to use by the caller. The type |
168 | * of the synchronization primitive always must be |
169 | * initialized by the caller. The type of the |
170 | * synchronization primitive determines whether it |
171 | * is initialized by the user before the call or by |
172 | * the backend as a result of this call. Single-shot |
173 | * synchronizatiom objects are initialized as a result |
174 | * of the call. Reusable synchronization objects are |
175 | * generally initialized by the user prior to the |
176 | * call. |
177 | * @param[out] traceEvents - optional pointer to onnxTraceEventList that can be |
178 | * NULL. If non-NULL then the backend is requested to |
179 | * populate the onnxTraceEventList with trace events |
180 | * describing the timeline of events that occurred |
181 | * while running the graph. |
182 | * |
183 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph |
184 | * inputs and outputs were matched to a memory |
185 | * location. |
186 | * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because |
187 | * graph is not an ONNXIFI graph handle. |
188 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
189 | * outputDescriptors pointer is NULL or |
190 | * inputDescriptors pointer is NULL while |
191 | * inputsCount is non-zero. |
192 | * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because one of |
193 | * the names in tensor descriptors doesn't |
194 | * match blob name in ModelProto.graph.input |
195 | * or ModelProto.graph.output, or the same |
196 | * name appears in more than one tensor |
197 | * descriptor. |
198 | * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of |
199 | * the shape dimensions is 0. |
200 | * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because |
201 | * one of the data types in |
202 | * inputDescriptors or outputDescriptors |
203 | * is unknown to the backend. |
204 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because |
205 | * one of the memory types in |
206 | * inputDescriptors or |
207 | * outputDescriptors is unknown to |
208 | * the backend. |
209 | * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed |
210 | * because one of the memory |
211 | * locations in inputDescriptors |
212 | * or outputDescriptors is not |
213 | * valid for the specified |
214 | * memory type (e.g. NULL pointer |
215 | * for ONNXIFI_MEMORY_TYPE_CPU). |
216 | * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because one |
217 | * of the tags in inputDescriptors or |
218 | * outputDescriptors is unknown to the |
219 | * backend (tag does not match |
220 | * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1). |
221 | * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the |
222 | * backend does not support the |
223 | * tensor shapes in an input or output |
224 | * of one of the operators. The |
225 | * problematic tensor shapes could be |
226 | * directly specified through |
227 | * inputDescriptors or |
228 | * outputDescriptors argument, |
229 | * or inferred from the inputs by the |
230 | * backend. This error code can be |
231 | * returned when the backend supports |
232 | * variable-size inputs and outputs, |
233 | * and the problematic tensor shape was |
234 | * provided in the ValueInfoProto as a |
235 | * symbolic variable. |
236 | * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed |
237 | * because the backend does not |
238 | * support one of the memory |
239 | * types in inputDescriptors or |
240 | * outputDescriptors. |
241 | * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because one |
242 | * of the ValueInfoProto.name value in |
243 | * ModelProto.graph.input or |
244 | * ModelProto.graph.output doesn't have |
245 | * a match in the inputDescriptors or |
246 | * outputDescriptors. |
247 | * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because |
248 | * the shapes specified through |
249 | * inputDescriptors or |
250 | * outputDescriptors argument are |
251 | * inconsistent with the shapes |
252 | * specified in the ONNX model graph. |
253 | * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because |
254 | * data types specified through |
255 | * inputDescriptors or |
256 | * outputDescriptors argument are |
257 | * inconsistent with the data types |
258 | * specified in the ONNX model |
259 | * graph. |
260 | * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the |
261 | * backend could not allocate enough |
262 | * system memory to parse, analyze, and |
263 | * initialize the tensor locations. |
264 | * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to |
265 | * insufficient non-memory system |
266 | * resources (e.g. file handles) to |
267 | * initialize the tensor locations. |
268 | * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to |
269 | * insufficient backend-specific memory |
270 | * to initialize the tensor locations. |
271 | * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to |
272 | * insufficient non-memory |
273 | * backend-specific resources (e.g. |
274 | * command queues) to initialize the |
275 | * tensor locations. |
276 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
277 | * the backend was disconnected or |
278 | * uninstalled from the system. |
279 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
280 | * backend experienced an unrecovered |
281 | * internal error. |
282 | */ |
283 | |
284 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxSetIOAndRunGraph( |
285 | onnxGraph graph, |
286 | uint32_t inputsCount, |
287 | const onnxTensorDescriptorV1* inputDescriptors, |
288 | uint32_t outputsCount, |
289 | const onnxTensorDescriptorV1* outputDescriptors, |
290 | onnxMemoryFenceV1* outputFence, |
291 | onnxTraceEventList* traceEvents); |
292 | |
293 | /** |
294 | * Release the onnxTraceEvents contained in traceEvents. |
295 | * |
296 | * @param traceEvents - a list of onnxTraceEvents to be released. |
297 | * |
298 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the |
299 | * onnxTraceEvents resources were released to the |
300 | * operating system. |
301 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because |
302 | * onnxTraceEventList pointer is NULL. |
303 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
304 | * backend experienced an unrecovered |
305 | * internal error. |
306 | */ |
307 | |
308 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI |
309 | onnxReleaseTraceEvents(onnxTraceEventList* traceEvents); |
310 | |
311 | /** |
312 | * Wait until an ONNXIFI event transitions to signalled state or for a specified |
313 | * number of milliseconds, whichever occurs first. Upon returning, if return |
314 | * value is ONNXIFI_STATUS_SUCCESS then eventState will be set to the state |
315 | * of the event. If this function returned due to timeout then eventState will |
316 | * be ONNXIFI_EVENT_STATE_NONSIGNALLED. If this function returns due to the |
317 | * event being signalled then eventState is ONNXIFI_EVENT_STATE_SIGNALLED |
318 | * and eventStatus will be set to a status representative of the event that is |
319 | * being waited on. |
320 | * |
321 | * @param event - event handle created by onnxRunGraph. While it is technically |
322 | * possible to use this function to events created by |
323 | * onnxInitEvent, this is not the intended use-case. |
324 | * |
325 | * @param timeoutMs - The number of milliseconds to wait on the event before |
326 | * returning. If timeoutMs is 0 then this function will block |
327 | * on the event without timing out similar to onnxWaitEvent. |
328 | * |
329 | * @param eventState - The state of the event upon returning. If a timeout |
330 | * occurred then this will be |
331 | * ONNXIFI_EVENT_STATE_NONSIGNALLED, otherwise if the |
332 | * function returns ONNXIFI_STATUS_SUCCESS and no timeout |
333 | * occurred this will be ONNXIFI_EVENT_STATE_SIGNALLED. |
334 | * |
335 | * @param eventStatus - A status that can be associated with the event when |
336 | * it is signalled. This is only guaranteed to be set if |
337 | * the eventState is ONNXIFI_EVENT_STATE_SIGNALLED. If |
338 | * the event was signalled by a method that doesn't support |
339 | * status signalling then eventStatus will be set to |
340 | * ONNXIFI_STATUS_SUCCESS as a default. |
341 | * |
342 | * @param message - A preallocated message buffer to be filled with error |
343 | * messages if any error has happened. Message is always null |
344 | * terminated. |
345 | * |
346 | * @param messageLength - At input, it carries the maximum size of the message |
347 | * buffer. At output, it carries the actual size of the |
348 | * message buffer. |
349 | * |
350 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function |
351 | * returned because event transitioned to |
352 | * signalled state or the timeout was hit. |
353 | * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event |
354 | * is not an ONNXIFI event handle. |
355 | * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because |
356 | * the backend was disconnected or |
357 | * uninstalled from the system. |
358 | * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the |
359 | * implementation experienced an |
360 | * unrecovered internal error. |
361 | */ |
362 | |
363 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxWaitEventFor( |
364 | onnxEvent event, |
365 | uint32_t timeoutMs, |
366 | onnxEventState* eventState, |
367 | onnxStatus* eventStatus, |
368 | char* message, |
369 | size_t* messageLength); |
370 | |
371 | /** |
372 | * Provide a option name, get its value in string form. |
373 | * @param optionName - Name of the option |
374 | * |
375 | * @param optionValue - String value of the option |
376 | * |
377 | * @param optionValueLength - At input, it carries the maximum size of the |
378 | * message buffer. At output, it arries the |
379 | * actual size of the message buffer. |
380 | * |
381 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function |
382 | * returned because event transitioned to |
383 | * signalled state or the timeout was hit. |
384 | * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because the |
385 | * backend doesn't recognize the option |
386 | * name. |
387 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because any |
388 | * of supplied pointers is invalid. |
389 | * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed |
390 | * because supplied value is not |
391 | * valid. |
392 | */ |
393 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxGetOption( |
394 | const char* optionName, |
395 | char* optionValue, |
396 | size_t* optionValueLength); |
397 | |
398 | /** |
399 | * Provide a option name, set its value in string form. |
400 | * @param optionName - Name of the option |
401 | * |
402 | * @param optionValue - String value of the option |
403 | * |
404 | * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function |
405 | * returned because event transitioned to |
406 | * signalled state or the timeout was hit. |
407 | * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because the |
408 | * backend doesn't recognize the option |
409 | * name. |
410 | * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because any |
411 | * of supplied pointers is invalid. |
412 | * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed |
413 | * because supplied value is not |
414 | * valid. |
415 | */ |
416 | ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxSetOption( |
417 | const char* optionName, |
418 | const char* optionValue); |
419 | |
420 | #ifdef __cplusplus |
421 | } /* extern "C" */ |
422 | #endif |
423 | |
424 | #endif /* !defined(ONNXIFI_EXT_H) */ |
425 | |