1#ifndef ONNXIFI_EXT_H
2#define ONNXIFI_EXT_H 1
3
4#include "foxi/onnxifi.h"
5
6#ifdef __cplusplus
7extern "C" {
8#endif
9
10/**
11 * Compile the graph with tracing events before and after all nodes enabled.
12 */
13#define ONNXIFI_GRAPH_PROPERTY_AUTO_INSTRUMENT_NODES 1
14
15/**
16 * Size in characters of the name field of onnxTraceEvent.
17 */
18#define ONNXIFI_TRACE_EVENT_NAME_SIZE 32
19
20typedef struct onnxTraceEvent {
21 /**
22 * Human readable name for the event, will be used to match up begin and end
23 * of an event duration.
24 */
25 char eventName[ONNXIFI_TRACE_EVENT_NAME_SIZE + 1];
26
27 /**
28 * Type of the event, can be one of the following:
29 * 'B': Beginning of event
30 * 'E': End of event
31 * 'I': Instantaneous event (no duration).
32 * 'X': Complete event (start + duration).
33 */
34 char eventType;
35
36 /**
37 * Time of the event, in microsecond since epoch.
38 */
39 uint64_t timestamp;
40
41 /**
42 * Thread Id for this event. All events with the same tid will be grouped
43 * together in the trace.
44 */
45 uint32_t tid;
46
47 /**
48 * For complete events, the duration of the event, in microseconds.
49 */
50 uint64_t duration;
51} onnxTraceEvent;
52
53typedef struct onnxTraceEventList {
54 /**
55 * The number of events in traceEvents.
56 */
57 uint64_t numEvents;
58
59 /**
60 * A pointer to an array of pointers to onnxTraceEvents allocated by the onnx
61 * backend, the length of which is indicated by numEvents.
62 */
63 onnxTraceEvent **traceEvents;
64} onnxTraceEventList;
65
66/**
67 * Generic ONNXIFI extension function pointer.
68 *
69 * The caller should convert this generic function pointer to the function
70 * pointer specific for an extension function type.
71 */
72typedef onnxStatus (ONNXIFI_ABI* onnxExtensionFunctionPointer)(void);
73
74/* Function pointer declarations for dynamic loading */
75typedef ONNXIFI_CHECK_RESULT onnxStatus
76 (ONNXIFI_ABI* onnxGetExtensionFunctionAddressFunction)(
77 onnxBackendID backendID,
78 const char* name,
79 onnxExtensionFunctionPointer* function);
80
81/**
82 * Query function pointer for an ONNXIFI extension function.
83 *
84 * The returned function pointer is specific to the provided backend ID, and
85 * MUST NOT be used with objects created for other backend IDs.
86 *
87 * This function is a part of onnx_extension_function extension. Backends which
88 * implement this function MUST list "onnx_extension_function" in the result of
89 * onnxGetBackendInfo with ONNXIFI_BACKEND_EXTENSIONS information type.
90 *
91 * @param backendID - ID of the backend to query for extension function.
92 * @param[in] name - name of the extension function to query.
93 * @param[out] function - pointer to a generic function pointer for an ONNXIFI
94 * extension function. If the function fails, the
95 * function pointer is initialized to NULL. The caller
96 * MUST cast this function pointer to the type specific
97 * for the extension function before use.
98 *
99 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the extension
100 * function pointer is stored in the location
101 * specified by function argument.
102 * @retval ONNXIFI_STATUS_INVALID_ID The function call failed because backendID
103 * is not an ONNXIFI backend ID.
104 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
105 * name or function argument is NULL.
106 * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because
107 * the backend does not implement
108 * the function identified by the name.
109 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
110 * backend experienced an unrecovered
111 * internal error.
112 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
113 * the backend was disconnected or
114 * uninstalled from the system.
115 */
116
117ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
118onnxGetExtensionFunctionAddress(
119 onnxBackendID backendID,
120 const char* name,
121 onnxExtensionFunctionPointer* function);
122
123/* Extension function pointer declarations for dynamic loading */
124typedef ONNXIFI_CHECK_RESULT onnxStatus
125 (ONNXIFI_ABI* onnxSetIOAndRunGraphFunction)(
126 onnxGraph graph,
127 uint32_t inputsCount,
128 const onnxTensorDescriptorV1* inputDescriptors,
129 uint32_t outputsCount,
130 const onnxTensorDescriptorV1* outputDescriptors,
131 onnxMemoryFenceV1* outputFence,
132 onnxTraceEventList* traceEvents);
133
134typedef ONNXIFI_CHECK_RESULT onnxStatus
135 (ONNXIFI_ABI* onnxReleaseTraceEventsFunction)(
136 onnxTraceEventList* traceEvents);
137
138typedef ONNXIFI_CHECK_RESULT onnxStatus
139 (ONNXIFI_ABI* onnxGetCurrentBatchSizeFunction)(
140 int64_t* currentBatchSize);
141
142/**
143 * A combination of onnxSetIO and onnxRunGraph, functionally equals to first run
144 * onnxSetIO(graph, inputsCount, inputDescriptors, outputsCount,
145 * outputDescriptors), then run onnxRunGraph(graph, inputFence, outputFence)
146 * with an internal inputFence.
147 *
148 * As two separate functions, it is difficult to do atomic evaluation.
149 * Therefore, we would like to unify this process and make it evaluable.
150 *
151 * @param graph - graph handle created by onnxInitGraph.
152 * @param inputsCount - number of elements in the inputDescriptors array.
153 * @param[in] inputDescriptors - descriptors of input tensors for the graph.
154 * Elements of this array must provide a location
155 * for each ValueInfoProto.name listed in
156 * ModelProto.graph.input of the ONNX graph.
157 * If inputsCount is non-zero, inputDescriptors
158 * pointer must be non-NULL.
159 * @param outputsCount - number of elements in the outputDescriptors array.
160 * Must be greater than zero.
161 * @param[in] outputDescriptors - descriptors of output tensors for the graph.
162 * outputDescriptors pointer must be non-NULL.
163 * Elements of this array must provide a location
164 * for each ValueInfoProto.name listed in
165 * ModelProto.graph.output of the ONNX graph.
166 * @param[out] outputFence - synchronization primitive that signals when graph
167 * outputs are ready to use by the caller. The type
168 * of the synchronization primitive always must be
169 * initialized by the caller. The type of the
170 * synchronization primitive determines whether it
171 * is initialized by the user before the call or by
172 * the backend as a result of this call. Single-shot
173 * synchronizatiom objects are initialized as a result
174 * of the call. Reusable synchronization objects are
175 * generally initialized by the user prior to the
176 * call.
177 * @param[out] traceEvents - optional pointer to onnxTraceEventList that can be
178 * NULL. If non-NULL then the backend is requested to
179 * populate the onnxTraceEventList with trace events
180 * describing the timeline of events that occurred
181 * while running the graph.
182 *
183 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the all graph
184 * inputs and outputs were matched to a memory
185 * location.
186 * @retval ONNXIFI_STATUS_INVALID_GRAPH The function call failed because
187 * graph is not an ONNXIFI graph handle.
188 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
189 * outputDescriptors pointer is NULL or
190 * inputDescriptors pointer is NULL while
191 * inputsCount is non-zero.
192 * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because one of
193 * the names in tensor descriptors doesn't
194 * match blob name in ModelProto.graph.input
195 * or ModelProto.graph.output, or the same
196 * name appears in more than one tensor
197 * descriptor.
198 * @retval ONNXIFI_STATUS_INVALID_SHAPE The function call failed because one of
199 * the shape dimensions is 0.
200 * @retval ONNXIFI_STATUS_INVALID_DATATYPE The function call failed because
201 * one of the data types in
202 * inputDescriptors or outputDescriptors
203 * is unknown to the backend.
204 * @retval ONNXIFI_STATUS_INVALID_MEMORY_TYPE The function call failed because
205 * one of the memory types in
206 * inputDescriptors or
207 * outputDescriptors is unknown to
208 * the backend.
209 * @retval ONNXIFI_STATUS_INVALID_MEMORY_LOCATION The function call failed
210 * because one of the memory
211 * locations in inputDescriptors
212 * or outputDescriptors is not
213 * valid for the specified
214 * memory type (e.g. NULL pointer
215 * for ONNXIFI_MEMORY_TYPE_CPU).
216 * @retval ONNXIFI_STATUS_UNSUPPORTED_TAG The function call failed because one
217 * of the tags in inputDescriptors or
218 * outputDescriptors is unknown to the
219 * backend (tag does not match
220 * ONNXIFI_TAG_TENSOR_DESCRIPTOR_V1).
221 * @retval ONNXIFI_STATUS_UNSUPPORTED_SHAPE The function call failed because the
222 * backend does not support the
223 * tensor shapes in an input or output
224 * of one of the operators. The
225 * problematic tensor shapes could be
226 * directly specified through
227 * inputDescriptors or
228 * outputDescriptors argument,
229 * or inferred from the inputs by the
230 * backend. This error code can be
231 * returned when the backend supports
232 * variable-size inputs and outputs,
233 * and the problematic tensor shape was
234 * provided in the ValueInfoProto as a
235 * symbolic variable.
236 * @retval ONNXIFI_STATUS_UNSUPPORTED_MEMORY_TYPE The function call failed
237 * because the backend does not
238 * support one of the memory
239 * types in inputDescriptors or
240 * outputDescriptors.
241 * @retval ONNXIFI_STATUS_UNIDENTIFIED_NAME The function call failed because one
242 * of the ValueInfoProto.name value in
243 * ModelProto.graph.input or
244 * ModelProto.graph.output doesn't have
245 * a match in the inputDescriptors or
246 * outputDescriptors.
247 * @retval ONNXIFI_STATUS_MISMATCHING_SHAPE The function call failed because
248 * the shapes specified through
249 * inputDescriptors or
250 * outputDescriptors argument are
251 * inconsistent with the shapes
252 * specified in the ONNX model graph.
253 * @retval ONNXIFI_STATUS_MISMATCHING_DATATYPE The function call failed because
254 * data types specified through
255 * inputDescriptors or
256 * outputDescriptors argument are
257 * inconsistent with the data types
258 * specified in the ONNX model
259 * graph.
260 * @retval ONNXIFI_STATUS_NO_SYSTEM_MEMORY The function call failed because the
261 * backend could not allocate enough
262 * system memory to parse, analyze, and
263 * initialize the tensor locations.
264 * @retval ONNXIFI_STATUS_NO_SYSTEM_RESOURCES The function call failed due to
265 * insufficient non-memory system
266 * resources (e.g. file handles) to
267 * initialize the tensor locations.
268 * @retval ONNXIFI_STATUS_NO_DEVICE_MEMORY The function call failed due to
269 * insufficient backend-specific memory
270 * to initialize the tensor locations.
271 * @retval ONNXIFI_STATUS_NO_DEVICE_RESOURCES The function call failed due to
272 * insufficient non-memory
273 * backend-specific resources (e.g.
274 * command queues) to initialize the
275 * tensor locations.
276 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
277 * the backend was disconnected or
278 * uninstalled from the system.
279 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
280 * backend experienced an unrecovered
281 * internal error.
282 */
283
284ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxSetIOAndRunGraph(
285 onnxGraph graph,
286 uint32_t inputsCount,
287 const onnxTensorDescriptorV1* inputDescriptors,
288 uint32_t outputsCount,
289 const onnxTensorDescriptorV1* outputDescriptors,
290 onnxMemoryFenceV1* outputFence,
291 onnxTraceEventList* traceEvents);
292
293/**
294 * Release the onnxTraceEvents contained in traceEvents.
295 *
296 * @param traceEvents - a list of onnxTraceEvents to be released.
297 *
298 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the
299 * onnxTraceEvents resources were released to the
300 * operating system.
301 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because
302 * onnxTraceEventList pointer is NULL.
303 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
304 * backend experienced an unrecovered
305 * internal error.
306 */
307
308ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI
309 onnxReleaseTraceEvents(onnxTraceEventList* traceEvents);
310
311/**
312 * Wait until an ONNXIFI event transitions to signalled state or for a specified
313 * number of milliseconds, whichever occurs first. Upon returning, if return
314 * value is ONNXIFI_STATUS_SUCCESS then eventState will be set to the state
315 * of the event. If this function returned due to timeout then eventState will
316 * be ONNXIFI_EVENT_STATE_NONSIGNALLED. If this function returns due to the
317 * event being signalled then eventState is ONNXIFI_EVENT_STATE_SIGNALLED
318 * and eventStatus will be set to a status representative of the event that is
319 * being waited on.
320 *
321 * @param event - event handle created by onnxRunGraph. While it is technically
322 * possible to use this function to events created by
323 * onnxInitEvent, this is not the intended use-case.
324 *
325 * @param timeoutMs - The number of milliseconds to wait on the event before
326 * returning. If timeoutMs is 0 then this function will block
327 * on the event without timing out similar to onnxWaitEvent.
328 *
329 * @param eventState - The state of the event upon returning. If a timeout
330 * occurred then this will be
331 * ONNXIFI_EVENT_STATE_NONSIGNALLED, otherwise if the
332 * function returns ONNXIFI_STATUS_SUCCESS and no timeout
333 * occurred this will be ONNXIFI_EVENT_STATE_SIGNALLED.
334 *
335 * @param eventStatus - A status that can be associated with the event when
336 * it is signalled. This is only guaranteed to be set if
337 * the eventState is ONNXIFI_EVENT_STATE_SIGNALLED. If
338 * the event was signalled by a method that doesn't support
339 * status signalling then eventStatus will be set to
340 * ONNXIFI_STATUS_SUCCESS as a default.
341 *
342 * @param message - A preallocated message buffer to be filled with error
343 * messages if any error has happened. Message is always null
344 * terminated.
345 *
346 * @param messageLength - At input, it carries the maximum size of the message
347 * buffer. At output, it carries the actual size of the
348 * message buffer.
349 *
350 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function
351 * returned because event transitioned to
352 * signalled state or the timeout was hit.
353 * @retval ONNXIFI_STATUS_INVALID_EVENT The function call failed because event
354 * is not an ONNXIFI event handle.
355 * @retval ONNXIFI_STATUS_BACKEND_UNAVAILABLE The function call failed because
356 * the backend was disconnected or
357 * uninstalled from the system.
358 * @retval ONNXIFI_STATUS_INTERNAL_ERROR The function call failed because the
359 * implementation experienced an
360 * unrecovered internal error.
361 */
362
363ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxWaitEventFor(
364 onnxEvent event,
365 uint32_t timeoutMs,
366 onnxEventState* eventState,
367 onnxStatus* eventStatus,
368 char* message,
369 size_t* messageLength);
370
371/**
372 * Provide a option name, get its value in string form.
373 * @param optionName - Name of the option
374 *
375 * @param optionValue - String value of the option
376 *
377 * @param optionValueLength - At input, it carries the maximum size of the
378 * message buffer. At output, it arries the
379 * actual size of the message buffer.
380 *
381 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function
382 * returned because event transitioned to
383 * signalled state or the timeout was hit.
384 * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because the
385 * backend doesn't recognize the option
386 * name.
387 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because any
388 * of supplied pointers is invalid.
389 * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed
390 * because supplied value is not
391 * valid.
392 */
393ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxGetOption(
394 const char* optionName,
395 char* optionValue,
396 size_t* optionValueLength);
397
398/**
399 * Provide a option name, set its value in string form.
400 * @param optionName - Name of the option
401 *
402 * @param optionValue - String value of the option
403 *
404 * @retval ONNXIFI_STATUS_SUCCESS The function call succeeded and the function
405 * returned because event transitioned to
406 * signalled state or the timeout was hit.
407 * @retval ONNXIFI_STATUS_INVALID_NAME The function call failed because the
408 * backend doesn't recognize the option
409 * name.
410 * @retval ONNXIFI_STATUS_INVALID_POINTER The function call failed because any
411 * of supplied pointers is invalid.
412 * @retval ONNXIFI_STATUS_UNSUPPORTED_ATTRIBUTE The function call failed
413 * because supplied value is not
414 * valid.
415 */
416ONNXIFI_PUBLIC ONNXIFI_CHECK_RESULT onnxStatus ONNXIFI_ABI onnxSetOption(
417 const char* optionName,
418 const char* optionValue);
419
420#ifdef __cplusplus
421} /* extern "C" */
422#endif
423
424#endif /* !defined(ONNXIFI_EXT_H) */
425