1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #ifndef GLOW_BACKENDS_BACKEND_H |
17 | #define GLOW_BACKENDS_BACKEND_H |
18 | |
19 | #include "glow/Backend/CompiledFunction.h" |
20 | #include "glow/Backends/BackendOptions.h" |
21 | #include "glow/Base/Traits.h" |
22 | #include "glow/Optimizer/GraphOptimizer/CompilationContext.h" |
23 | #include "glow/Optimizer/GraphOptimizer/FunctionPassPipeline.h" |
24 | #include "glow/Optimizer/IROptimizer/IRFunctionPassPipeline.h" |
25 | #include "glow/Support/Register.h" |
26 | |
27 | #include "llvm/ADT/StringRef.h" |
28 | #include "llvm/Support/MemoryBuffer.h" |
29 | |
30 | #if FACEBOOK_INTERNAL |
31 | namespace folly { |
32 | struct dynamic; |
33 | } |
34 | namespace glow { |
35 | struct CompiledResult; |
36 | } |
37 | #endif |
38 | |
39 | namespace glow { |
40 | |
41 | class IRFunction; |
42 | class Node; |
43 | class PlaceholderBindings; |
44 | class IRGenVisitor; |
45 | class TensorLayoutCommon; |
46 | |
47 | /// Information about an entry point of a saved bundle. |
48 | struct BundleEntry { |
49 | /// Name of the bundle entry point for the function to be saved. |
50 | std::string name; |
51 | /// Function to be saved. |
52 | Function *func; |
53 | }; |
54 | |
55 | namespace runtime { |
56 | |
57 | class DeviceManager; |
58 | struct DeviceInfo; |
59 | struct DeviceConfig; |
60 | struct ContextBinding; |
61 | |
62 | struct DAGNode; |
63 | |
64 | } // namespace runtime |
65 | |
66 | // This is the interface that glow backends need to implement. |
67 | class Backend : public Named { |
68 | public: |
69 | Backend() : Named("" ) {} |
70 | /// Dtor. |
71 | virtual ~Backend() = default; |
72 | |
73 | // \returns backend name. |
74 | virtual std::string getBackendName() const = 0; |
75 | |
76 | /// Generate code for a vector of functions, \p functions. Each compilation |
77 | /// has its own settings in \p opts. This allows the compiler to |
78 | /// support shared constants between functions. |
79 | virtual Expected<llvm::StringMap<std::unique_ptr<CompiledFunction>>> |
80 | compileFunctions(std::vector<Function *> &functions, |
81 | llvm::StringMap<BackendOptions> &optsMap) const; |
82 | |
83 | virtual Expected<std::unique_ptr<CompiledFunction>> |
84 | compile(Function *F) const { |
85 | BackendOptions opts; |
86 | return compile(F, opts); |
87 | } |
88 | |
89 | /// Generate compiled function from FXIR |
90 | |
91 | #if FACEBOOK_INTERNAL |
92 | virtual Expected<std::unique_ptr<CompiledFunction>> |
93 | compileFX(const folly::dynamic &FXIR, const std::string &submod, |
94 | const llvm::StringMap<const void *> &constants, |
95 | const BackendOptions &opts, Module *glowModule) const { |
96 | LOG(FATAL) << "Compiling FXIR is not supported by the backend" ; |
97 | } |
98 | |
99 | virtual Expected<std::unique_ptr<CompiledResult>> |
100 | compileFXToCompiledResults(const folly::dynamic &FXIR, |
101 | const llvm::StringMap<const void *> &constants, |
102 | bool /* lowerToLLVMIR */ = true, |
103 | bool /* eagerMode */ = false) const { |
104 | LOG(FATAL) << "Compiling FXIR is not supported by the backend" ; |
105 | } |
106 | #endif |
107 | |
108 | /// Generate code for input function \param F given settings in \p opts. |
109 | virtual Expected<std::unique_ptr<CompiledFunction>> |
110 | compile(Function *F, const BackendOptions &opts) const = 0; |
111 | |
112 | /// Save the bundle for \p F for a later standalone execution in \p outputDir |
113 | /// under name \p bundleName. Make \p mainEntryName the function name for the |
114 | /// entry point of the network and prepend all generated files with this name. |
115 | virtual void save(Function *F, llvm::StringRef outputDir, |
116 | llvm::StringRef bundleName, |
117 | llvm::StringRef mainEntryName) const { |
118 | LOG(FATAL) << "Saving a bundle is not supported by the backend" ; |
119 | } |
120 | |
121 | virtual void saveFunctions(llvm::ArrayRef<BundleEntry> entries, |
122 | llvm::StringRef outputDir, |
123 | llvm::StringRef bundleName) const { |
124 | LOG(FATAL) << "Saving a bundle is not supported by the backend" ; |
125 | } |
126 | |
127 | /// Used by the compiler during graph optimization and before code generation, |
128 | /// giving the backend an opportunity to transform the graph before IRGen. The |
129 | /// backend may insert backend and device-specific nodes. The backend is |
130 | /// responsible for cleaning up after itself. |
131 | /// \returns an Expected True if the graph was modified. |
132 | virtual Expected<bool> transformPostLowering( |
133 | Function *F, CompilationContext &cctx, |
134 | const glow::runtime::DeviceInfo *devInfo = nullptr) const { |
135 | return false; |
136 | } |
137 | |
138 | /// \returns true if Constants must be actually quantized before |
139 | /// Post-Lowering, false if it must be done after post-lowering. |
140 | virtual bool shouldPreQuantizeConstants() const { return true; } |
141 | |
142 | /// \returns true if a module should be stripped after deployment/compilation. |
143 | virtual bool shouldStripModule() const { return true; } |
144 | |
145 | /// \returns whether the provided \p NI is supported by the backend. |
146 | virtual bool isOpSupported(const NodeInfo &NI) const = 0; |
147 | |
148 | /// \returns whether the backend would like to accept \p NI for execution. By |
149 | /// default falls back to just checking for support via \ref isOpSupported(), |
150 | /// however can also take into account things like performance considerations. |
151 | virtual bool acceptForExecution(const NodeInfo &NI) const { |
152 | return isOpSupported(NI); |
153 | } |
154 | |
155 | /// \returns whether all nodes inside \p F are supported. \p verbose |
156 | /// represents whether to print Nodes that are unsupported. |
157 | bool checkAllNodesSupported(const Function &F, bool verbose = true) const; |
158 | |
159 | /// \returns whether the provided \p F conforms to the backend-dependent graph |
160 | /// constraints. Giving the backend an opportunity to check that everything |
161 | /// conforms to its specific restrictions by overriding this function. It is |
162 | /// highly recommended for backends to make their backend specific |
163 | /// verifications a super-set of target independent Function::verify() by |
164 | /// calling it in their overridden implementation. It is not a strict |
165 | /// requirement, of course, in case they diverge / the backend has a good |
166 | /// reason not to call Function::verify(). \p verbose represents whether to |
167 | /// print out nodes that are unsupported by the backend. |
168 | virtual bool verify(const Function &F, bool verbose = true) const; |
169 | |
170 | /// \returns whether the provided \p IR conforms to the backend-dependent |
171 | /// graph constraints. Giving the backend an opportunity to check that |
172 | /// everything conforms to its specific restrictions by overriding this |
173 | /// function. It is highly recommended for backends to make their backend |
174 | /// specific verifications a super-set of target independent |
175 | /// IRFunction::verify() by calling it in their overridden implementation. It |
176 | /// is not a strict requirement, of course, in case they diverge / the backend |
177 | /// has a good reason not to call IRFunction::verify(). |
178 | virtual bool verify(const IRFunction &IR) const; |
179 | |
180 | /// \returns a reference to the backend-specific tensor layout requirements |
181 | /// singleton. If not overridden, the default requirement is Glow's |
182 | /// "canonical" form. |
183 | virtual TensorLayoutCommon &getTensorLayoutRequirements() const; |
184 | |
185 | /// \returns true if the supplied Node \N should be lowered. By default, all |
186 | /// Nodes are candidates for lowering. |
187 | virtual bool shouldLower(const Node *N) const { return true; } |
188 | |
189 | /// \returns true if the Backend wants the buffer sharing optimization |
190 | /// performed. |
191 | virtual bool shouldShareBuffers() const { return true; } |
192 | |
193 | /// Modify the \p optimizationOpts however desired. |
194 | virtual std::unique_ptr<FunctionPassPipeline> getOptimizationPipeline() const; |
195 | /// Modify the \p optimizationOpts however desired. |
196 | virtual std::unique_ptr<IRFunctionPassPipeline> |
197 | getIROptimizationPipeline() const; |
198 | |
199 | /// \returns true if the Backend supports partial, unpadded tensors for |
200 | /// inputs that can have variable size (e.g., embedding indices). |
201 | virtual bool supportsPartialTensors() const { return false; } |
202 | |
203 | /// \returns true if the Backend supports static Placeholders. This means |
204 | /// an input can be treated as a placeholder that can be reused on the device |
205 | /// for multiple requests. |
206 | virtual bool supportsStaticPlaceholders() const { return false; } |
207 | |
208 | /// \returns whether the backend supports fusing \p activation into \p parent. |
209 | virtual bool supportsFusedActivation(Node *parent, Node *activation) const { |
210 | return false; |
211 | } |
212 | |
213 | /// \returns true if Backend generated Instruction for Node \p N, |
214 | /// using IRGenVisitor \p irgen. |
215 | virtual bool generateInst(Node *N, IRGenVisitor &irgen) const { |
216 | return false; |
217 | } |
218 | |
219 | virtual size_t getTraceEventDataSize() const { return 0; } |
220 | |
221 | /// Called outside/after the end of the optimization pipeline and before code |
222 | /// generation, giving the backend another opportunity to transform the graph |
223 | /// before IRGen. The backend may insert backend and device-specific |
224 | /// nodes. This is generally used to process backend-specific node info, as we |
225 | /// skip the optimization pipeline in such cases in order to preserve the Node |
226 | /// -> NodeInfo mapping. This is only called if not using the DAG optimizer. |
227 | /// The backend is responsible for cleaning up after itself. \returns an |
228 | /// Expected boolean for whether the graph was modified. |
229 | virtual Expected<bool> |
230 | transformPostOptPipeline(Function *F, CompilationContext &cctx) const { |
231 | return false; |
232 | } |
233 | |
234 | /// \returns an estimate of the node cost in unitless terms, or an error |
235 | /// if the cost of the node is unknown |
236 | virtual Expected<double> estimateNodeCost(const Node *node) const { |
237 | return MAKE_ERR("Backend does not support estimateNodeCost" ); |
238 | } |
239 | |
240 | /// Create device manager corresponding to the backend based on the |
241 | /// deviceConfig. |
242 | virtual runtime::DeviceManager * |
243 | createDeviceManager(const runtime::DeviceConfig &deviceConfig); |
244 | |
245 | /// Walks the provided /p bindings and does any setup needed for copying data |
246 | /// to/from host or peers. Also has access to /p root the root node of the |
247 | /// graph, which contains partition dependency and symbol information. Any |
248 | /// state information should be stored in the ExecutionContext or |
249 | /// DeviceManager. |
250 | virtual Error bindContexts(llvm::ArrayRef<runtime::ContextBinding> bindings, |
251 | const runtime::DAGNode *root, bool enableP2P, |
252 | bool enableDRT) { |
253 | return Error::success(); |
254 | } |
255 | |
256 | /// Get the number of copies of inputs/outputs that will be reserved on a |
257 | /// device per network. |
258 | virtual unsigned getContextCount(CompilationContext & /* unused */) const { |
259 | return 1; |
260 | } |
261 | |
262 | /// \returns the supported options for compiled functions (name=>description). |
263 | virtual llvm::StringMap<std::string> |
264 | getSupportedCompiledFunctionOptions() const { |
265 | return llvm::StringMap<std::string>(); |
266 | }; |
267 | |
268 | /// \returns the supported options for device managers (name=>description). |
269 | virtual llvm::StringMap<std::string> |
270 | getSupportedDeviceManagerOptions() const { |
271 | return llvm::StringMap<std::string>(); |
272 | }; |
273 | |
274 | /// \returns true if network supports Type Lowering from \p T1 to \p T2. |
275 | /// Populates PrecisionConfiguration with black list of operations that can't |
276 | /// be converted. |
277 | virtual bool |
278 | canDoIndexTypeDemotion(ElemKind fromTy, ElemKind toTy, |
279 | PrecisionConfiguration &precConfig) const { |
280 | return false; |
281 | }; |
282 | |
283 | /// \returns an array of raw objects which are statically allocated and |
284 | /// initialized by the backend and which can be used for various purposes, |
285 | /// for example to store object files (binary code) which are compiled with |
286 | /// other compilers than clang/LLVM. The raw buffers are encoded as type |
287 | /// MemoryBufferRef which stores for each buffer a name, a raw pointer |
288 | /// and a size (in bytes). |
289 | virtual llvm::ArrayRef<llvm::MemoryBufferRef> getObjectRegistry() const { |
290 | return llvm::ArrayRef<llvm::MemoryBufferRef>(); |
291 | } |
292 | |
293 | protected: |
294 | /// Parses the graph \F and builds a TraceInfo structure from any found |
295 | /// TraceEventNodes. |
296 | TraceInfo buildManualTraceInfo(Function *F) const; |
297 | |
298 | /// Inserts a TraceEventInst between every instruction, the most basic form of |
299 | /// auto instrumentation. Necessary only if the Backend doesn't provide |
300 | /// profiling/tracing in another way. |
301 | /// Modifies \p IR and updates \p traceInfo. |
302 | void autoInstrument(TraceInfo &traceInfo, IRFunction *IR) const; |
303 | }; |
304 | |
305 | /// Create a backend based on the registered backend name \p backendName. |
306 | Backend *createBackend(llvm::StringRef backendName); |
307 | |
308 | // Backends that use Glow low-level IR should inherit from this class. It allows |
309 | // for unit tests to create low-level IR to compile and run. |
310 | class BackendUsingGlowIR : public Backend { |
311 | public: |
312 | /// Generate code for input IR function \param IR. |
313 | /// This is used only for unit testing. |
314 | virtual std::unique_ptr<CompiledFunction> |
315 | compileIR(std::unique_ptr<IRFunction> IR) const = 0; |
316 | }; |
317 | |
318 | /// Perform Backend Factory registration. |
319 | #define REGISTER_GLOW_BACKEND_FACTORY(FactoryName, BackendClass) \ |
320 | class FactoryName : public BaseFactory<std::string, Backend> { \ |
321 | public: \ |
322 | Backend *create() override { return new BackendClass(); } \ |
323 | std::string getRegistrationKey() const override { \ |
324 | return BackendClass::getName(); \ |
325 | } \ |
326 | unsigned numDevices() const override { \ |
327 | return BackendClass::numDevices(); \ |
328 | } \ |
329 | std::vector<unsigned> scanDeviceIDs() const override { \ |
330 | return BackendClass::scanDeviceIDs(); \ |
331 | } \ |
332 | }; \ |
333 | static RegisterFactory<std::string, FactoryName, Backend> \ |
334 | FactoryName##_REGISTERED; |
335 | |
336 | /// Perform dynamic Backend Factory registration. Register the backend factory |
337 | /// under the provided \p Name and use \p CreateFn expression to create new |
338 | /// instances of the backends. |
339 | #define REGISTER_DYNAMIC_GLOW_BACKEND_FACTORY(FactoryName, BackendClass, Name, \ |
340 | CreateFn) \ |
341 | class FactoryName : public BaseFactory<std::string, Backend> { \ |
342 | public: \ |
343 | FactoryName() : registrationKey_(Name) {} \ |
344 | Backend *create() override { return CreateFn; } \ |
345 | std::string getRegistrationKey() const override { \ |
346 | return registrationKey_; \ |
347 | } \ |
348 | unsigned numDevices() const override { \ |
349 | return BackendClass::numDevices(); \ |
350 | } \ |
351 | std::vector<unsigned> scanDeviceIDs() const override { \ |
352 | return BackendClass::scanDeviceIDs(); \ |
353 | } \ |
354 | \ |
355 | private: \ |
356 | std::string registrationKey_; \ |
357 | }; \ |
358 | RegisterFactory<std::string, FactoryName, Backend> FactoryName##_REGISTERED; |
359 | |
360 | /// \returns the set of names for all available, registered backends. |
361 | std::vector<std::string> getAvailableBackends(); |
362 | |
363 | /// The backend name used in Glow quantization profiling. |
364 | #ifdef GLOW_WITH_CPU |
365 | constexpr const char *profilingBackend = "CPU" ; |
366 | #else |
367 | constexpr const char *profilingBackend = "Interpreter" ; |
368 | #endif |
369 | |
370 | } // namespace glow |
371 | |
372 | #endif // GLOW_BACKENDS_BACKEND_H |
373 | |