1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#ifndef GLOW_BACKENDS_BACKEND_H
17#define GLOW_BACKENDS_BACKEND_H
18
19#include "glow/Backend/CompiledFunction.h"
20#include "glow/Backends/BackendOptions.h"
21#include "glow/Base/Traits.h"
22#include "glow/Optimizer/GraphOptimizer/CompilationContext.h"
23#include "glow/Optimizer/GraphOptimizer/FunctionPassPipeline.h"
24#include "glow/Optimizer/IROptimizer/IRFunctionPassPipeline.h"
25#include "glow/Support/Register.h"
26
27#include "llvm/ADT/StringRef.h"
28#include "llvm/Support/MemoryBuffer.h"
29
30#if FACEBOOK_INTERNAL
31namespace folly {
32struct dynamic;
33}
34namespace glow {
35struct CompiledResult;
36}
37#endif
38
39namespace glow {
40
41class IRFunction;
42class Node;
43class PlaceholderBindings;
44class IRGenVisitor;
45class TensorLayoutCommon;
46
47/// Information about an entry point of a saved bundle.
48struct BundleEntry {
49 /// Name of the bundle entry point for the function to be saved.
50 std::string name;
51 /// Function to be saved.
52 Function *func;
53};
54
55namespace runtime {
56
57class DeviceManager;
58struct DeviceInfo;
59struct DeviceConfig;
60struct ContextBinding;
61
62struct DAGNode;
63
64} // namespace runtime
65
66// This is the interface that glow backends need to implement.
67class Backend : public Named {
68public:
69 Backend() : Named("") {}
70 /// Dtor.
71 virtual ~Backend() = default;
72
73 // \returns backend name.
74 virtual std::string getBackendName() const = 0;
75
76 /// Generate code for a vector of functions, \p functions. Each compilation
77 /// has its own settings in \p opts. This allows the compiler to
78 /// support shared constants between functions.
79 virtual Expected<llvm::StringMap<std::unique_ptr<CompiledFunction>>>
80 compileFunctions(std::vector<Function *> &functions,
81 llvm::StringMap<BackendOptions> &optsMap) const;
82
83 virtual Expected<std::unique_ptr<CompiledFunction>>
84 compile(Function *F) const {
85 BackendOptions opts;
86 return compile(F, opts);
87 }
88
89 /// Generate compiled function from FXIR
90
91#if FACEBOOK_INTERNAL
92 virtual Expected<std::unique_ptr<CompiledFunction>>
93 compileFX(const folly::dynamic &FXIR, const std::string &submod,
94 const llvm::StringMap<const void *> &constants,
95 const BackendOptions &opts, Module *glowModule) const {
96 LOG(FATAL) << "Compiling FXIR is not supported by the backend";
97 }
98
99 virtual Expected<std::unique_ptr<CompiledResult>>
100 compileFXToCompiledResults(const folly::dynamic &FXIR,
101 const llvm::StringMap<const void *> &constants,
102 bool /* lowerToLLVMIR */ = true,
103 bool /* eagerMode */ = false) const {
104 LOG(FATAL) << "Compiling FXIR is not supported by the backend";
105 }
106#endif
107
108 /// Generate code for input function \param F given settings in \p opts.
109 virtual Expected<std::unique_ptr<CompiledFunction>>
110 compile(Function *F, const BackendOptions &opts) const = 0;
111
112 /// Save the bundle for \p F for a later standalone execution in \p outputDir
113 /// under name \p bundleName. Make \p mainEntryName the function name for the
114 /// entry point of the network and prepend all generated files with this name.
115 virtual void save(Function *F, llvm::StringRef outputDir,
116 llvm::StringRef bundleName,
117 llvm::StringRef mainEntryName) const {
118 LOG(FATAL) << "Saving a bundle is not supported by the backend";
119 }
120
121 virtual void saveFunctions(llvm::ArrayRef<BundleEntry> entries,
122 llvm::StringRef outputDir,
123 llvm::StringRef bundleName) const {
124 LOG(FATAL) << "Saving a bundle is not supported by the backend";
125 }
126
127 /// Used by the compiler during graph optimization and before code generation,
128 /// giving the backend an opportunity to transform the graph before IRGen. The
129 /// backend may insert backend and device-specific nodes. The backend is
130 /// responsible for cleaning up after itself.
131 /// \returns an Expected True if the graph was modified.
132 virtual Expected<bool> transformPostLowering(
133 Function *F, CompilationContext &cctx,
134 const glow::runtime::DeviceInfo *devInfo = nullptr) const {
135 return false;
136 }
137
138 /// \returns true if Constants must be actually quantized before
139 /// Post-Lowering, false if it must be done after post-lowering.
140 virtual bool shouldPreQuantizeConstants() const { return true; }
141
142 /// \returns true if a module should be stripped after deployment/compilation.
143 virtual bool shouldStripModule() const { return true; }
144
145 /// \returns whether the provided \p NI is supported by the backend.
146 virtual bool isOpSupported(const NodeInfo &NI) const = 0;
147
148 /// \returns whether the backend would like to accept \p NI for execution. By
149 /// default falls back to just checking for support via \ref isOpSupported(),
150 /// however can also take into account things like performance considerations.
151 virtual bool acceptForExecution(const NodeInfo &NI) const {
152 return isOpSupported(NI);
153 }
154
155 /// \returns whether all nodes inside \p F are supported. \p verbose
156 /// represents whether to print Nodes that are unsupported.
157 bool checkAllNodesSupported(const Function &F, bool verbose = true) const;
158
159 /// \returns whether the provided \p F conforms to the backend-dependent graph
160 /// constraints. Giving the backend an opportunity to check that everything
161 /// conforms to its specific restrictions by overriding this function. It is
162 /// highly recommended for backends to make their backend specific
163 /// verifications a super-set of target independent Function::verify() by
164 /// calling it in their overridden implementation. It is not a strict
165 /// requirement, of course, in case they diverge / the backend has a good
166 /// reason not to call Function::verify(). \p verbose represents whether to
167 /// print out nodes that are unsupported by the backend.
168 virtual bool verify(const Function &F, bool verbose = true) const;
169
170 /// \returns whether the provided \p IR conforms to the backend-dependent
171 /// graph constraints. Giving the backend an opportunity to check that
172 /// everything conforms to its specific restrictions by overriding this
173 /// function. It is highly recommended for backends to make their backend
174 /// specific verifications a super-set of target independent
175 /// IRFunction::verify() by calling it in their overridden implementation. It
176 /// is not a strict requirement, of course, in case they diverge / the backend
177 /// has a good reason not to call IRFunction::verify().
178 virtual bool verify(const IRFunction &IR) const;
179
180 /// \returns a reference to the backend-specific tensor layout requirements
181 /// singleton. If not overridden, the default requirement is Glow's
182 /// "canonical" form.
183 virtual TensorLayoutCommon &getTensorLayoutRequirements() const;
184
185 /// \returns true if the supplied Node \N should be lowered. By default, all
186 /// Nodes are candidates for lowering.
187 virtual bool shouldLower(const Node *N) const { return true; }
188
189 /// \returns true if the Backend wants the buffer sharing optimization
190 /// performed.
191 virtual bool shouldShareBuffers() const { return true; }
192
193 /// Modify the \p optimizationOpts however desired.
194 virtual std::unique_ptr<FunctionPassPipeline> getOptimizationPipeline() const;
195 /// Modify the \p optimizationOpts however desired.
196 virtual std::unique_ptr<IRFunctionPassPipeline>
197 getIROptimizationPipeline() const;
198
199 /// \returns true if the Backend supports partial, unpadded tensors for
200 /// inputs that can have variable size (e.g., embedding indices).
201 virtual bool supportsPartialTensors() const { return false; }
202
203 /// \returns true if the Backend supports static Placeholders. This means
204 /// an input can be treated as a placeholder that can be reused on the device
205 /// for multiple requests.
206 virtual bool supportsStaticPlaceholders() const { return false; }
207
208 /// \returns whether the backend supports fusing \p activation into \p parent.
209 virtual bool supportsFusedActivation(Node *parent, Node *activation) const {
210 return false;
211 }
212
213 /// \returns true if Backend generated Instruction for Node \p N,
214 /// using IRGenVisitor \p irgen.
215 virtual bool generateInst(Node *N, IRGenVisitor &irgen) const {
216 return false;
217 }
218
219 virtual size_t getTraceEventDataSize() const { return 0; }
220
221 /// Called outside/after the end of the optimization pipeline and before code
222 /// generation, giving the backend another opportunity to transform the graph
223 /// before IRGen. The backend may insert backend and device-specific
224 /// nodes. This is generally used to process backend-specific node info, as we
225 /// skip the optimization pipeline in such cases in order to preserve the Node
226 /// -> NodeInfo mapping. This is only called if not using the DAG optimizer.
227 /// The backend is responsible for cleaning up after itself. \returns an
228 /// Expected boolean for whether the graph was modified.
229 virtual Expected<bool>
230 transformPostOptPipeline(Function *F, CompilationContext &cctx) const {
231 return false;
232 }
233
234 /// \returns an estimate of the node cost in unitless terms, or an error
235 /// if the cost of the node is unknown
236 virtual Expected<double> estimateNodeCost(const Node *node) const {
237 return MAKE_ERR("Backend does not support estimateNodeCost");
238 }
239
240 /// Create device manager corresponding to the backend based on the
241 /// deviceConfig.
242 virtual runtime::DeviceManager *
243 createDeviceManager(const runtime::DeviceConfig &deviceConfig);
244
245 /// Walks the provided /p bindings and does any setup needed for copying data
246 /// to/from host or peers. Also has access to /p root the root node of the
247 /// graph, which contains partition dependency and symbol information. Any
248 /// state information should be stored in the ExecutionContext or
249 /// DeviceManager.
250 virtual Error bindContexts(llvm::ArrayRef<runtime::ContextBinding> bindings,
251 const runtime::DAGNode *root, bool enableP2P,
252 bool enableDRT) {
253 return Error::success();
254 }
255
256 /// Get the number of copies of inputs/outputs that will be reserved on a
257 /// device per network.
258 virtual unsigned getContextCount(CompilationContext & /* unused */) const {
259 return 1;
260 }
261
262 /// \returns the supported options for compiled functions (name=>description).
263 virtual llvm::StringMap<std::string>
264 getSupportedCompiledFunctionOptions() const {
265 return llvm::StringMap<std::string>();
266 };
267
268 /// \returns the supported options for device managers (name=>description).
269 virtual llvm::StringMap<std::string>
270 getSupportedDeviceManagerOptions() const {
271 return llvm::StringMap<std::string>();
272 };
273
274 /// \returns true if network supports Type Lowering from \p T1 to \p T2.
275 /// Populates PrecisionConfiguration with black list of operations that can't
276 /// be converted.
277 virtual bool
278 canDoIndexTypeDemotion(ElemKind fromTy, ElemKind toTy,
279 PrecisionConfiguration &precConfig) const {
280 return false;
281 };
282
283 /// \returns an array of raw objects which are statically allocated and
284 /// initialized by the backend and which can be used for various purposes,
285 /// for example to store object files (binary code) which are compiled with
286 /// other compilers than clang/LLVM. The raw buffers are encoded as type
287 /// MemoryBufferRef which stores for each buffer a name, a raw pointer
288 /// and a size (in bytes).
289 virtual llvm::ArrayRef<llvm::MemoryBufferRef> getObjectRegistry() const {
290 return llvm::ArrayRef<llvm::MemoryBufferRef>();
291 }
292
293protected:
294 /// Parses the graph \F and builds a TraceInfo structure from any found
295 /// TraceEventNodes.
296 TraceInfo buildManualTraceInfo(Function *F) const;
297
298 /// Inserts a TraceEventInst between every instruction, the most basic form of
299 /// auto instrumentation. Necessary only if the Backend doesn't provide
300 /// profiling/tracing in another way.
301 /// Modifies \p IR and updates \p traceInfo.
302 void autoInstrument(TraceInfo &traceInfo, IRFunction *IR) const;
303};
304
305/// Create a backend based on the registered backend name \p backendName.
306Backend *createBackend(llvm::StringRef backendName);
307
308// Backends that use Glow low-level IR should inherit from this class. It allows
309// for unit tests to create low-level IR to compile and run.
310class BackendUsingGlowIR : public Backend {
311public:
312 /// Generate code for input IR function \param IR.
313 /// This is used only for unit testing.
314 virtual std::unique_ptr<CompiledFunction>
315 compileIR(std::unique_ptr<IRFunction> IR) const = 0;
316};
317
318/// Perform Backend Factory registration.
319#define REGISTER_GLOW_BACKEND_FACTORY(FactoryName, BackendClass) \
320 class FactoryName : public BaseFactory<std::string, Backend> { \
321 public: \
322 Backend *create() override { return new BackendClass(); } \
323 std::string getRegistrationKey() const override { \
324 return BackendClass::getName(); \
325 } \
326 unsigned numDevices() const override { \
327 return BackendClass::numDevices(); \
328 } \
329 std::vector<unsigned> scanDeviceIDs() const override { \
330 return BackendClass::scanDeviceIDs(); \
331 } \
332 }; \
333 static RegisterFactory<std::string, FactoryName, Backend> \
334 FactoryName##_REGISTERED;
335
336/// Perform dynamic Backend Factory registration. Register the backend factory
337/// under the provided \p Name and use \p CreateFn expression to create new
338/// instances of the backends.
339#define REGISTER_DYNAMIC_GLOW_BACKEND_FACTORY(FactoryName, BackendClass, Name, \
340 CreateFn) \
341 class FactoryName : public BaseFactory<std::string, Backend> { \
342 public: \
343 FactoryName() : registrationKey_(Name) {} \
344 Backend *create() override { return CreateFn; } \
345 std::string getRegistrationKey() const override { \
346 return registrationKey_; \
347 } \
348 unsigned numDevices() const override { \
349 return BackendClass::numDevices(); \
350 } \
351 std::vector<unsigned> scanDeviceIDs() const override { \
352 return BackendClass::scanDeviceIDs(); \
353 } \
354 \
355 private: \
356 std::string registrationKey_; \
357 }; \
358 RegisterFactory<std::string, FactoryName, Backend> FactoryName##_REGISTERED;
359
360/// \returns the set of names for all available, registered backends.
361std::vector<std::string> getAvailableBackends();
362
363/// The backend name used in Glow quantization profiling.
364#ifdef GLOW_WITH_CPU
365constexpr const char *profilingBackend = "CPU";
366#else
367constexpr const char *profilingBackend = "Interpreter";
368#endif
369
370} // namespace glow
371
372#endif // GLOW_BACKENDS_BACKEND_H
373