1 | /* |
2 | * Licensed to the Apache Software Foundation (ASF) under one |
3 | * or more contributor license agreements. See the NOTICE file |
4 | * distributed with this work for additional information |
5 | * regarding copyright ownership. The ASF licenses this file |
6 | * to you under the Apache License, Version 2.0 (the |
7 | * "License"); you may not use this file except in compliance |
8 | * with the License. You may obtain a copy of the License at |
9 | * |
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
11 | * |
12 | * Unless required by applicable law or agreed to in writing, |
13 | * software distributed under the License is distributed on an |
14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
15 | * KIND, either express or implied. See the License for the |
16 | * specific language governing permissions and limitations |
17 | * under the License. |
18 | */ |
19 | |
20 | /*! |
21 | * \file codegen_llvm.h |
22 | * \brief Common base class for generating into LLVM IR |
23 | */ |
24 | #ifndef TVM_TARGET_LLVM_CODEGEN_LLVM_H_ |
25 | #define TVM_TARGET_LLVM_CODEGEN_LLVM_H_ |
26 | |
27 | #ifdef TVM_LLVM_VERSION |
28 | |
29 | #include <llvm/ADT/ArrayRef.h> |
30 | #include <llvm/ADT/StringRef.h> |
31 | #include <llvm/IR/BasicBlock.h> |
32 | #include <llvm/IR/ConstantFolder.h> |
33 | #include <llvm/IR/Constants.h> |
34 | #include <llvm/IR/DerivedTypes.h> |
35 | #if TVM_LLVM_VERSION >= 150 |
36 | #include <llvm/IR/FMF.h> |
37 | #else |
38 | #include <llvm/IR/Operator.h> |
39 | #endif |
40 | #include <llvm/IR/DebugInfoMetadata.h> |
41 | #include <llvm/IR/GlobalValue.h> |
42 | #include <llvm/IR/IRBuilder.h> |
43 | #include <llvm/IR/Instructions.h> |
44 | #include <llvm/IR/Intrinsics.h> |
45 | #include <llvm/Support/Casting.h> |
46 | #if TVM_LLVM_VERSION >= 140 |
47 | #include <llvm/MC/TargetRegistry.h> |
48 | #else |
49 | #include <llvm/Support/TargetRegistry.h> |
50 | #endif |
51 | |
52 | #include <tvm/arith/analyzer.h> |
53 | #include <tvm/ir/module.h> |
54 | #include <tvm/target/codegen.h> |
55 | #include <tvm/tir/analysis.h> |
56 | #include <tvm/tir/expr.h> |
57 | #include <tvm/tir/function.h> |
58 | #include <tvm/tir/op.h> |
59 | #include <tvm/tir/op_attr_types.h> |
60 | #include <tvm/tir/stmt.h> |
61 | #include <tvm/tir/stmt_functor.h> |
62 | |
63 | #include <algorithm> |
64 | #include <memory> |
65 | #include <string> |
66 | #include <unordered_map> |
67 | #include <unordered_set> |
68 | #include <utility> |
69 | #include <vector> |
70 | |
71 | #include "../../runtime/thread_storage_scope.h" |
72 | #include "../../tir/transforms/ir_utils.h" |
73 | #include "codegen_params.h" |
74 | #include "llvm_instance.h" |
75 | |
76 | namespace llvm { |
77 | class Argument; |
78 | class CallInst; |
79 | class Function; |
80 | class GlobalVariable; |
81 | class Instruction; |
82 | class PassManagerBuilder; |
83 | class DIFile; |
84 | class DICompileUnit; |
85 | class MDNode; |
86 | |
87 | // Used in std::unique_ptr |
88 | class Module; |
89 | class DataLayout; |
90 | class DIBuilder; |
91 | class MDBuilder; |
92 | } // namespace llvm |
93 | |
94 | namespace tvm { |
95 | namespace codegen { |
96 | |
97 | using namespace tir; |
98 | |
99 | /*! |
100 | * \brief A base class to generate a LLVM. |
101 | */ |
102 | class CodeGenLLVM : public ExprFunctor<llvm::Value*(const PrimExpr&)>, |
103 | public StmtFunctor<void(const Stmt&)> { |
104 | public: |
105 | CodeGenLLVM(); // Do not make it default here. |
106 | virtual ~CodeGenLLVM(); // Do not make it default here. |
107 | |
108 | /*! |
109 | * \brief Create new code generator based on target machine. |
110 | * \param tm The target machine |
111 | * \return The created llvm generator. |
112 | */ |
113 | static std::unique_ptr<CodeGenLLVM> Create(LLVMTarget* llvm_target); |
114 | /*! |
115 | * \brief Initialize the code generator with given context |
116 | * \param module_name The name of the module. |
117 | * \param tm Target machine model |
118 | * \param ctx The context. |
119 | * \param system_lib Whether to insert system library registration. |
120 | * \param dynamic_lookup Whether dynamically lookup runtime function |
121 | * or use the runtime function table passed by caller. |
122 | * \param target_c_runtime If true, generate a module to be executed by the C runtime. In practice |
123 | * this option influences whether global ctors are used. |
124 | */ |
125 | virtual void Init(const std::string& module_name, LLVMTarget* llvm_target, bool system_lib, |
126 | bool dynamic_lookup, bool target_c_runtime); |
127 | |
128 | /*! |
129 | * \brief Turn on fast math flags for floating point operations. |
130 | * \param fmf FastMathFlags to use for code generation. |
131 | */ |
132 | void SetFastMathFlags(llvm::FastMathFlags fmf); |
133 | |
134 | /*! |
135 | * \brief Compile and add function f to the current module. |
136 | * \param f The function to be added. |
137 | */ |
138 | virtual void AddFunction(const PrimFunc& f); |
139 | /*! |
140 | * \brief Add main function as the entry name |
141 | * \param entry_func_name The name of entry function to be added. |
142 | */ |
143 | virtual void AddMainFunction(const std::string& entry_func_name); |
144 | /*! |
145 | * \brief Finish current pass of codegen, get the module. |
146 | * \return the created module. |
147 | */ |
148 | virtual std::unique_ptr<llvm::Module> Finish(); |
149 | /*! |
150 | * \brief Add functions from the (unordered) range to the current module in a deterministic order. |
151 | * The range consists of objects convertible to PrimFunc. |
152 | * \param begin The beginning of the range. |
153 | * \param end The end of the range. |
154 | * \param pfunc Converter function from the range element type to PrimFunc. |
155 | */ |
156 | template <typename IterType, typename ConvType> |
157 | void AddFunctionsOrdered(IterType begin, IterType end, ConvType pfunc); |
158 | /*! |
159 | * \brief Add functions from the (unordered) range of elements of type PrimFunc to the current |
160 | * module in a deterministic order. |
161 | * \param begin The beginning of the range. |
162 | * \param end The end of the range. |
163 | */ |
164 | template <typename IterType> |
165 | void AddFunctionsOrdered(IterType begin, IterType end) { |
166 | this->AddFunctionsOrdered(begin, end, [](auto f) { return f; }); |
167 | } |
168 | /*! |
169 | * \brief Add mod to be linked with the generated module |
170 | * \param mod The module to be linked. |
171 | */ |
172 | void AddLinkModule(std::unique_ptr<llvm::Module>&& mod); |
173 | /*! |
174 | * \brief Create Value for expression e |
175 | * \param e The expression to be created value for. |
176 | * \return created value. |
177 | */ |
178 | llvm::Value* MakeValue(const PrimExpr& e) { return VisitExpr(e); } |
179 | // Short hande code to get a constant int 32 |
180 | llvm::Constant* ConstInt32(int64_t value) const { |
181 | return llvm::ConstantInt::getSigned(t_int32_, value); |
182 | } |
183 | // override codegen |
184 | llvm::Value* VisitExpr_(const VarNode* op) override; |
185 | llvm::Value* VisitExpr_(const CastNode* op) override; |
186 | llvm::Value* VisitExpr_(const IntImmNode* op) override; |
187 | llvm::Value* VisitExpr_(const FloatImmNode* op) override; |
188 | llvm::Value* VisitExpr_(const StringImmNode* op) override; |
189 | llvm::Value* VisitExpr_(const AddNode* op) override; |
190 | llvm::Value* VisitExpr_(const SubNode* op) override; |
191 | llvm::Value* VisitExpr_(const MulNode* op) override; |
192 | llvm::Value* VisitExpr_(const DivNode* op) override; |
193 | llvm::Value* VisitExpr_(const ModNode* op) override; |
194 | llvm::Value* VisitExpr_(const MinNode* op) override; |
195 | llvm::Value* VisitExpr_(const MaxNode* op) override; |
196 | llvm::Value* VisitExpr_(const LTNode* op) override; |
197 | llvm::Value* VisitExpr_(const LENode* op) override; |
198 | llvm::Value* VisitExpr_(const GTNode* op) override; |
199 | llvm::Value* VisitExpr_(const GENode* op) override; |
200 | llvm::Value* VisitExpr_(const EQNode* op) override; |
201 | llvm::Value* VisitExpr_(const NENode* op) override; |
202 | llvm::Value* VisitExpr_(const AndNode* op) override; |
203 | llvm::Value* VisitExpr_(const OrNode* op) override; |
204 | llvm::Value* VisitExpr_(const NotNode* op) override; |
205 | llvm::Value* VisitExpr_(const SelectNode* op) override; |
206 | llvm::Value* VisitExpr_(const LetNode* op) override; |
207 | llvm::Value* VisitExpr_(const LoadNode* op) override; |
208 | llvm::Value* VisitExpr_(const BufferLoadNode* op) override; |
209 | llvm::Value* VisitExpr_(const CallNode* op) override; |
210 | llvm::Value* VisitExpr_(const RampNode* op) override; |
211 | llvm::Value* VisitExpr_(const ShuffleNode* op) override; |
212 | llvm::Value* VisitExpr_(const BroadcastNode* op) override; |
213 | // stmt |
214 | void VisitStmt_(const StoreNode* op) override; |
215 | void VisitStmt_(const BufferStoreNode* op) override; |
216 | void VisitStmt_(const ForNode* op) override; |
217 | void VisitStmt_(const WhileNode* op) override; |
218 | void VisitStmt_(const IfThenElseNode* op) override; |
219 | void VisitStmt_(const AllocateNode* op) override; |
220 | void VisitStmt_(const AllocateConstNode* op) override; |
221 | void VisitStmt_(const AttrStmtNode* op) override; |
222 | void VisitStmt_(const AssertStmtNode* op) override; |
223 | void VisitStmt_(const LetStmtNode* op) override; |
224 | void VisitStmt_(const SeqStmtNode* op) override; |
225 | void VisitStmt_(const EvaluateNode* op) override; |
226 | |
227 | // Get constant string |
228 | llvm::Constant* GetConstString(const std::string& str); |
229 | |
230 | llvm::Constant* GetGlobalConstant( |
231 | llvm::Constant* const_data, const std::string& name = "" , |
232 | llvm::GlobalValue::LinkageTypes linkage_type = llvm::GlobalValue::InternalLinkage); |
233 | |
234 | protected: |
235 | /*! |
236 | * \brief Address and type pair to assist in handling opaque pointers. |
237 | */ |
238 | struct TypedPointer { |
239 | TypedPointer() = default; |
240 | TypedPointer(llvm::Type* t, llvm::Value* a) : type(t), addr(a) {} |
241 | llvm::Type* type = nullptr; /*!< Type of the value pointed to. */ |
242 | llvm::Value* addr = nullptr; /*!< Address of the value. */ |
243 | }; |
244 | /*! \brief The storage information */ |
245 | struct StorageInfo { |
246 | /*! \brief The alignment of allocation */ |
247 | int alignment{0}; |
248 | }; |
249 | /*! |
250 | * \brief Convert tvm::runtime::String into llvm::StringRef |
251 | */ |
252 | static llvm::StringRef MakeStringRef(const String& string) { |
253 | return llvm::StringRef(string.c_str(), string.size()); |
254 | } |
255 | /*! |
256 | * \brief Execute falloca at the beginning of the |
257 | * currrent function and obtain its return value. |
258 | * |
259 | * This is a helper function to make sure that |
260 | * alloca always happen in the beginning of the function. |
261 | * |
262 | * \param falloca The allocation function to be executed. |
263 | * \tparam F The function to be executed. |
264 | * \return The result. |
265 | */ |
266 | template <typename F> |
267 | llvm::AllocaInst* WithFunctionEntry(F falloca) { |
268 | llvm::BasicBlock* current = builder_->GetInsertBlock(); |
269 | llvm::BasicBlock* entry = &(function_->getEntryBlock()); |
270 | builder_->SetInsertPoint(entry, entry->begin()); |
271 | llvm::AllocaInst* res = falloca(); |
272 | builder_->SetInsertPoint(current); |
273 | return res; |
274 | } |
275 | // create intrinstic given call |
276 | virtual llvm::Value* CreateIntrinsic(const CallNode* op); |
277 | // create extern function call |
278 | // skip first arg mode used for call extern intrinsic. |
279 | virtual llvm::Value* CreateCallExtern(Type ret_type, String global_symbol, |
280 | const Array<PrimExpr>& args, bool skip_first_arg); |
281 | |
282 | /*! \brief Insert a printf() call to the generated LLVM |
283 | * |
284 | * This is intended solely for debugging purposes. After calling |
285 | * printf(), immediately calls fflush() to flush the stdout buffer |
286 | * in case of segfault. |
287 | */ |
288 | virtual void CreatePrintf(const std::string& format, llvm::ArrayRef<llvm::Value*> format_args); |
289 | |
290 | /*! \brief Lookup return address, for debugging purposes |
291 | * |
292 | * This is intended solely for debugging purposes. Calls the |
293 | * `llvm::Intrinsic::returnaddress`, returning the return address of |
294 | * the current function call. |
295 | * |
296 | * \param level Look up the return address of a frame `level` steps |
297 | * above the current stack frame. |
298 | */ |
299 | llvm::Value* CreateLookupReturnAddress(unsigned int level = 0); |
300 | |
301 | // Get the corresponding thread index |
302 | virtual llvm::Value* GetThreadIndex(const IterVar& iv); |
303 | // Get the corresponding thread index |
304 | virtual llvm::Value* CreateStorageSync(const CallNode* op); |
305 | #if TVM_LLVM_VERSION < 160 |
306 | // This function only works with the legacy pass manager. |
307 | // apply optimization on the module. |
308 | virtual void InitPassManagerBuilder(llvm::PassManagerBuilder* builder); |
309 | #endif |
310 | // Scalarize by iterating elements of e. |
311 | // f is a callback that takes index and v. |
312 | void Scalarize(const PrimExpr& e, std::function<void(int i, llvm::Value* v)> f); |
313 | |
314 | /* \brief Helper function for handling buffer access |
315 | * |
316 | * \param buffer The buffer being accessed |
317 | * |
318 | * \param indices The indices at which the buffer is being accessed. |
319 | * |
320 | * \param value_dtype The datatype to be read from (BufferLoad) or |
321 | * written to (BufferStore) the buffer. |
322 | * |
323 | * \param make_instruction A callback function that generates that |
324 | * actual call. |
325 | * |
326 | * - buffer_ptr: A typed pointer to the element being accessed |
327 | * |
328 | * - subelement_i: The index of a vectorized type to be |
329 | * stored/loaded. If -1, indicates that the entire type, |
330 | * vector or scalar, should be written. |
331 | * |
332 | * - alignment: The alignment to be used for the read/write. |
333 | * |
334 | * - is_volatile: Whether the read/write should be volatile. |
335 | * |
336 | * - Should return the generated expression. |
337 | */ |
338 | void BufferAccessHelper( |
339 | Buffer buffer, Array<PrimExpr> indices, DataType value_dtype, |
340 | std::function<llvm::Instruction*(TypedPointer buffer_ptr, int subelement_i, int alignment, |
341 | bool is_volatile)> |
342 | make_instruction); |
343 | // Initialize target |
344 | virtual void InitTarget(); |
345 | // Add module startup function if needed. |
346 | virtual void AddStartupFunction() {} |
347 | // apply optimization on the module. |
348 | virtual void Optimize(); |
349 | // Get the maximim storage align bits of buffer pointer given storage scope. |
350 | virtual int NativeVectorBits(const runtime::StorageScope& storage_scope) const; |
351 | // Get correct address space depending on the backend |
352 | virtual unsigned GetGlobalAddressSpace() const; |
353 | void AddFunctionInternal(const PrimFunc& f, bool ret_void); |
354 | // Create extern call |
355 | llvm::CallInst* CreateCallExtern(llvm::Type* ret, const std::string& name, |
356 | const std::vector<llvm::Value*>& value); |
357 | /*! |
358 | * \brief Get the LLVM Type for a given runtime type. |
359 | * \param dtype The runtime dtype. |
360 | * |
361 | * \note Only use this function for dealing with PrimTypes. |
362 | * For Call and Var that could have more refined types, |
363 | * use GetLLVMType instead. |
364 | * |
365 | * \return LLVM type of dtype |
366 | */ |
367 | llvm::Type* DTypeToLLVMType(const DataType& dtype) const; |
368 | /*! |
369 | * \brief Get the LLVM Type for a given type. |
370 | * \param dtype The runtime dtype. |
371 | * \param type The corresponding TVM Type. |
372 | */ |
373 | llvm::Type* GetLLVMType(const Type& type) const; |
374 | /*! |
375 | * \brief Get the LLVM Type for a given type. |
376 | * \param dtype The runtime dtype. |
377 | * \param type The corresponding TVM Type. |
378 | */ |
379 | llvm::Type* GetLLVMType(const PrimExpr& expr) const; |
380 | /*! |
381 | * \brief Get the declaration of the LLVM intrinsic based on the intrinsic |
382 | * id, and the type of the actual call. |
383 | * |
384 | * \param id The intrinsic id. |
385 | * \param ret_type The call return type. |
386 | * \param arg_types The types of the call arguments. |
387 | * |
388 | * \return Return the llvm::Function pointer, or nullptr if the declaration |
389 | * could not be generated (e.g. if the argument/return types do not |
390 | * match). |
391 | */ |
392 | llvm::Function* GetIntrinsicDecl(llvm::Intrinsic::ID id, llvm::Type* ret_type, |
393 | llvm::ArrayRef<llvm::Type*> arg_types); |
394 | /*! |
395 | * \brief Set target-related attributes on the LLVM function \p func. This |
396 | * includes "target-cpu" and "target-features" if present. |
397 | * |
398 | * \param func The function to set attributes on. |
399 | */ |
400 | void SetTargetAttributes(llvm::Function* func); |
401 | /*! |
402 | * \brief Emit LLVM IR for conversion functions __extendhfsf2 and __truncsfhf2 |
403 | * into the current llvm::Module. |
404 | * |
405 | * \param use_float16_abi Whether to use floating-point or integer ABI. |
406 | */ |
407 | void EmitFloat16ConversionBuiltins(bool use_float16_abi); |
408 | |
409 | /*! |
410 | * \brief Get the number of elements in the given vector value. |
411 | * \param vec The value, must be of a vector type. |
412 | */ |
413 | inline int GetVectorNumElements(llvm::Value* vec); |
414 | // initialize the function state. |
415 | void InitFuncState(); |
416 | // Get alignment given index. |
417 | void GetAlignment(DataType t, const VarNode* buf_var, const PrimExpr& index, int* p_alignment, |
418 | int* p_native_bits); |
419 | // Returns whether the LLVM type has padding for alignment |
420 | bool HasAlignmentPadding(DataType dtype); |
421 | // do a scalarize call with f |
422 | llvm::Value* CreateScalarizedCall(const CallNode* op, llvm::Function* f, |
423 | const std::vector<llvm::Value*>& args); |
424 | // handle module import |
425 | void HandleImport(const std::string& code); |
426 | // cast operatpr |
427 | llvm::Value* CreateCast(DataType from, DataType to, llvm::Value* value); |
428 | // comparison op |
429 | llvm::Value* GetVarValue(const VarNode* v) const; |
430 | llvm::Value* CreateLT(DataType t, llvm::Value* a, llvm::Value* b); |
431 | llvm::Value* CreateLE(DataType t, llvm::Value* a, llvm::Value* b); |
432 | llvm::Value* CreateGT(DataType t, llvm::Value* a, llvm::Value* b); |
433 | llvm::Value* CreateGE(DataType t, llvm::Value* a, llvm::Value* b); |
434 | llvm::Value* CreateAdd(DataType t, llvm::Value* a, llvm::Value* b); |
435 | llvm::Value* CreateSub(DataType t, llvm::Value* a, llvm::Value* b); |
436 | llvm::Value* CreateMul(DataType t, llvm::Value* a, llvm::Value* b); |
437 | llvm::Value* CreateBroadcast(llvm::Value* value, int lanes); |
438 | virtual TypedPointer CreateBufferPtr(llvm::Value* buffer_ptr, DataType buffer_element_dtype, |
439 | llvm::ArrayRef<llvm::Value*> indices, DataType value_dtype); |
440 | // Vector concatenation. |
441 | llvm::Value* CreateVecSlice(llvm::Value* vec, int begin, int extent); |
442 | llvm::Value* CreateVecFlip(llvm::Value* vec); |
443 | llvm::Value* CreateVecConcat(std::vector<llvm::Value*> vecs); |
444 | llvm::Value* CreateVecPad(llvm::Value* vec, int target_lanes); |
445 | // Create serial for |
446 | void CreateSerialFor(llvm::Value* begin, llvm::Value* end, llvm::Value* stride, |
447 | const Var& loop_var, const Stmt& body); |
448 | // add alias information. |
449 | void AddAliasInfo(llvm::Instruction* inst, const VarNode* buffer_var, PrimExpr index, |
450 | DataType access_dtype); |
451 | |
452 | llvm::GlobalVariable* AllocateSharedMemory(DataType dtype, size_t size, |
453 | unsigned int shared_address_space, int alignment, |
454 | llvm::GlobalValue::LinkageTypes linkage); |
455 | |
456 | /*! |
457 | * \brief Get the `i`th argument to the given function, respecting LLVM API changes. |
458 | * |
459 | * NOTE: in LLVM < 10.0, the underlying API returns a const llvm::Argument*. To provide a uniform |
460 | * API, const is removed here. Proper usage of LLVM APIs depends on having a non-const Argument*, |
461 | * so we take this appraoch here rather than adding const. |
462 | * |
463 | * \param function The function containing the arguments. |
464 | * \param i The index of the argument to retrieve. |
465 | * \return The retrieved argument. |
466 | */ |
467 | llvm::Argument* GetArg(const llvm::Function* function, int i) const { |
468 | #if TVM_LLVM_VERSION >= 100 |
469 | return function->getArg(i); |
470 | #elif TVM_LLVM_VERSION >= 50 |
471 | return const_cast<llvm::Argument*>(&function->arg_begin()[i]); |
472 | #else |
473 | return const_cast<llvm::Argument*>(&*std::next(function->arg_begin(), i)); |
474 | #endif |
475 | } |
476 | |
477 | // The IRBuilder. |
478 | using IRBuilder = llvm::IRBuilder<llvm::ConstantFolder, llvm::IRBuilderDefaultInserter>; |
479 | // The current function |
480 | llvm::Function* function_; |
481 | // Internal builder |
482 | std::unique_ptr<IRBuilder> builder_; |
483 | // The module to be returned; |
484 | std::unique_ptr<llvm::Module> module_; |
485 | std::unique_ptr<llvm::DataLayout> data_layout_; |
486 | // Internal metabuilder |
487 | std::unique_ptr<llvm::MDBuilder> md_builder_; |
488 | // llvm target info |
489 | LLVMTarget* llvm_target_{nullptr}; |
490 | // helpful data types |
491 | llvm::Type* t_void_{nullptr}; |
492 | llvm::PointerType* t_void_p_{nullptr}; |
493 | llvm::Type* t_int_{nullptr}; |
494 | llvm::Type* t_char_{nullptr}; |
495 | llvm::Type* t_int8_{nullptr}; |
496 | llvm::Type* t_int16_{nullptr}; |
497 | llvm::Type* t_int32_{nullptr}; |
498 | llvm::Type* t_int64_{nullptr}; |
499 | llvm::Type* t_float64_{nullptr}; |
500 | // meta data |
501 | llvm::MDNode* md_very_likely_branch_{nullptr}; |
502 | llvm::MDNode* md_tbaa_root_{nullptr}; |
503 | llvm::MDNode* md_tbaa_alias_set_{nullptr}; |
504 | // modules to be linked. |
505 | std::vector<std::unique_ptr<llvm::Module>> link_modules_; |
506 | /*! \brief native vector bits of current targetx*/ |
507 | int native_vector_bits_{0}; |
508 | /*! \brief the storage scope of allocation */ |
509 | std::unordered_map<const VarNode*, StorageInfo> alloc_storage_info_; |
510 | // The definition of local variable. |
511 | std::unordered_map<const VarNode*, llvm::Value*> var_map_; |
512 | // global strings |
513 | std::unordered_map<std::string, llvm::Constant*> str_map_; |
514 | // Whether current function is restricted |
515 | bool is_restricted_{true}; |
516 | // The analyzer information |
517 | std::unique_ptr<arith::Analyzer> analyzer_; |
518 | // set of var that are not restricted(can alias) |
519 | std::unordered_set<const VarNode*> alias_var_set_; |
520 | // set of volatile buffer. |
521 | std::unordered_set<const VarNode*> volatile_buf_; |
522 | // deep comparison of PrimExpr |
523 | ExprDeepEqual deep_equal_; |
524 | // binding of let variables. Enables duplicate var defs that map to same value |
525 | std::unordered_map<Var, const LetNode*, ObjectPtrHash, ObjectPtrEqual> let_binding_; |
526 | // debug info for function being compiled |
527 | llvm::DISubprogram* di_subprogram_; |
528 | // Cache potential common path ops to slightly improve lookup time. |
529 | // global symbol table. |
530 | OpAttrMap<TGlobalSymbol> op_attr_global_symbol_ = Op::GetAttrMap<TGlobalSymbol>("TGlobalSymbol" ); |
531 | const Op& builtin_call_extern_ = builtin::call_extern(); |
532 | const Op& builtin_call_pure_extern_ = builtin::call_pure_extern(); |
533 | const Op& builtin_call_llvm_intrin_ = builtin::call_llvm_intrin(); |
534 | const Op& builtin_call_llvm_pure_intrin_ = builtin::call_llvm_pure_intrin(); |
535 | const Op& builtin_lookup_param_ = builtin::lookup_param(); |
536 | const Op& builtin_tvm_call_cpacked_lowered_ = builtin::tvm_call_cpacked_lowered(); |
537 | |
538 | void EmitDebugLocation(); |
539 | void EmitDebugLocation(const Span& span); |
540 | void EmitDebugLocation(const StmtNode* op); |
541 | |
542 | /*! \brief Helper struct for debug infos. */ |
543 | struct DebugInfo { |
544 | ~DebugInfo(); // Because of the std::unique_ptr. |
545 | std::unique_ptr<llvm::DIBuilder> di_builder_; |
546 | llvm::DICompileUnit* compilation_unit_{nullptr}; |
547 | llvm::DIFile* file_{nullptr}; |
548 | }; |
549 | /*! |
550 | * \brief Create a new DebugInfo struct from the given Module that |
551 | * initializes file and compilation_unit_ to TVM defaults. |
552 | */ |
553 | static std::unique_ptr<DebugInfo> CreateDebugInfo(llvm::Module* module); |
554 | }; |
555 | |
556 | inline int CodeGenLLVM::GetVectorNumElements(llvm::Value* vec) { |
557 | #if TVM_LLVM_VERSION >= 120 |
558 | return llvm::cast<llvm::FixedVectorType>(vec->getType())->getNumElements(); |
559 | #else |
560 | return llvm::cast<llvm::VectorType>(vec->getType())->getNumElements(); |
561 | #endif |
562 | } |
563 | |
564 | template <typename IterType, typename ConvType> |
565 | void CodeGenLLVM::AddFunctionsOrdered(IterType begin, IterType end, ConvType pfunc) { |
566 | std::vector<PrimFunc> funcs; |
567 | for (auto it = begin; it != end; ++it) { |
568 | funcs.push_back(pfunc(*it)); |
569 | } |
570 | std::sort(funcs.begin(), funcs.end(), [](PrimFunc func_a, PrimFunc func_b) { |
571 | std::string name_a = func_a->GetAttr<String>(tvm::attr::kGlobalSymbol).value(); |
572 | std::string name_b = func_b->GetAttr<String>(tvm::attr::kGlobalSymbol).value(); |
573 | return name_a < name_b; |
574 | }); |
575 | for (auto& f : funcs) { |
576 | auto global_symbol = f->GetAttr<String>(tvm::attr::kGlobalSymbol); |
577 | AddFunction(f); |
578 | } |
579 | } |
580 | |
581 | } // namespace codegen |
582 | } // namespace tvm |
583 | |
584 | #endif // TVM_LLVM_VERSION |
585 | #endif // TVM_TARGET_LLVM_CODEGEN_LLVM_H_ |
586 | |