1 | /* |
2 | * Licensed to the Apache Software Foundation (ASF) under one |
3 | * or more contributor license agreements. See the NOTICE file |
4 | * distributed with this work for additional information |
5 | * regarding copyright ownership. The ASF licenses this file |
6 | * to you under the Apache License, Version 2.0 (the |
7 | * "License"); you may not use this file except in compliance |
8 | * with the License. You may obtain a copy of the License at |
9 | * |
10 | * http://www.apache.org/licenses/LICENSE-2.0 |
11 | * |
12 | * Unless required by applicable law or agreed to in writing, |
13 | * software distributed under the License is distributed on an |
14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
15 | * KIND, either express or implied. See the License for the |
16 | * specific language governing permissions and limitations |
17 | * under the License. |
18 | */ |
19 | |
20 | /*! |
21 | * \file llvm_module.cc |
22 | * \brief LLVM runtime module for TVM |
23 | */ |
24 | #ifdef TVM_LLVM_VERSION |
25 | |
26 | #include "llvm_module.h" |
27 | |
28 | #include <dmlc/io.h> |
29 | #include <llvm/ADT/SmallString.h> |
30 | #include <llvm/ADT/StringRef.h> |
31 | #include <llvm/Bitcode/BitcodeWriter.h> |
32 | #include <llvm/ExecutionEngine/ExecutionEngine.h> |
33 | #include <llvm/ExecutionEngine/MCJIT.h> // Force linking of MCJIT |
34 | #include <llvm/IR/DataLayout.h> |
35 | #include <llvm/IR/Function.h> |
36 | #include <llvm/IR/Intrinsics.h> |
37 | #include <llvm/IR/LLVMContext.h> |
38 | #include <llvm/IR/LegacyPassManager.h> |
39 | #include <llvm/IR/MDBuilder.h> |
40 | #include <llvm/IR/Metadata.h> |
41 | #include <llvm/IR/Module.h> |
42 | #include <llvm/IR/Verifier.h> |
43 | #include <llvm/IRReader/IRReader.h> |
44 | #include <llvm/Support/FileSystem.h> |
45 | #include <llvm/Support/SourceMgr.h> |
46 | #include <llvm/Support/raw_ostream.h> |
47 | #include <llvm/Target/TargetMachine.h> |
48 | #include <llvm/Target/TargetOptions.h> |
49 | #include <llvm/Transforms/Utils/Cloning.h> |
50 | #include <tvm/ir/module.h> |
51 | #include <tvm/relay/runtime.h> |
52 | #include <tvm/runtime/container/array.h> |
53 | #include <tvm/runtime/container/string.h> |
54 | #include <tvm/runtime/logging.h> |
55 | #include <tvm/runtime/metadata.h> |
56 | #include <tvm/runtime/module.h> |
57 | #include <tvm/runtime/object.h> |
58 | #include <tvm/runtime/packed_func.h> |
59 | #include <tvm/runtime/registry.h> |
60 | #include <tvm/support/with.h> |
61 | #include <tvm/target/codegen.h> |
62 | #include <tvm/target/target.h> |
63 | |
64 | #include <algorithm> |
65 | #include <memory> |
66 | #include <mutex> |
67 | #include <sstream> |
68 | #include <string> |
69 | #include <system_error> |
70 | #include <utility> |
71 | #include <vector> |
72 | |
73 | #include "../../runtime/file_utils.h" |
74 | #include "../../runtime/library_module.h" |
75 | #include "../func_registry_generator.h" |
76 | #include "codegen_blob.h" |
77 | #include "codegen_cpu.h" |
78 | #include "codegen_llvm.h" |
79 | #include "llvm_instance.h" |
80 | |
81 | namespace tvm { |
82 | namespace codegen { |
83 | |
84 | using runtime::PackedFunc; |
85 | using runtime::TVMArgs; |
86 | using runtime::TVMRetValue; |
87 | |
88 | class LLVMModuleNode final : public runtime::ModuleNode { |
89 | public: |
90 | ~LLVMModuleNode(); |
91 | |
92 | const char* type_key() const final { return "llvm" ; } |
93 | |
94 | PackedFunc GetFunction(const std::string& name, const ObjectPtr<Object>& sptr_to_self) final; |
95 | |
96 | void SaveToFile(const std::string& file_name, const std::string& format) final; |
97 | void SaveToBinary(dmlc::Stream* stream) final; |
98 | std::string GetSource(const std::string& format) final; |
99 | |
100 | void Init(const IRModule& mod, const Target& target); |
101 | void Init(std::unique_ptr<llvm::Module> module, std::unique_ptr<LLVMInstance> llvm_instance); |
102 | void LoadIR(const std::string& file_name); |
103 | bool IsDSOExportable() const final { return true; } |
104 | |
105 | bool ImplementsFunction(const String& name, bool query_imports) final; |
106 | |
107 | private: |
108 | void LazyInitJIT(); |
109 | bool IsCompatibleWithHost(const llvm::TargetMachine* tm) const; |
110 | void* GetGlobalAddr(const std::string& name, const LLVMTarget& llvm_target) const; |
111 | void* GetFunctionAddr(const std::string& name, const LLVMTarget& llvm_target) const; |
112 | |
113 | // The LLVM scope object. |
114 | std::unique_ptr<LLVMInstance> llvm_instance_; |
115 | // JIT lock |
116 | std::mutex mutex_; |
117 | // execution engine |
118 | llvm::ExecutionEngine* ee_{nullptr}; |
119 | // The raw pointer to the module. |
120 | llvm::Module* module_{nullptr}; |
121 | // The unique_ptr owning the module. This becomes empty once JIT has been initialized |
122 | // (EngineBuilder takes ownership of the module). |
123 | std::unique_ptr<llvm::Module> module_owning_ptr_; |
124 | /* \brief names of the functions declared in this module */ |
125 | Array<String> function_names_; |
126 | }; |
127 | |
128 | LLVMModuleNode::~LLVMModuleNode() { |
129 | if (ee_ != nullptr) { |
130 | ee_->runStaticConstructorsDestructors(true); |
131 | delete ee_; |
132 | } |
133 | module_owning_ptr_.reset(); |
134 | } |
135 | |
136 | PackedFunc LLVMModuleNode::GetFunction(const std::string& name, |
137 | const ObjectPtr<Object>& sptr_to_self) { |
138 | if (name == "__tvm_is_system_module" ) { |
139 | bool flag = (module_->getFunction("__tvm_module_startup" ) != nullptr); |
140 | return PackedFunc([flag](TVMArgs args, TVMRetValue* rv) { *rv = flag; }); |
141 | } else if (name == "get_func_names" ) { |
142 | return PackedFunc( |
143 | [sptr_to_self, this](TVMArgs args, TVMRetValue* rv) { *rv = this->function_names_; }); |
144 | } else if (name == "get_symbol" ) { |
145 | return PackedFunc(nullptr); |
146 | } else if (name == "get_const_vars" ) { |
147 | return PackedFunc(nullptr); |
148 | } else if (name == "_get_target_string" ) { |
149 | std::string target_string = LLVMTarget::GetTargetMetadata(*module_); |
150 | return PackedFunc([target_string](TVMArgs args, TVMRetValue* rv) { *rv = target_string; }); |
151 | } |
152 | if (ee_ == nullptr) LazyInitJIT(); |
153 | |
154 | std::lock_guard<std::mutex> lock(mutex_); |
155 | |
156 | TVMBackendPackedCFunc faddr; |
157 | With<LLVMTarget> llvm_target(*llvm_instance_, LLVMTarget::GetTargetMetadata(*module_)); |
158 | if (name == runtime::symbol::tvm_module_main) { |
159 | const char* entry_name = reinterpret_cast<const char*>( |
160 | GetGlobalAddr(runtime::symbol::tvm_module_main, *llvm_target)); |
161 | ICHECK(entry_name != nullptr) << "Symbol " << runtime::symbol::tvm_module_main |
162 | << " is not presented" ; |
163 | faddr = reinterpret_cast<TVMBackendPackedCFunc>(GetFunctionAddr(entry_name, *llvm_target)); |
164 | } else { |
165 | faddr = reinterpret_cast<TVMBackendPackedCFunc>(GetFunctionAddr(name, *llvm_target)); |
166 | } |
167 | if (faddr == nullptr) return PackedFunc(); |
168 | return WrapPackedFunc(faddr, sptr_to_self); |
169 | } |
170 | |
171 | void LLVMModuleNode::SaveToFile(const std::string& file_name, const std::string& format) { |
172 | std::string fmt = runtime::GetFileFormat(file_name, format); |
173 | std::error_code ecode; |
174 | #if TVM_LLVM_VERSION <= 70 |
175 | llvm::raw_fd_ostream dest(file_name, ecode, llvm::sys::fs::F_None); |
176 | #else |
177 | llvm::raw_fd_ostream dest(file_name, ecode, llvm::sys::fs::OF_None); |
178 | #endif |
179 | ICHECK_EQ(ecode.value(), 0) << "Cannot open file: " << file_name << " " << ecode.message(); |
180 | if (fmt == "o" || fmt == "obj" ) { |
181 | With<LLVMTarget> llvm_target(*llvm_instance_, LLVMTarget::GetTargetMetadata(*module_)); |
182 | #if TVM_LLVM_VERSION <= 60 |
183 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(module_); |
184 | #else |
185 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(*module_); |
186 | #endif |
187 | llvm::legacy::PassManager pass; |
188 | llvm::TargetMachine* tm = llvm_target->GetOrCreateTargetMachine(); |
189 | #if TVM_LLVM_VERSION <= 60 |
190 | ICHECK(tm->addPassesToEmitFile(pass, dest, llvm::TargetMachine::CGFT_ObjectFile) == 0) |
191 | << "Cannot emit target CGFT_ObjectFile" ; |
192 | #elif TVM_LLVM_VERSION <= 90 |
193 | ICHECK(tm->addPassesToEmitFile(pass, dest, nullptr, llvm::TargetMachine::CGFT_ObjectFile) == 0) |
194 | << "Cannot emit target CGFT_ObjectFile" ; |
195 | #else |
196 | ICHECK(tm->addPassesToEmitFile(pass, dest, nullptr, llvm::CGFT_ObjectFile) == 0) |
197 | << "Cannot emit target CGFT_ObjectFile" ; |
198 | #endif |
199 | pass.run(*m); |
200 | } else if (fmt == "s" || fmt == "asm" ) { |
201 | With<LLVMTarget> llvm_target(*llvm_instance_, LLVMTarget::GetTargetMetadata(*module_)); |
202 | #if TVM_LLVM_VERSION <= 60 |
203 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(module_); |
204 | #else |
205 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(*module_); |
206 | #endif |
207 | llvm::legacy::PassManager pass; |
208 | llvm::TargetMachine* tm = llvm_target->GetOrCreateTargetMachine(); |
209 | #if TVM_LLVM_VERSION <= 60 |
210 | ICHECK(tm->addPassesToEmitFile(pass, dest, llvm::TargetMachine::CGFT_AssemblyFile) == 0) |
211 | << "Cannot emit target CGFT_AssemblyFile" ; |
212 | #elif TVM_LLVM_VERSION <= 90 |
213 | ICHECK(tm->addPassesToEmitFile(pass, dest, nullptr, llvm::TargetMachine::CGFT_AssemblyFile) == |
214 | 0) |
215 | << "Cannot emit target CGFT_AssemblyFile" ; |
216 | #else |
217 | ICHECK(tm->addPassesToEmitFile(pass, dest, nullptr, llvm::CGFT_AssemblyFile) == 0) |
218 | << "Cannot emit target CGFT_AssemblyFile" ; |
219 | #endif |
220 | pass.run(*m); |
221 | } else if (fmt == "ll" ) { |
222 | module_->print(dest, nullptr); |
223 | } else if (fmt == "bc" ) { |
224 | #if TVM_LLVM_VERSION <= 60 |
225 | llvm::WriteBitcodeToFile(module_, dest); |
226 | #else |
227 | llvm::WriteBitcodeToFile(*module_, dest); |
228 | #endif |
229 | } else { |
230 | LOG(FATAL) << "Do not know how to save file " << file_name << " with format=\'" << format |
231 | << "\'" ; |
232 | } |
233 | dest.close(); |
234 | } |
235 | |
236 | void LLVMModuleNode::SaveToBinary(dmlc::Stream* stream) { |
237 | LOG(FATAL) << "LLVMModule: SaveToBinary not supported" ; |
238 | } |
239 | |
240 | std::string LLVMModuleNode::GetSource(const std::string& format) { |
241 | std::string fmt = runtime::GetFileFormat("" , format); |
242 | std::string type_str; |
243 | llvm::SmallString<256> str; |
244 | llvm::raw_svector_ostream rso(str); |
245 | |
246 | if (fmt == "s" || fmt == "asm" ) { |
247 | With<LLVMTarget> llvm_target(*llvm_instance_, LLVMTarget::GetTargetMetadata(*module_)); |
248 | #if TVM_LLVM_VERSION <= 60 |
249 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(module_); |
250 | #else |
251 | std::unique_ptr<llvm::Module> m = llvm::CloneModule(*module_); |
252 | #endif |
253 | llvm::legacy::PassManager pass; |
254 | llvm::TargetMachine* tm = llvm_target->GetOrCreateTargetMachine(); |
255 | #if TVM_LLVM_VERSION <= 60 |
256 | ICHECK(tm->addPassesToEmitFile(pass, rso, llvm::TargetMachine::CGFT_AssemblyFile) == 0) |
257 | << "Cannot emit target CGFT_AssemblyFile" ; |
258 | #elif TVM_LLVM_VERSION <= 90 |
259 | ICHECK(tm->addPassesToEmitFile(pass, rso, nullptr, llvm::TargetMachine::CGFT_AssemblyFile) == 0) |
260 | << "Cannot emit target CGFT_AssemblyFile" ; |
261 | #else |
262 | ICHECK(tm->addPassesToEmitFile(pass, rso, nullptr, llvm::CGFT_AssemblyFile) == 0) |
263 | << "Cannot emit target CGFT_AssemblyFile" ; |
264 | #endif |
265 | pass.run(*m); |
266 | return rso.str().str(); |
267 | } else if (fmt == "" || fmt == "ll" ) { |
268 | std::string type_str; |
269 | llvm::raw_string_ostream rso(type_str); |
270 | ICHECK(module_ != nullptr); |
271 | module_->print(rso, nullptr); |
272 | return rso.str(); |
273 | } else { |
274 | LOG(FATAL) << "Do not know how to get source code with format: " << format << "\'" ; |
275 | } |
276 | return "" ; |
277 | } |
278 | |
279 | void LLVMModuleNode::Init(const IRModule& mod, const Target& target) { |
280 | llvm_instance_ = std::make_unique<LLVMInstance>(); |
281 | With<LLVMTarget> llvm_target(*llvm_instance_, target); |
282 | llvm::TargetMachine* tm = llvm_target->GetOrCreateTargetMachine(); |
283 | std::unique_ptr<CodeGenLLVM> cg = CodeGenLLVM::Create(llvm_target.get()); |
284 | |
285 | std::vector<PrimFunc> funcs; |
286 | std::string entry_func; |
287 | relay::Runtime runtime = |
288 | mod->GetAttr<relay::Runtime>(tvm::attr::kRuntime).value_or(relay::Runtime::Create("cpp" )); |
289 | bool system_lib = runtime->GetAttr<Bool>("system-lib" ).value_or(Bool(false)); |
290 | bool target_c_runtime = runtime->name == "crt" ; |
291 | |
292 | for (auto kv : mod->functions) { |
293 | if (!kv.second->IsInstance<PrimFuncNode>()) { |
294 | // (@jroesch): we relax constraints here, Relay functions will just be ignored. |
295 | DLOG(INFO) << "Can only lower IR Module with PrimFuncs, but got " << kv.second->GetTypeKey(); |
296 | continue; |
297 | } |
298 | auto f = Downcast<PrimFunc>(kv.second); |
299 | auto global_symbol = f->GetAttr<String>(tvm::attr::kGlobalSymbol); |
300 | ICHECK(global_symbol.defined()); |
301 | function_names_.push_back(global_symbol.value()); |
302 | if (f->HasNonzeroAttr(tir::attr::kIsEntryFunc)) { |
303 | entry_func = global_symbol.value(); |
304 | } |
305 | funcs.push_back(f); |
306 | } |
307 | // TODO(@jroesch): follow up on this condition. |
308 | // ICHECK(funcs.size() > 0); |
309 | // TODO(tqchen): remove the entry function behavior as it does not |
310 | // makes sense when we start to use multiple modules. |
311 | cg->Init("TVMMod" , llvm_target.get(), system_lib, system_lib, target_c_runtime); |
312 | cg->SetFastMathFlags(llvm_target->GetFastMathFlags()); |
313 | |
314 | cg->AddFunctionsOrdered(funcs.begin(), funcs.end()); |
315 | if (entry_func.length() != 0) { |
316 | cg->AddMainFunction(entry_func); |
317 | } |
318 | |
319 | module_owning_ptr_ = cg->Finish(); |
320 | module_ = module_owning_ptr_.get(); |
321 | llvm_target->SetTargetMetadata(module_); |
322 | module_->addModuleFlag(llvm::Module::Override, "Debug Info Version" , |
323 | llvm::DEBUG_METADATA_VERSION); |
324 | |
325 | if (tm->getTargetTriple().isOSDarwin()) { |
326 | module_->addModuleFlag(llvm::Module::Override, "Dwarf Version" , 2); |
327 | } |
328 | |
329 | std::string verify_errors_storage; |
330 | llvm::raw_string_ostream verify_errors(verify_errors_storage); |
331 | LOG_IF(FATAL, llvm::verifyModule(*module_, &verify_errors)) |
332 | << "LLVM module verification failed with the following errors: \n" |
333 | << verify_errors.str(); |
334 | } |
335 | |
336 | void LLVMModuleNode::Init(std::unique_ptr<llvm::Module> module, |
337 | std::unique_ptr<LLVMInstance> llvm_instance) { |
338 | module_owning_ptr_ = std::move(module); |
339 | module_ = module_owning_ptr_.get(); |
340 | llvm_instance_ = std::move(llvm_instance); |
341 | } |
342 | |
343 | void LLVMModuleNode::LoadIR(const std::string& file_name) { |
344 | auto llvm_instance = std::make_unique<LLVMInstance>(); |
345 | std::unique_ptr<llvm::Module> module = llvm_instance->LoadIR(file_name); |
346 | Init(std::move(module), std::move(llvm_instance)); |
347 | } |
348 | |
349 | bool LLVMModuleNode::ImplementsFunction(const String& name, bool query_imports) { |
350 | return std::find(function_names_.begin(), function_names_.end(), name) != function_names_.end(); |
351 | } |
352 | |
353 | void LLVMModuleNode::LazyInitJIT() { |
354 | std::lock_guard<std::mutex> lock(mutex_); |
355 | if (ee_) { |
356 | return; |
357 | } |
358 | With<LLVMTarget> llvm_target(*llvm_instance_, LLVMTarget::GetTargetMetadata(*module_)); |
359 | llvm::EngineBuilder builder(std::move(module_owning_ptr_)); |
360 | builder.setEngineKind(llvm::EngineKind::JIT); |
361 | builder.setOptLevel(llvm::CodeGenOpt::Aggressive); |
362 | builder.setMCPU(llvm_target->GetCPU()); |
363 | builder.setMAttrs(llvm_target->GetTargetFeatures()); |
364 | builder.setTargetOptions(llvm_target->GetTargetOptions()); |
365 | auto tm = std::unique_ptr<llvm::TargetMachine>(builder.selectTarget()); |
366 | if (!IsCompatibleWithHost(tm.get())) { |
367 | LOG(FATAL) << "Cannot run module, architecture mismatch" ; |
368 | } |
369 | llvm::DataLayout layout(tm->createDataLayout()); |
370 | ICHECK(layout == module_->getDataLayout()) |
371 | << "Data layout mismatch between module(" |
372 | << module_->getDataLayout().getStringRepresentation() << ")" |
373 | << " and ExecutionEngine (" << layout.getStringRepresentation() << ")" ; |
374 | ee_ = builder.create(tm.release()); |
375 | ICHECK(ee_ != nullptr) << "Failed to initialize jit engine for " << module_->getTargetTriple(); |
376 | ee_->runStaticConstructorsDestructors(false); |
377 | |
378 | if (void** ctx_addr = |
379 | reinterpret_cast<void**>(GetGlobalAddr(runtime::symbol::tvm_module_ctx, *llvm_target))) { |
380 | *ctx_addr = this; |
381 | } |
382 | runtime::InitContextFunctions( |
383 | [this, &llvm_target](const char* name) { return GetGlobalAddr(name, *llvm_target); }); |
384 | // There is a problem when a JITed function contains a call to a runtime function. |
385 | // The runtime function (e.g. __truncsfhf2) may not be resolved, and calling it will |
386 | // lead to a runtime crash. |
387 | // Do name lookup on a symbol that doesn't exist. This will force MCJIT to finalize |
388 | // all loaded objects, which will resolve symbols in JITed code. |
389 | ee_->getFunctionAddress("__some_name_that_hopefully_doesnt_exist__b49f8aaade5877eaba7583b91" ); |
390 | } |
391 | |
392 | bool LLVMModuleNode::IsCompatibleWithHost(const llvm::TargetMachine* tm) const { |
393 | LLVMTargetInfo host_target(*llvm_instance_, "llvm" ); |
394 | auto tm_host = host_target.GetOrCreateTargetMachine(); |
395 | if (tm_host->getTargetTriple().getArch() != tm->getTargetTriple().getArch()) { |
396 | LOG(INFO) << "Architecture mismatch: module=" << tm->getTargetTriple().str() |
397 | << " host=" << tm_host->getTargetTriple().str(); |
398 | return false; |
399 | } |
400 | return true; |
401 | } |
402 | |
403 | // Get global address from execution engine. |
404 | void* LLVMModuleNode::GetGlobalAddr(const std::string& name, const LLVMTarget& llvm_target) const { |
405 | // first verifies if GV exists. |
406 | if (module_->getGlobalVariable(name) != nullptr) { |
407 | return reinterpret_cast<void*>(ee_->getGlobalValueAddress(name)); |
408 | } else { |
409 | return nullptr; |
410 | } |
411 | } |
412 | |
413 | void* LLVMModuleNode::GetFunctionAddr(const std::string& name, |
414 | const LLVMTarget& llvm_target) const { |
415 | // first verifies if GV exists. |
416 | if (module_->getFunction(name) != nullptr) { |
417 | return reinterpret_cast<void*>(ee_->getFunctionAddress(name)); |
418 | } else { |
419 | return nullptr; |
420 | } |
421 | } |
422 | |
423 | TVM_REGISTER_GLOBAL("target.build.llvm" ) |
424 | .set_body_typed([](IRModule mod, Target target) -> runtime::Module { |
425 | auto n = make_object<LLVMModuleNode>(); |
426 | n->Init(mod, target); |
427 | return runtime::Module(n); |
428 | }); |
429 | |
430 | TVM_REGISTER_GLOBAL("codegen.LLVMModuleCreate" ) |
431 | .set_body_typed([](std::string target_str, std::string module_name) -> runtime::Module { |
432 | auto llvm_instance = std::make_unique<LLVMInstance>(); |
433 | With<LLVMTarget> llvm_target(*llvm_instance, target_str); |
434 | auto n = make_object<LLVMModuleNode>(); |
435 | // Generate a LLVM module from an input target string |
436 | auto module = std::make_unique<llvm::Module>(module_name, *llvm_target->GetContext()); |
437 | llvm_target->SetTargetMetadata(module.get()); |
438 | module->setTargetTriple(llvm_target->GetTargetTriple()); |
439 | module->setDataLayout(llvm_target->GetOrCreateTargetMachine()->createDataLayout()); |
440 | n->Init(std::move(module), std::move(llvm_instance)); |
441 | return runtime::Module(n); |
442 | }); |
443 | |
444 | TVM_REGISTER_GLOBAL("target.llvm_lookup_intrinsic_id" ) |
445 | .set_body_typed([](std::string name) -> int64_t { |
446 | return static_cast<int64_t>(llvm::Function::lookupIntrinsicID(name)); |
447 | }); |
448 | |
449 | TVM_REGISTER_GLOBAL("target.llvm_get_intrinsic_name" ).set_body_typed([](int64_t id) -> String { |
450 | #if TVM_LLVM_VERSION >= 130 |
451 | return std::string(llvm::Intrinsic::getBaseName(static_cast<llvm::Intrinsic::ID>(id))); |
452 | #elif TVM_LLVM_VERSION >= 40 |
453 | // This is the version of Intrinsic::getName that works for overloaded |
454 | // intrinsics. Helpfully, if we provide no types to this function, it |
455 | // will give us the overloaded name without the types appended. This |
456 | // should be enough information for most uses. |
457 | return std::string(llvm::Intrinsic::getName(static_cast<llvm::Intrinsic::ID>(id), {})); |
458 | #else |
459 | // Nothing to do, just return the intrinsic id number |
460 | return std::to_string(id); |
461 | #endif |
462 | }); |
463 | |
464 | TVM_REGISTER_GLOBAL("target.llvm_version_major" ).set_body_typed([]() -> int { |
465 | return TVM_LLVM_VERSION / 10; |
466 | }); |
467 | |
468 | TVM_REGISTER_GLOBAL("runtime.module.loadfile_ll" ) |
469 | .set_body_typed([](std::string filename, std::string fmt) -> runtime::Module { |
470 | auto n = make_object<LLVMModuleNode>(); |
471 | n->LoadIR(filename); |
472 | return runtime::Module(n); |
473 | }); |
474 | |
475 | TVM_REGISTER_GLOBAL("codegen.llvm_target_enabled" ) |
476 | .set_body_typed([](std::string target_str) -> bool { |
477 | LLVMInstance llvm_instance; |
478 | auto* tm = With<LLVMTarget>(llvm_instance, target_str) |
479 | ->GetOrCreateTargetMachine(/*allow_missing=*/true); |
480 | return tm != nullptr; |
481 | }); |
482 | |
483 | TVM_REGISTER_GLOBAL("codegen.codegen_blob" ) |
484 | .set_body_typed([](std::string data, bool system_lib, |
485 | std::string llvm_target_string) -> runtime::Module { |
486 | auto n = make_object<LLVMModuleNode>(); |
487 | auto llvm_instance = std::make_unique<LLVMInstance>(); |
488 | With<LLVMTarget> llvm_target(*llvm_instance, llvm_target_string); |
489 | std::unique_ptr<llvm::Module> blob = CodeGenBlob(data, system_lib, llvm_target.get()); |
490 | n->Init(std::move(blob), std::move(llvm_instance)); |
491 | return runtime::Module(n); |
492 | }); |
493 | |
494 | runtime::Module CreateLLVMCppMetadataModule(runtime::metadata::Metadata metadata, Target target, |
495 | tvm::relay::Runtime runtime) { |
496 | auto llvm_instance = std::make_unique<LLVMInstance>(); |
497 | With<LLVMTarget> llvm_target(*llvm_instance, target); |
498 | bool system_lib = runtime->GetAttr<Bool>("system-lib" ).value_or(Bool(false)); |
499 | auto cg = std::make_unique<CodeGenCPU>(); |
500 | |
501 | cg->Init("TVMMetadataMod" , llvm_target.get(), system_lib, system_lib, |
502 | /*target_c_runtime=*/false); |
503 | |
504 | cg->DefineMetadata(metadata); |
505 | auto mod = cg->Finish(); |
506 | llvm_target->SetTargetMetadata(mod.get()); |
507 | mod->addModuleFlag(llvm::Module::Override, "Debug Info Version" , llvm::DEBUG_METADATA_VERSION); |
508 | |
509 | if (llvm_target->GetOrCreateTargetMachine()->getTargetTriple().isOSDarwin()) { |
510 | mod->addModuleFlag(llvm::Module::Override, "Dwarf Version" , 2); |
511 | } |
512 | |
513 | std::string verify_errors_storage; |
514 | llvm::raw_string_ostream verify_errors(verify_errors_storage); |
515 | LOG_IF(FATAL, llvm::verifyModule(*mod, &verify_errors)) |
516 | << "LLVM module verification failed with the following errors: \n" |
517 | << verify_errors.str(); |
518 | |
519 | auto n = make_object<LLVMModuleNode>(); |
520 | n->Init(std::move(mod), std::move(llvm_instance)); |
521 | |
522 | auto meta_mod = MetadataModuleCreate(metadata); |
523 | meta_mod->Import(runtime::Module(n)); |
524 | return meta_mod; |
525 | } |
526 | |
527 | runtime::Module CreateLLVMCrtMetadataModule(const Array<runtime::Module>& modules, Target target, |
528 | tvm::relay::Runtime runtime) { |
529 | Array<String> func_names; |
530 | for (runtime::Module mod : modules) { |
531 | auto pf_funcs = mod.GetFunction("get_func_names" ); |
532 | if (pf_funcs != nullptr) { |
533 | Array<String> func_names_ = pf_funcs(); |
534 | for (const auto& fname : func_names_) { |
535 | func_names.push_back(fname); |
536 | } |
537 | } |
538 | } |
539 | |
540 | auto llvm_instance = std::make_unique<LLVMInstance>(); |
541 | With<LLVMTarget> llvm_target(*llvm_instance, target); |
542 | bool system_lib = runtime->GetAttr<Bool>("system-lib" ).value_or(Bool(false)); |
543 | bool target_c_runtime = runtime->name == "crt" ; |
544 | ICHECK(system_lib && target_c_runtime) |
545 | << "For LLVM C-runtime metadata module, must include --system-lib and --runtime=c; " |
546 | << "got target: " << target->str(); |
547 | auto cg = std::make_unique<CodeGenCPU>(); |
548 | cg->Init("TVMMetadataMod" , llvm_target.operator->(), system_lib, system_lib, target_c_runtime); |
549 | |
550 | cg->DefineFunctionRegistry(func_names); |
551 | auto mod = cg->Finish(); |
552 | llvm_target->SetTargetMetadata(mod.get()); |
553 | mod->addModuleFlag(llvm::Module::Override, "Debug Info Version" , llvm::DEBUG_METADATA_VERSION); |
554 | |
555 | if (llvm_target->GetOrCreateTargetMachine()->getTargetTriple().isOSDarwin()) { |
556 | mod->addModuleFlag(llvm::Module::Override, "Dwarf Version" , 2); |
557 | } |
558 | |
559 | std::string verify_errors_storage; |
560 | llvm::raw_string_ostream verify_errors(verify_errors_storage); |
561 | LOG_IF(FATAL, llvm::verifyModule(*mod, &verify_errors)) |
562 | << "LLVM module verification failed with the following errors: \n" |
563 | << verify_errors.str(); |
564 | |
565 | auto n = make_object<LLVMModuleNode>(); |
566 | n->Init(std::move(mod), std::move(llvm_instance)); |
567 | for (auto m : modules) { |
568 | n->Import(m); |
569 | } |
570 | return runtime::Module(n); |
571 | } |
572 | |
573 | TVM_REGISTER_GLOBAL("runtime.CreateLLVMCrtMetadataModule" ) |
574 | .set_body_typed(CreateLLVMCrtMetadataModule); |
575 | |
576 | } // namespace codegen |
577 | } // namespace tvm |
578 | |
579 | #endif // TVM_LLVM_VERSION |
580 | |