1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | |
17 | /* |
18 | * This is a debugging tool that can be used to detect errors in a test |
19 | * backend. it works by loading a model from a protobuf file, compile this model |
20 | * on two backends; a reference backend (Interpreter) and a a test backend. It |
21 | * compares the results from running these networks on a layer by layer fashion |
22 | * to detect which layer generated wrong results. |
23 | * |
24 | * Sample run line: |
25 | * ./network-debuger --model function_0.zip --inputs input_0.onnx -backend=CPU |
26 | * |
27 | * The tool will print to screen when a faulty layer is detected. Input and |
28 | * output tensors for the layers will be printed from the reference network to |
29 | * disk. |
30 | * |
31 | */ |
32 | |
33 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
34 | #include "glow/Exporter/ONNXModelWriter.h" |
35 | #include "glow/Importer/ONNXModelLoader.h" |
36 | #include "glow/Support/Debug.h" |
37 | #include "llvm/Support/CommandLine.h" |
38 | #include "llvm/Support/Signals.h" |
39 | |
40 | #include "NetworkComparator.h" |
41 | #include "glow/Converter/Float16Converter.h" |
42 | |
43 | #include <string> |
44 | |
45 | using namespace glow; |
46 | |
47 | namespace { |
48 | llvm::cl::OptionCategory debuggerTestCat("Debugger Category" ); |
49 | |
50 | llvm::cl::opt<std::string> modelPathOpt("model" , llvm::cl::desc("Input models" ), |
51 | llvm::cl::value_desc("modelPath" ), |
52 | llvm::cl::Required, |
53 | llvm::cl::cat(debuggerTestCat)); |
54 | llvm::cl::list<std::string> inputsOpt("inputs" , llvm::cl::desc("Inputs" ), |
55 | llvm::cl::value_desc("Inputs" ), |
56 | llvm::cl::Required, llvm::cl::OneOrMore, |
57 | llvm::cl::cat(debuggerTestCat)); |
58 | llvm::cl::opt<std::string> |
59 | testBackend("backend" , |
60 | llvm::cl::desc("Backend to use, e.g. Interpreter, CPU, NNPI:" ), |
61 | llvm::cl::init("Interpreter" ), llvm::cl::cat(debuggerTestCat)); |
62 | |
63 | llvm::cl::opt<std::string> comparatorType( |
64 | "comparator" , |
65 | llvm::cl::desc("The type of comparator to use, Recursive or Intermediate" ), |
66 | llvm::cl::init("Intermediate" ), llvm::cl::cat(debuggerTestCat)); |
67 | |
68 | llvm::cl::opt<float> numericCmpThreshold( |
69 | "threshold" , llvm::cl::desc("Threshold for tensor numeric comparison" ), |
70 | llvm::cl::Optional, llvm::cl::init(1e-5), llvm::cl::cat(debuggerTestCat)); |
71 | |
72 | llvm::cl::opt<bool> dumpTensors( |
73 | "dump_tensors" , |
74 | llvm::cl::desc("Dump input(s)\\output(s) of an errant layer to files." ), |
75 | llvm::cl::Optional, llvm::cl::init(false), llvm::cl::cat(debuggerTestCat)); |
76 | |
77 | llvm::cl::opt<bool> |
78 | globalFp16Opt("glow_global_fp16" , |
79 | llvm::cl::desc("Enable fp16 lowering for all ops on the net" ), |
80 | llvm::cl::Optional, llvm::cl::cat(debuggerTestCat)); |
81 | |
82 | llvm::cl::opt<bool> fuseScaleOffsetFp16Opt( |
83 | "glow_global_fused_scale_offset_fp16" , |
84 | llvm::cl::desc( |
85 | "Enable fp16 lowering for all op inputs using fused scale/offset" ), |
86 | llvm::cl::Optional, llvm::cl::cat(debuggerTestCat)); |
87 | |
88 | llvm::cl::opt<bool> |
89 | ClipFp16Opt("glow_clip_fp16" , |
90 | llvm::cl::desc("Force glow to clip fp16 values to min/max" ), |
91 | llvm::cl::Optional, llvm::cl::cat(debuggerTestCat)); |
92 | |
93 | llvm::cl::opt<bool> |
94 | forceFP16AccumSLSOpt("glow_global_force_sls_fp16_accum" , |
95 | llvm::cl::desc("Force FP16 accumulation for SLS ops" ), |
96 | llvm::cl::Optional, llvm::cl::cat(debuggerTestCat)); |
97 | |
98 | #define DEBUG_TYPE "verifier" |
99 | |
100 | void parseCommandLine(int argc, char **argv) { |
101 | llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); |
102 | llvm::cl::ParseCommandLineOptions( |
103 | argc, argv, |
104 | " Network Debugger tool, part of the Glow compiler\n\n" |
105 | "Glow is a compiler for neural network accelerators.\n" ); |
106 | } |
107 | |
108 | // Utility functions: |
109 | void convertNetwork(Function *F) { |
110 | CompilationContext cctx; |
111 | PrecisionConfiguration precConfig; |
112 | if (globalFp16Opt) { |
113 | precConfig.convertToFP16 = globalFp16Opt; |
114 | } |
115 | if (fuseScaleOffsetFp16Opt) { |
116 | precConfig.convertFusedToFP16 = fuseScaleOffsetFp16Opt; |
117 | } |
118 | if (ClipFp16Opt) { |
119 | precConfig.clipFP16 = ClipFp16Opt; |
120 | } |
121 | if (forceFP16AccumSLSOpt) { |
122 | precConfig.forceFP16AccumSLS = true; |
123 | } |
124 | // Convert Network to fp16. |
125 | if (globalFp16Opt || fuseScaleOffsetFp16Opt || ClipFp16Opt || |
126 | forceFP16AccumSLSOpt) |
127 | glow::convertFunctionToFloat16(F, precConfig); |
128 | |
129 | // Optimize network after conversion to remove unneeded converts. |
130 | glow::optimize(F, CompilationMode::Infer); |
131 | } |
132 | |
133 | /// Whether the ONNX loader loaded a model that was exporting with custom Glow |
134 | /// ops. This should be in sync with exporting of inputs and so is saved for use |
135 | /// with fillPlaceholders(). |
136 | bool usingGlowCustomOps = false; |
137 | |
138 | void loadModelIntoFunc(Function *F) { |
139 | Error err = Error::empty(); |
140 | { |
141 | ONNXModelLoader onnxLD(modelPathOpt, {}, {}, *F, &err, /*zipMode*/ true); |
142 | usingGlowCustomOps = onnxLD.usingGlowCustomOps(); |
143 | } |
144 | CHECK(!ERR_TO_BOOL(std::move(err))) |
145 | << "ONNXModelLoader failed to load model: " << modelPathOpt; |
146 | convertNetwork(F); |
147 | } |
148 | |
149 | bool run() { |
150 | LOG(INFO) << "Comparing the " << testBackend |
151 | << " backend against the Interpreter " |
152 | "(reference backend)" ; |
153 | Module mod; |
154 | bool allPass = true; |
155 | loadModelIntoFunc(mod.createFunction("test" )); |
156 | // Create a Comparator based on the type. |
157 | // NetworkComparatorBase *netCompare; |
158 | std::unique_ptr<NetworkComparatorBase> netCompare; |
159 | if (comparatorType == "comparatorType" ) { |
160 | netCompare.reset(new IntermediateLayerComparator( |
161 | mod, "Interpreter" , testBackend, numericCmpThreshold, dumpTensors)); |
162 | } else { |
163 | netCompare.reset(new RecursiveLayerComparator( |
164 | mod, "Interpreter" , testBackend, numericCmpThreshold, dumpTensors)); |
165 | } |
166 | PlaceholderBindings inputBindings; |
167 | inputBindings.allocate(mod.getPlaceholders()); |
168 | for (size_t idx = 0; idx != inputsOpt.size(); idx++) { |
169 | fillPlaceholders(inputsOpt[idx], &inputBindings, |
170 | /* partialTensorPayloads */ nullptr, usingGlowCustomOps); |
171 | // Test the network with these inputs |
172 | allPass &= netCompare->verify(&inputBindings); |
173 | inputBindings.clear(); |
174 | } |
175 | return allPass; |
176 | } |
177 | } // namespace |
178 | |
179 | int main(int argc, char **argv) { |
180 | parseCommandLine(argc, argv); |
181 | if (run()) { |
182 | LOG(INFO) << "All layers match with no errors\n" ; |
183 | return 0; |
184 | } else { |
185 | LOG(ERROR) << "Errors found!" ; |
186 | return -1; |
187 | } |
188 | } |
189 | |