1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17/*
18 * This is a debugging tool that can be used to detect errors in a test
19 * backend. it works by loading a model from a protobuf file, compile this model
20 * on two backends; a reference backend (Interpreter) and a a test backend. It
21 * compares the results from running these networks on a layer by layer fashion
22 * to detect which layer generated wrong results.
23 *
24 * Sample run line:
25 * ./network-debuger --model function_0.zip --inputs input_0.onnx -backend=CPU
26 *
27 * The tool will print to screen when a faulty layer is detected. Input and
28 * output tensors for the layers will be printed from the reference network to
29 * disk.
30 *
31 */
32
33#include "glow/ExecutionEngine/ExecutionEngine.h"
34#include "glow/Exporter/ONNXModelWriter.h"
35#include "glow/Importer/ONNXModelLoader.h"
36#include "glow/Support/Debug.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/Signals.h"
39
40#include "NetworkComparator.h"
41#include "glow/Converter/Float16Converter.h"
42
43#include <string>
44
45using namespace glow;
46
47namespace {
48llvm::cl::OptionCategory debuggerTestCat("Debugger Category");
49
50llvm::cl::opt<std::string> modelPathOpt("model", llvm::cl::desc("Input models"),
51 llvm::cl::value_desc("modelPath"),
52 llvm::cl::Required,
53 llvm::cl::cat(debuggerTestCat));
54llvm::cl::list<std::string> inputsOpt("inputs", llvm::cl::desc("Inputs"),
55 llvm::cl::value_desc("Inputs"),
56 llvm::cl::Required, llvm::cl::OneOrMore,
57 llvm::cl::cat(debuggerTestCat));
58llvm::cl::opt<std::string>
59 testBackend("backend",
60 llvm::cl::desc("Backend to use, e.g. Interpreter, CPU, NNPI:"),
61 llvm::cl::init("Interpreter"), llvm::cl::cat(debuggerTestCat));
62
63llvm::cl::opt<std::string> comparatorType(
64 "comparator",
65 llvm::cl::desc("The type of comparator to use, Recursive or Intermediate"),
66 llvm::cl::init("Intermediate"), llvm::cl::cat(debuggerTestCat));
67
68llvm::cl::opt<float> numericCmpThreshold(
69 "threshold", llvm::cl::desc("Threshold for tensor numeric comparison"),
70 llvm::cl::Optional, llvm::cl::init(1e-5), llvm::cl::cat(debuggerTestCat));
71
72llvm::cl::opt<bool> dumpTensors(
73 "dump_tensors",
74 llvm::cl::desc("Dump input(s)\\output(s) of an errant layer to files."),
75 llvm::cl::Optional, llvm::cl::init(false), llvm::cl::cat(debuggerTestCat));
76
77llvm::cl::opt<bool>
78 globalFp16Opt("glow_global_fp16",
79 llvm::cl::desc("Enable fp16 lowering for all ops on the net"),
80 llvm::cl::Optional, llvm::cl::cat(debuggerTestCat));
81
82llvm::cl::opt<bool> fuseScaleOffsetFp16Opt(
83 "glow_global_fused_scale_offset_fp16",
84 llvm::cl::desc(
85 "Enable fp16 lowering for all op inputs using fused scale/offset"),
86 llvm::cl::Optional, llvm::cl::cat(debuggerTestCat));
87
88llvm::cl::opt<bool>
89 ClipFp16Opt("glow_clip_fp16",
90 llvm::cl::desc("Force glow to clip fp16 values to min/max"),
91 llvm::cl::Optional, llvm::cl::cat(debuggerTestCat));
92
93llvm::cl::opt<bool>
94 forceFP16AccumSLSOpt("glow_global_force_sls_fp16_accum",
95 llvm::cl::desc("Force FP16 accumulation for SLS ops"),
96 llvm::cl::Optional, llvm::cl::cat(debuggerTestCat));
97
98#define DEBUG_TYPE "verifier"
99
100void parseCommandLine(int argc, char **argv) {
101 llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
102 llvm::cl::ParseCommandLineOptions(
103 argc, argv,
104 " Network Debugger tool, part of the Glow compiler\n\n"
105 "Glow is a compiler for neural network accelerators.\n");
106}
107
108// Utility functions:
109void convertNetwork(Function *F) {
110 CompilationContext cctx;
111 PrecisionConfiguration precConfig;
112 if (globalFp16Opt) {
113 precConfig.convertToFP16 = globalFp16Opt;
114 }
115 if (fuseScaleOffsetFp16Opt) {
116 precConfig.convertFusedToFP16 = fuseScaleOffsetFp16Opt;
117 }
118 if (ClipFp16Opt) {
119 precConfig.clipFP16 = ClipFp16Opt;
120 }
121 if (forceFP16AccumSLSOpt) {
122 precConfig.forceFP16AccumSLS = true;
123 }
124 // Convert Network to fp16.
125 if (globalFp16Opt || fuseScaleOffsetFp16Opt || ClipFp16Opt ||
126 forceFP16AccumSLSOpt)
127 glow::convertFunctionToFloat16(F, precConfig);
128
129 // Optimize network after conversion to remove unneeded converts.
130 glow::optimize(F, CompilationMode::Infer);
131}
132
133/// Whether the ONNX loader loaded a model that was exporting with custom Glow
134/// ops. This should be in sync with exporting of inputs and so is saved for use
135/// with fillPlaceholders().
136bool usingGlowCustomOps = false;
137
138void loadModelIntoFunc(Function *F) {
139 Error err = Error::empty();
140 {
141 ONNXModelLoader onnxLD(modelPathOpt, {}, {}, *F, &err, /*zipMode*/ true);
142 usingGlowCustomOps = onnxLD.usingGlowCustomOps();
143 }
144 CHECK(!ERR_TO_BOOL(std::move(err)))
145 << "ONNXModelLoader failed to load model: " << modelPathOpt;
146 convertNetwork(F);
147}
148
149bool run() {
150 LOG(INFO) << "Comparing the " << testBackend
151 << " backend against the Interpreter "
152 "(reference backend)";
153 Module mod;
154 bool allPass = true;
155 loadModelIntoFunc(mod.createFunction("test"));
156 // Create a Comparator based on the type.
157 // NetworkComparatorBase *netCompare;
158 std::unique_ptr<NetworkComparatorBase> netCompare;
159 if (comparatorType == "comparatorType") {
160 netCompare.reset(new IntermediateLayerComparator(
161 mod, "Interpreter", testBackend, numericCmpThreshold, dumpTensors));
162 } else {
163 netCompare.reset(new RecursiveLayerComparator(
164 mod, "Interpreter", testBackend, numericCmpThreshold, dumpTensors));
165 }
166 PlaceholderBindings inputBindings;
167 inputBindings.allocate(mod.getPlaceholders());
168 for (size_t idx = 0; idx != inputsOpt.size(); idx++) {
169 fillPlaceholders(inputsOpt[idx], &inputBindings,
170 /* partialTensorPayloads */ nullptr, usingGlowCustomOps);
171 // Test the network with these inputs
172 allPass &= netCompare->verify(&inputBindings);
173 inputBindings.clear();
174 }
175 return allPass;
176}
177} // namespace
178
179int main(int argc, char **argv) {
180 parseCommandLine(argc, argv);
181 if (run()) {
182 LOG(INFO) << "All layers match with no errors\n";
183 return 0;
184 } else {
185 LOG(ERROR) << "Errors found!";
186 return -1;
187 }
188}
189