1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16#ifndef GLOW_GRAPH_NETWORKCOMPARATOR_H
17#define GLOW_GRAPH_NETWORKCOMPARATOR_H
18
19#include "glow/ExecutionEngine/ExecutionEngine.h"
20#include "glow/Graph/Graph.h"
21#include "glow/Graph/PlaceholderBindings.h"
22#include "llvm/ADT/StringRef.h"
23#include <list>
24#include <string>
25#include <unordered_map>
26#include <vector>
27
28namespace glow {
29
30class Function;
31class Node;
32class Placeholder;
33class SaveNode;
34class Tensor;
35class Module;
36
37/// Base class for building a comparator that takes an input network \p
38/// inputModule_ and run it on two backends; a test backend and a reference
39/// backend and saves the list of layers that generate wrong results on the test
40/// net in \p brokenLayers_ . This class has a pure virtual function (verify)
41/// that must be implemented by a subclass.
42class NetworkComparatorBase {
43public:
44 /// Struct to save the list of input and output tensors of a node\layer.
45 struct InOutTensors {
46 std::unordered_map<std::string, Tensor *> inputs;
47 std::unordered_map<std::string, Tensor *> outputs;
48 };
49
50protected:
51 /// Execution Engine to run the Network tested on the reference backend.
52 ExecutionEngine EERefNet_;
53 /// Execution Engine to run the Network tested on the test backend.
54 ExecutionEngine EETestNet_;
55 /// Stores layer names found to be broken on the test backend.
56 std::vector<std::string> brokenLayers_;
57 /// Threshold of numerical comparison for tensors.
58 float numericCmpThreshold_;
59 /// Pointer to the module being tested.
60 Module *inputModule_;
61
62 /// Dump input\output tensors of a bad layer.
63 bool dumpTensorsForBadLayer_;
64 /// Prints out the input or output \p tensors associated with a \p layerName
65 /// the output file is a concatenation of the \p prefix and the \p layerName.
66 virtual void dumpTensors(std::unordered_map<std::string, Tensor *> tensors,
67 const std::string &layerName,
68 const std::string &prefix);
69 /// Compares tensors in \p refOuts with tensors in \p checkOuts using the
70 /// isEqual Tensor method.
71 bool checkTensors(std::unordered_map<std::string, Tensor *> &refOuts,
72 std::unordered_map<std::string, Tensor *> &checkOuts);
73
74public:
75 /// Constructor for the base class tester that tests the network passed in
76 /// \p mod on the \p testBackend using the \p referenceBackend as a reference
77 /// backend. \p numericCmpThreshold is accepted error threshold.
78 /// Inputs\outputs of a detected bad layer are dumped if \p
79 /// dumpTensorsForBadLayer is set.
80 NetworkComparatorBase(Module &mod, const std::string &referenceBackend,
81 const std::string &testBackend,
82 float numericCmpThreshold, bool dumpTensorsForBadLayer);
83 virtual ~NetworkComparatorBase() {}
84 /// Test the network with the inputs passed in \p bindings. The function
85 /// is pure virtual to be implemented by the underlying strategy in sub
86 /// classes. \returns True if all checks passed.
87 virtual bool verify(PlaceholderBindings *bindings) = 0;
88};
89
90/// A comparator class that tests layers by creating a subnet of the original
91/// graph for every layer. The subnets are created by recursively visiting the
92/// inputs of the layer. The results or running the subnet are compared between
93/// the test and reference backend. The subnet might have more than one broken
94/// layer.
95class RecursiveLayerComparator : public NetworkComparatorBase {
96private:
97 PlaceholderBindings inferBindingsRef_;
98 PlaceholderBindings inferBindingsCheck_;
99 /// Takes a \p layerName, creates a subnet comprised of all the predecessor
100 /// nodes till the inputs. The outputs of this layer are saved in the
101 /// inferenceBindingsRef if \p isRef is true and in inferenceBindingsCheck if
102 /// false. The inputs are passed in \p bindings. The run saves the inputs to
103 /// the layer \p layerName.
104 NetworkComparatorBase::InOutTensors hookAndRun(llvm::StringRef layerName,
105 PlaceholderBindings *bindings,
106 bool isRef);
107
108public:
109 /// Constructor for the RecursiveLayerComparator tester that tests the network
110 /// passed in \p mod on the \p testBackend using the \p referenceBackend as a
111 /// reference backend. \p numericCmpThreshold_. Inputs\outputs of
112 /// a detected bad layer are dumped if \p dumpTensorsForBadLayer is set.
113 RecursiveLayerComparator(Module &mod, const std::string &referenceBackend,
114 const std::string &testBackend,
115 float numericCmpThreshold_,
116 bool dumpTensorsForBadLayer);
117 /// Takes the \p bindings as an input. For every layer in the Network creates
118 /// a subnet comprised of all the predecessor nodes till the placeholders.
119 /// This subnet is compiled and run on the reference and on the test backend
120 /// and results are compared.
121 bool verify(PlaceholderBindings *bindings);
122};
123
124/// A comparator class that first tests layer all at once to find suspicious
125/// layers that are different between the reference and test backends runs. Then
126/// all the layers are tested one layer at a time to find out which
127/// of them are actually producing wrong results on their own and not as a
128/// result of errors in any previous layers.
129class IntermediateLayerComparator : public NetworkComparatorBase {
130private:
131 /// Save results (including intermediates) for running the network on the
132 /// reference backend.
133 PlaceholderBindings refHookedBindings_;
134 /// Save results (including intermediates) for running the network on the test
135 /// backend.
136 PlaceholderBindings testHookedBindings_;
137 /// Inserts Save Nodes for the outputs of \p node and saves them in \p
138 /// saveNodes and the placeholders in \p hookPlaceholders.
139 void hookSingleNodeInPlace(Node &node, std::list<SaveNode *> &saveNodes,
140 std::list<Placeholder *> &hookPlaceholders);
141 /// Hook outputs of all the layers in \p func and saves the pointers to
142 /// the inserted Save nodes and corresponding placeholders in \p saveNodes
143 /// and \p hookPlaceholders.
144 void hookNodesInPlace(Function *func, std::list<SaveNode *> &saveNodes,
145 std::list<Placeholder *> &hookPlaceholders);
146 /// Populate tensors in \p hookedBindigs with values from inputBindings.
147 void copyInputBindingsToHookedBindings(PlaceholderBindings &hookedBindigs,
148 PlaceholderBindings &inputBindings);
149 /// Uses the \p networkExecEngine to get all the intermediate results of
150 /// running the network and save the results in \p hookedBindigs. Uses \p
151 /// inputBindings for inputs for the run.
152 void getIntermediateResults(ExecutionEngine &networkExecEngine,
153 PlaceholderBindings *inputBindings,
154 PlaceholderBindings &hookedBindigs);
155 /// Takes \p originalNode , that represents the layer being tested,
156 /// and fills in the inputs of \p singleLayerNode with placeholders from
157 /// running the hooked network on a ref backend or constants
158 /// from the original module. Saves the inputs in \p singleLayerInputMap and
159 /// the placeholders in \p singleLayerBindings.
160 void fillSingleLayerInputs(
161 const Node &originalNode, Node *singleLayerNode, Module &singleLayerMod,
162 std::unordered_map<std::string, Tensor *> &singleLayerInputMap,
163 PlaceholderBindings &singleLayerBindings);
164 /// Runs the network made of a single layer \p singleLayerNode that is getting
165 /// tested. The outputs are saved in \p singleLayerOutputs and bindings in \p
166 /// singleLayerBindings \p refOutputs are populated fron the reference
167 /// bindings run.
168 void runAndGetoutputSingleLayer(
169 ExecutionEngine &singleLayerExecEng,
170 PlaceholderBindings &singleLayerBindings, Node *singleLayerNode,
171 std::unordered_map<std::string, Tensor *> &singleLayerOutputs,
172 std::unordered_map<std::string, Tensor *> &refOutputs);
173 /// Tests a single layer node passed in \p node.
174 bool testSingleLayer(const Node *node);
175
176public:
177 /// Constructor for the IntermediateLayerComparator tester that tests the
178 /// network passed in \p mod on the \p testBackend using the \p
179 /// referenceBackend as a reference backend. \p numericCmpThreshold_.
180 /// Inputs\outputs of a detected bad layer are dumped if \p
181 /// dumpTensorsForBadLayer is set.
182 IntermediateLayerComparator(Module &mod, const std::string &referenceBackend,
183 const std::string &testBackend,
184 float numericCmpThreshold,
185 bool IntermediateLayerComparator);
186 /// Takes the \p bindings as an input. For every layer in the Network creates
187 /// a subnet comprised of all the predecessor nodes till the placeholders.
188 /// This subnet is compiled and run on the reference and on the test backend
189 /// and results are compared.
190 virtual bool verify(PlaceholderBindings *bindings) override;
191};
192
193} // namespace glow
194
195#endif // GLOW_GRAPH_NETWORKCOMPARATOR_H
196