1 | /** |
2 | * Copyright (c) Glow Contributors. See CONTRIBUTORS file. |
3 | * |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. |
6 | * You may obtain a copy of the License at |
7 | * |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * |
10 | * Unless required by applicable law or agreed to in writing, software |
11 | * distributed under the License is distributed on an "AS IS" BASIS, |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
13 | * See the License for the specific language governing permissions and |
14 | * limitations under the License. |
15 | */ |
16 | #ifndef GLOW_GRAPH_NETWORKCOMPARATOR_H |
17 | #define GLOW_GRAPH_NETWORKCOMPARATOR_H |
18 | |
19 | #include "glow/ExecutionEngine/ExecutionEngine.h" |
20 | #include "glow/Graph/Graph.h" |
21 | #include "glow/Graph/PlaceholderBindings.h" |
22 | #include "llvm/ADT/StringRef.h" |
23 | #include <list> |
24 | #include <string> |
25 | #include <unordered_map> |
26 | #include <vector> |
27 | |
28 | namespace glow { |
29 | |
30 | class Function; |
31 | class Node; |
32 | class Placeholder; |
33 | class SaveNode; |
34 | class Tensor; |
35 | class Module; |
36 | |
37 | /// Base class for building a comparator that takes an input network \p |
38 | /// inputModule_ and run it on two backends; a test backend and a reference |
39 | /// backend and saves the list of layers that generate wrong results on the test |
40 | /// net in \p brokenLayers_ . This class has a pure virtual function (verify) |
41 | /// that must be implemented by a subclass. |
42 | class NetworkComparatorBase { |
43 | public: |
44 | /// Struct to save the list of input and output tensors of a node\layer. |
45 | struct InOutTensors { |
46 | std::unordered_map<std::string, Tensor *> inputs; |
47 | std::unordered_map<std::string, Tensor *> outputs; |
48 | }; |
49 | |
50 | protected: |
51 | /// Execution Engine to run the Network tested on the reference backend. |
52 | ExecutionEngine EERefNet_; |
53 | /// Execution Engine to run the Network tested on the test backend. |
54 | ExecutionEngine EETestNet_; |
55 | /// Stores layer names found to be broken on the test backend. |
56 | std::vector<std::string> brokenLayers_; |
57 | /// Threshold of numerical comparison for tensors. |
58 | float numericCmpThreshold_; |
59 | /// Pointer to the module being tested. |
60 | Module *inputModule_; |
61 | |
62 | /// Dump input\output tensors of a bad layer. |
63 | bool dumpTensorsForBadLayer_; |
64 | /// Prints out the input or output \p tensors associated with a \p layerName |
65 | /// the output file is a concatenation of the \p prefix and the \p layerName. |
66 | virtual void dumpTensors(std::unordered_map<std::string, Tensor *> tensors, |
67 | const std::string &layerName, |
68 | const std::string &prefix); |
69 | /// Compares tensors in \p refOuts with tensors in \p checkOuts using the |
70 | /// isEqual Tensor method. |
71 | bool checkTensors(std::unordered_map<std::string, Tensor *> &refOuts, |
72 | std::unordered_map<std::string, Tensor *> &checkOuts); |
73 | |
74 | public: |
75 | /// Constructor for the base class tester that tests the network passed in |
76 | /// \p mod on the \p testBackend using the \p referenceBackend as a reference |
77 | /// backend. \p numericCmpThreshold is accepted error threshold. |
78 | /// Inputs\outputs of a detected bad layer are dumped if \p |
79 | /// dumpTensorsForBadLayer is set. |
80 | NetworkComparatorBase(Module &mod, const std::string &referenceBackend, |
81 | const std::string &testBackend, |
82 | float numericCmpThreshold, bool dumpTensorsForBadLayer); |
83 | virtual ~NetworkComparatorBase() {} |
84 | /// Test the network with the inputs passed in \p bindings. The function |
85 | /// is pure virtual to be implemented by the underlying strategy in sub |
86 | /// classes. \returns True if all checks passed. |
87 | virtual bool verify(PlaceholderBindings *bindings) = 0; |
88 | }; |
89 | |
90 | /// A comparator class that tests layers by creating a subnet of the original |
91 | /// graph for every layer. The subnets are created by recursively visiting the |
92 | /// inputs of the layer. The results or running the subnet are compared between |
93 | /// the test and reference backend. The subnet might have more than one broken |
94 | /// layer. |
95 | class RecursiveLayerComparator : public NetworkComparatorBase { |
96 | private: |
97 | PlaceholderBindings inferBindingsRef_; |
98 | PlaceholderBindings inferBindingsCheck_; |
99 | /// Takes a \p layerName, creates a subnet comprised of all the predecessor |
100 | /// nodes till the inputs. The outputs of this layer are saved in the |
101 | /// inferenceBindingsRef if \p isRef is true and in inferenceBindingsCheck if |
102 | /// false. The inputs are passed in \p bindings. The run saves the inputs to |
103 | /// the layer \p layerName. |
104 | NetworkComparatorBase::InOutTensors hookAndRun(llvm::StringRef layerName, |
105 | PlaceholderBindings *bindings, |
106 | bool isRef); |
107 | |
108 | public: |
109 | /// Constructor for the RecursiveLayerComparator tester that tests the network |
110 | /// passed in \p mod on the \p testBackend using the \p referenceBackend as a |
111 | /// reference backend. \p numericCmpThreshold_. Inputs\outputs of |
112 | /// a detected bad layer are dumped if \p dumpTensorsForBadLayer is set. |
113 | RecursiveLayerComparator(Module &mod, const std::string &referenceBackend, |
114 | const std::string &testBackend, |
115 | float numericCmpThreshold_, |
116 | bool dumpTensorsForBadLayer); |
117 | /// Takes the \p bindings as an input. For every layer in the Network creates |
118 | /// a subnet comprised of all the predecessor nodes till the placeholders. |
119 | /// This subnet is compiled and run on the reference and on the test backend |
120 | /// and results are compared. |
121 | bool verify(PlaceholderBindings *bindings); |
122 | }; |
123 | |
124 | /// A comparator class that first tests layer all at once to find suspicious |
125 | /// layers that are different between the reference and test backends runs. Then |
126 | /// all the layers are tested one layer at a time to find out which |
127 | /// of them are actually producing wrong results on their own and not as a |
128 | /// result of errors in any previous layers. |
129 | class IntermediateLayerComparator : public NetworkComparatorBase { |
130 | private: |
131 | /// Save results (including intermediates) for running the network on the |
132 | /// reference backend. |
133 | PlaceholderBindings refHookedBindings_; |
134 | /// Save results (including intermediates) for running the network on the test |
135 | /// backend. |
136 | PlaceholderBindings testHookedBindings_; |
137 | /// Inserts Save Nodes for the outputs of \p node and saves them in \p |
138 | /// saveNodes and the placeholders in \p hookPlaceholders. |
139 | void hookSingleNodeInPlace(Node &node, std::list<SaveNode *> &saveNodes, |
140 | std::list<Placeholder *> &hookPlaceholders); |
141 | /// Hook outputs of all the layers in \p func and saves the pointers to |
142 | /// the inserted Save nodes and corresponding placeholders in \p saveNodes |
143 | /// and \p hookPlaceholders. |
144 | void hookNodesInPlace(Function *func, std::list<SaveNode *> &saveNodes, |
145 | std::list<Placeholder *> &hookPlaceholders); |
146 | /// Populate tensors in \p hookedBindigs with values from inputBindings. |
147 | void copyInputBindingsToHookedBindings(PlaceholderBindings &hookedBindigs, |
148 | PlaceholderBindings &inputBindings); |
149 | /// Uses the \p networkExecEngine to get all the intermediate results of |
150 | /// running the network and save the results in \p hookedBindigs. Uses \p |
151 | /// inputBindings for inputs for the run. |
152 | void getIntermediateResults(ExecutionEngine &networkExecEngine, |
153 | PlaceholderBindings *inputBindings, |
154 | PlaceholderBindings &hookedBindigs); |
155 | /// Takes \p originalNode , that represents the layer being tested, |
156 | /// and fills in the inputs of \p singleLayerNode with placeholders from |
157 | /// running the hooked network on a ref backend or constants |
158 | /// from the original module. Saves the inputs in \p singleLayerInputMap and |
159 | /// the placeholders in \p singleLayerBindings. |
160 | void fillSingleLayerInputs( |
161 | const Node &originalNode, Node *singleLayerNode, Module &singleLayerMod, |
162 | std::unordered_map<std::string, Tensor *> &singleLayerInputMap, |
163 | PlaceholderBindings &singleLayerBindings); |
164 | /// Runs the network made of a single layer \p singleLayerNode that is getting |
165 | /// tested. The outputs are saved in \p singleLayerOutputs and bindings in \p |
166 | /// singleLayerBindings \p refOutputs are populated fron the reference |
167 | /// bindings run. |
168 | void runAndGetoutputSingleLayer( |
169 | ExecutionEngine &singleLayerExecEng, |
170 | PlaceholderBindings &singleLayerBindings, Node *singleLayerNode, |
171 | std::unordered_map<std::string, Tensor *> &singleLayerOutputs, |
172 | std::unordered_map<std::string, Tensor *> &refOutputs); |
173 | /// Tests a single layer node passed in \p node. |
174 | bool testSingleLayer(const Node *node); |
175 | |
176 | public: |
177 | /// Constructor for the IntermediateLayerComparator tester that tests the |
178 | /// network passed in \p mod on the \p testBackend using the \p |
179 | /// referenceBackend as a reference backend. \p numericCmpThreshold_. |
180 | /// Inputs\outputs of a detected bad layer are dumped if \p |
181 | /// dumpTensorsForBadLayer is set. |
182 | IntermediateLayerComparator(Module &mod, const std::string &referenceBackend, |
183 | const std::string &testBackend, |
184 | float numericCmpThreshold, |
185 | bool IntermediateLayerComparator); |
186 | /// Takes the \p bindings as an input. For every layer in the Network creates |
187 | /// a subnet comprised of all the predecessor nodes till the placeholders. |
188 | /// This subnet is compiled and run on the reference and on the test backend |
189 | /// and results are compared. |
190 | virtual bool verify(PlaceholderBindings *bindings) override; |
191 | }; |
192 | |
193 | } // namespace glow |
194 | |
195 | #endif // GLOW_GRAPH_NETWORKCOMPARATOR_H |
196 | |