1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "glow/Base/Image.h"
18#include "glow/ExecutionEngine/ExecutionEngine.h"
19#include "glow/Graph/Graph.h"
20#include "glow/Importer/Caffe2ModelLoader.h"
21#include "glow/Runtime/HostManager/HostManager.h"
22#include "glow/Runtime/RuntimeTypes.h"
23#include "glow/Support/Error.h"
24
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/FileSystem.h"
27
28#include <glog/logging.h>
29
30#include <chrono>
31#include <future>
32
33using namespace glow;
34using namespace glow::runtime;
35
36namespace {
37llvm::cl::OptionCategory category("resnet-runtime Options");
38llvm::cl::opt<std::string>
39 inputDirectory(llvm::cl::desc("input directory for images, which must be "
40 "png's with standard imagenet normalization"),
41 llvm::cl::init("../tests/images/imagenet/"),
42 llvm::cl::Positional, llvm::cl::cat(category));
43llvm::cl::opt<unsigned> numDevices("num-devices",
44 llvm::cl::desc("Number of Devices to use"),
45 llvm::cl::init(5), llvm::cl::value_desc("N"),
46 llvm::cl::cat(category));
47llvm::cl::opt<unsigned>
48 maxImages("max-images",
49 llvm::cl::desc("Maximum number of images to load and classify"),
50 llvm::cl::init(100), llvm::cl::value_desc("N"),
51 llvm::cl::cat(category));
52llvm::cl::opt<std::string> tracePath("trace-path",
53 llvm::cl::desc("Write trace logs to disk"),
54 llvm::cl::init(""),
55 llvm::cl::cat(category));
56llvm::cl::opt<std::string>
57 backend("backend",
58 llvm::cl::desc("Backend to use, e.g., Interpreter, CPU, OpenCL:"),
59 llvm::cl::Optional, llvm::cl::init("CPU"), llvm::cl::cat(category));
60
61llvm::cl::opt<bool>
62 autoInstrument("auto-instrument",
63 llvm::cl::desc("Add instrumentation for operator tracing"),
64 llvm::cl::Optional, llvm::cl::init(false),
65 llvm::cl::cat(category));
66
67std::mutex eventLock;
68std::unique_ptr<TraceContext> traceContext;
69
70} // namespace
71
72/// Loads the model into /p module and returns the input and output
73/// Placeholders. Appending count to the function name.
74Placeholder *loadResnet50Model(TypeRef inputType, Module *module,
75 unsigned int count) {
76 Function *F = module->createFunction("resnet50" + std::to_string(count));
77
78 LOG(INFO) << "Loading resnet50 model.";
79
80 const char inputName[] = "gpu_0/data";
81 Caffe2ModelLoader loader("resnet50/predict_net.pb", "resnet50/init_net.pb",
82 {inputName}, {inputType}, *F);
83 Placeholder *input = llvm::cast<Placeholder>(
84 EXIT_ON_ERR(loader.getNodeValueByName(inputName)));
85 return input;
86}
87
88/// Starts a run of resnet50 on the given image. The image must be already
89/// loaded into the input placeholder in /p context.
90/// If, at the end of the run the number of \p returned results is equal to
91/// maxImages, the \p finished promise is set.
92void dispatchClassify(unsigned int id, HostManager *hostManager,
93 std::string path,
94 std::unique_ptr<ExecutionContext> context,
95 std::atomic<size_t> &returned,
96 std::promise<void> &finished) {
97 auto runid = hostManager->runNetwork(
98 "resnet50" + std::to_string(id), std::move(context),
99 [path, &returned, &finished](RunIdentifierTy runid, Error err,
100 std::unique_ptr<ExecutionContext> context) {
101 EXIT_ON_ERR(std::move(err));
102 auto *bindings = context->getPlaceholderBindings();
103 size_t maxIdx =
104 bindings->get(bindings->getPlaceholderByNameSlow("gpu_0_softmax"))
105 ->getHandle()
106 .minMaxArg()
107 .second;
108 // This output is verified by OutputCheck in tests so must be written to
109 // stdout.
110 llvm::outs() << "(" << runid << ") " << path << ": " << maxIdx << "\n";
111
112 if (!tracePath.empty()) {
113 std::lock_guard<std::mutex> l(eventLock);
114 // Merge this run's TraceEvents into the global TraceContext.
115 traceContext->merge(context->getTraceContext());
116 }
117
118 if (++returned == maxImages) {
119 finished.set_value();
120 }
121 });
122 LOG(INFO) << "Started run ID: " << runid;
123}
124
125/// Run ResNet concurrently on the number of devices provided by the user.
126int main(int argc, char **argv) {
127 llvm::cl::ParseCommandLineOptions(
128 argc, argv, "Run ResNet concurrently on a fixed number of devices");
129
130 LOG(INFO) << "Initializing " << numDevices << " " << backend
131 << " devices on HostManager.";
132
133 std::vector<std::unique_ptr<DeviceConfig>> configs;
134 for (unsigned int i = 0; i < numDevices; ++i) {
135 auto config = glow::make_unique<DeviceConfig>(backend);
136 configs.push_back(std::move(config));
137 }
138
139 std::unique_ptr<HostManager> hostManager =
140 glow::make_unique<HostManager>(std::move(configs));
141
142 // If tracing is enabled, create a TraceContext to merge each runs events
143 // into.
144 if (!tracePath.empty()) {
145 traceContext = glow::make_unique<TraceContext>(TraceLevel::STANDARD);
146 }
147
148 // Load model, create a context, and add to HostManager.
149
150 std::vector<dim_t> inputShape{1, 3, 224, 224};
151
152 Placeholder *input;
153 PlaceholderList phList;
154
155 std::unique_ptr<Module> module = glow::make_unique<Module>();
156 TypeRef inputType = module->uniqueType(ElemKind::FloatTy, inputShape);
157 input = loadResnet50Model(inputType, module.get(), 0);
158 phList = module->getPlaceholders();
159 CompilationContext cctx;
160 cctx.backendOpts.autoInstrument = autoInstrument;
161 cctx.saturateHost = true;
162 EXIT_ON_ERR(hostManager->addNetwork(std::move(module), cctx));
163
164 LOG(INFO) << "Loading files from " << inputDirectory;
165 std::error_code code;
166 llvm::sys::fs::directory_iterator dirIt(inputDirectory, code);
167 if (code.value()) {
168 LOG(ERROR) << "Couldn't read from directory: " << inputDirectory
169 << " - code" << code.value() << "\n";
170 exit(code.value());
171 }
172
173 std::promise<void> finished;
174
175 size_t started = 0;
176 std::atomic<size_t> returned{0};
177
178 // Run up to maxImages classifications.
179 unsigned int currDevice{0};
180 while (started++ < maxImages) {
181 if (code.value() != 0 || dirIt == llvm::sys::fs::directory_iterator()) {
182 started--;
183 returned += maxImages - started;
184
185 if (returned == maxImages) {
186 finished.set_value();
187 }
188 break;
189 }
190
191 std::string path = dirIt->path();
192
193 auto image = readPngPpmImageAndPreprocess(
194 path, ImageNormalizationMode::k0to1, ImageChannelOrder::BGR,
195 ImageLayout::NCHW, imagenetNormMean, imagenetNormStd);
196 std::unique_ptr<ExecutionContext> context =
197 glow::make_unique<ExecutionContext>();
198 context->setTraceContext(
199 glow::make_unique<TraceContext>(TraceLevel::STANDARD));
200
201 context->getPlaceholderBindings()->allocate(phList);
202 Tensor batch = image.getUnowned(inputShape);
203 updateInputPlaceholders(*(context->getPlaceholderBindings()), {input},
204 {&batch});
205
206 dispatchClassify(0, hostManager.get(), std::move(path), std::move(context),
207 returned, finished);
208
209 dirIt.increment(code);
210 currDevice++;
211 }
212
213 finished.get_future().wait();
214
215 LOG(INFO) << "Finished classifying " << started << " images.";
216
217 if (!tracePath.empty()) {
218 traceContext->dump(tracePath, "resnet-runtime");
219 }
220
221 return 0;
222}
223