1/**
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "glow/Base/Image.h"
18
19#include "llvm/Support/FileSystem.h"
20
21#include "gtest/gtest.h"
22
23#include <cstdio>
24#include <utility>
25
26using namespace glow;
27
28class ImageTest : public ::testing::Test {
29protected:
30 void SetUp() override { initImageCmdArgVars(); }
31 void TearDown() override {}
32};
33
34static void numpyTestHelper(llvm::ArrayRef<std::string> filenames,
35 llvm::ArrayRef<dim_t> expDims,
36 std::vector<float> &vals,
37 llvm::ArrayRef<ImageLayout> imgLayout,
38 llvm::ArrayRef<ImageLayout> inLayout,
39 llvm::ArrayRef<ImageNormalizationMode> normMode,
40 VecVecRef<float> mean = {{}},
41 VecVecRef<float> stddev = {{}}) {
42 Tensor image;
43 loadImagesAndPreprocess({filenames}, {&image}, normMode,
44 {ImageChannelOrder::RGB}, imgLayout, inLayout, mean,
45 stddev);
46
47 ASSERT_EQ(ElemKind::FloatTy, image.getType().getElementType());
48 ASSERT_EQ(expDims.size(), image.dims().size());
49 EXPECT_EQ(image.dims(), expDims);
50 auto H = image.getHandle();
51 for (dim_t i = 0; i < H.size(); i++) {
52 EXPECT_NEAR(H.raw(i), vals[i], 0.000001) << "at index: " << i;
53 }
54}
55
56// Test loading numpy 1D U8 tensor with mean/stddev.
57TEST_F(ImageTest, readNpyTensor1D_U8) {
58 std::vector<float> vals;
59 for (int i = 0; i < 48; i++) {
60 vals.push_back((i - 5) / 2.);
61 }
62 numpyTestHelper({"tests/images/npy/tensor48_u8.npy"}, {48}, vals,
63 {ImageLayout::Unspecified}, {},
64 ImageNormalizationMode::k0to255, {{5.}}, {{2.}});
65}
66
67// Test loading numpy 1D U8 tensor with mean/stddev and normalization.
68TEST_F(ImageTest, readNpyTensor1D_U8Norm) {
69 std::vector<float> vals = {
70 -1.000000, -0.992157, -0.984314, -0.976471, -0.968627, -0.960784,
71 -0.952941, -0.945098, -0.937255, -0.929412, -0.921569, -0.913725,
72 -0.905882, -0.898039, -0.890196, -0.882353, -0.874510, -0.866667,
73 -0.858824, -0.850980, -0.843137, -0.835294, -0.827451, -0.819608,
74 -0.811765, -0.803922, -0.796078, -0.788235, -0.780392, -0.772549,
75 -0.764706, -0.756863, -0.749020, -0.741176, -0.733333, -0.725490,
76 -0.717647, -0.709804, -0.701961, -0.694118, -0.686275, -0.678431,
77 -0.670588, -0.662745, -0.654902, -0.647059, -0.639216, -0.631373};
78 numpyTestHelper({"tests/images/npy/tensor48_u8.npy"}, {48}, vals,
79 {ImageLayout::Unspecified}, {},
80 ImageNormalizationMode::kneg1to1, {{0.}}, {{1.}});
81}
82
83// Test loading numpy 1D I8 tensors.
84TEST_F(ImageTest, readNpyTensor1D_I8) {
85 std::vector<float> vals;
86 for (int i = 0; i < 48; i++) {
87 vals.push_back((i - 5) / 2.);
88 }
89 numpyTestHelper({"tests/images/npy/tensor48_i8.npy"}, {48}, vals,
90 ImageLayout::Unspecified, {}, {}, {{5.}}, {{2.}});
91}
92
93// Test loading numpy 1D I8 with normalization tensors.
94TEST_F(ImageTest, readNpyTensor1D_I8Norm) {
95 std::vector<float> vals = {
96 0.003922, 0.011765, 0.019608, 0.027451, 0.035294, 0.043137, 0.050980,
97 0.058824, 0.066667, 0.074510, 0.082353, 0.090196, 0.098039, 0.105882,
98 0.113726, 0.121569, 0.129412, 0.137255, 0.145098, 0.152941, 0.160784,
99 0.168628, 0.176471, 0.184314, 0.192157, 0.200000, 0.207843, 0.215686,
100 0.223529, 0.231373, 0.239216, 0.247059, 0.254902, 0.262745, 0.270588,
101 0.278431, 0.286275, 0.294118, 0.301961, 0.309804, 0.317647, 0.325490,
102 0.333333, 0.341177, 0.349020, 0.356863, 0.364706, 0.372549};
103 numpyTestHelper({"tests/images/npy/tensor48_i8.npy"}, {48}, vals,
104 {ImageLayout::Unspecified}, {},
105 ImageNormalizationMode::kneg1to1, {{0.}}, {{1.}});
106}
107
108// Test loading numpy 2D U8 tensors.
109TEST_F(ImageTest, readNpyTensor2D_U8) {
110 std::vector<float> vals;
111 for (int i = 0; i < 48; i++) {
112 vals.push_back((i - 2) / 3.);
113 }
114 numpyTestHelper({"tests/images/npy/tensor3x16_u8.npy"}, {3, 16}, vals,
115 {ImageLayout::Unspecified}, {}, {}, {{2.}}, {{3.}});
116}
117
118// Test loading numpy 3D U8 tensors.
119TEST_F(ImageTest, readNpyTensor3D_U8) {
120 std::vector<float> vals;
121 for (int i = 0; i < 48; i++) {
122 vals.push_back((i - 2) / 3.);
123 }
124 numpyTestHelper({"tests/images/npy/tensor2x3x8_u8.npy"}, {2, 3, 8}, vals,
125 {ImageLayout::Unspecified}, {}, {}, {{2.}}, {{3.}});
126}
127
128// Test loading numpy 4D U8 tensors.
129TEST_F(ImageTest, readNpyTensor4D_U8) {
130 std::vector<float> vals;
131 for (int i = 0; i < 48; i++) {
132 vals.push_back((i - 2) / 3.);
133 }
134 numpyTestHelper({"tests/images/npy/tensor1x2x3x8_u8.npy"}, {1, 2, 3, 8}, vals,
135 {ImageLayout::Unspecified}, {}, {}, {{2.}}, {{3.}});
136}
137
138// Test loading numpy 1D I16 tensors without normalization.
139TEST_F(ImageTest, readNpyTensor1D_I16) {
140 std::vector<float> vals = {497.5, 498., 498.5, 499., 499.5, 500.,
141 500.5, 501., 501.5, 502., 502.5, 503.,
142 503.5, 504., 504.5, 505.};
143 numpyTestHelper({"tests/images/npy/tensor16_i16.npy"}, {16}, vals,
144 {ImageLayout::Unspecified}, {}, ImageNormalizationMode::S16,
145 {{5.}}, {{2.}});
146}
147
148// Test loading numpy 1D U16 tensors with normalization.
149TEST_F(ImageTest, readNpyTensor1D_U16Norm) {
150 std::vector<float> vals = {-0.969482, -0.969451, -0.969421, -0.969390,
151 -0.969360, -0.969329, -0.969299, -0.969268,
152 -0.969238, -0.969207, -0.969177, -0.969146,
153 -0.969116, -0.969085, -0.969055, -0.969024};
154 numpyTestHelper({"tests/images/npy/tensor16_u16.npy"}, {16}, vals,
155 {ImageLayout::Unspecified}, {},
156 ImageNormalizationMode::kneg1to1, {{0.}}, {{1.}});
157}
158
159// Test loading from numpy file w/o changing layout.
160TEST_F(ImageTest, readNpyNCHWtoNCHW_4D_image) {
161 std::vector<float> vals;
162 for (int i = 0; i < 48; i++) {
163 vals.push_back(i);
164 }
165 numpyTestHelper({"tests/images/npy/tensor3x4x2x2_u8.npy"}, {3, 4, 2, 2}, vals,
166 {ImageLayout::NCHW}, {ImageLayout::NCHW}, {});
167}
168
169// Test loading from numpy file with mean/stddev.
170TEST_F(ImageTest, readNpy_stddev_mean) {
171 std::vector<float> vals;
172 std::vector<float> mean = {1.1, 1.2, 1.3, 1.4};
173 std::vector<float> stddev = {2.1, 2.2, 2.3, 2.4};
174 for (int i = 0; i < 48; i++) {
175 vals.push_back(((float)i - mean[(i / 4) % 4]) / stddev[(i / 4) % 4]);
176 }
177 numpyTestHelper({"tests/images/npy/tensor3x4x2x2_u8.npy"}, {3, 4, 2, 2}, vals,
178 {ImageLayout::NCHW}, {ImageLayout::NCHW}, {}, {mean},
179 {stddev});
180}
181
182// Test loading 3D image from numpy file.
183TEST_F(ImageTest, readNpyNHWCtoNHWC_3D_image) {
184 std::vector<float> vals;
185 for (int i = 0; i < 24; i++) {
186 vals.push_back(i);
187 }
188 numpyTestHelper({"tests/images/npy/tensor3x4x2_u8.npy"}, {1, 3, 4, 2}, vals,
189 {ImageLayout::NHWC}, {ImageLayout::NHWC}, {});
190}
191
192// Test loading from numpy file with change of layout.
193TEST_F(ImageTest, readNpyNCHWtoNHWC_4D_image) {
194 std::vector<float> vals;
195 for (int i = 0; i < 48; i++) {
196 vals.push_back(i);
197 }
198 Tensor tensor(ElemKind::FloatTy, {3, 4, 2, 2});
199 tensor.getHandle() = vals;
200 Tensor transposed;
201 tensor.transpose(&transposed, {0u, 2u, 3u, 1u});
202 vals.clear();
203 for (int i = 0; i < 48; i++) {
204 vals.push_back(transposed.getHandle().raw(i));
205 }
206 numpyTestHelper({"tests/images/npy/tensor3x4x2x2_u8.npy"}, transposed.dims(),
207 vals, {ImageLayout::NHWC}, {ImageLayout::NCHW}, {});
208}
209
210// Test loading 3D image from numpy file with change of layout.
211TEST_F(ImageTest, readNpyNHWCtoNCHW_3D_image) {
212 std::vector<float> vals;
213 for (int i = 0; i < 24; i++) {
214 vals.push_back(i);
215 }
216 Tensor tensor(ElemKind::FloatTy, {1, 3, 4, 2});
217 tensor.getHandle() = vals;
218 Tensor transposed;
219 tensor.transpose(&transposed, {0u, 3u, 1u, 2u});
220 vals.clear();
221 for (int i = 0; i < 24; i++) {
222 vals.push_back(transposed.getHandle().raw(i));
223 }
224 numpyTestHelper({"tests/images/npy/tensor3x4x2_i8.npy"}, transposed.dims(),
225 vals, {ImageLayout::NCHW}, {ImageLayout::NHWC}, {});
226}
227
228// Test loading multiple images from numpy files.
229TEST_F(ImageTest, readNpyNHWCtoNHWC_multi_image) {
230 std::vector<float> vals;
231 for (int i = 0; i < 48; i++) {
232 vals.push_back(i);
233 }
234 for (int i = 0; i < 48; i++) {
235 vals.push_back(i);
236 }
237 numpyTestHelper({"tests/images/npy/tensor3x4x2x2_u8.npy",
238 "tests/images/npy/tensor3x4x2x2_i8.npy"},
239 {6, 4, 2, 2}, vals, {ImageLayout::NHWC}, {ImageLayout::NHWC},
240 {});
241}
242
243TEST_F(ImageTest, readNonSquarePngImage) {
244 auto range = std::make_pair(0.f, 1.f);
245 Tensor vgaTensor;
246 bool loadSuccess =
247 !readPngImage(&vgaTensor, "tests/images/other/vga_image.png", range);
248 ASSERT_TRUE(loadSuccess);
249
250 auto &type = vgaTensor.getType();
251 auto shape = vgaTensor.dims();
252
253 // The loaded image is a 3D HWC tensor
254 ASSERT_EQ(ElemKind::FloatTy, type.getElementType());
255 ASSERT_EQ(3, shape.size());
256 ASSERT_EQ(480, shape[0]);
257 ASSERT_EQ(640, shape[1]);
258 ASSERT_EQ(3, shape[2]);
259}
260
261TEST_F(ImageTest, readBadImages) {
262 auto range = std::make_pair(0.f, 1.f);
263 Tensor tensor;
264 bool loadSuccess =
265 !readPngImage(&tensor, "tests/images/other/dog_corrupt.png", range);
266 ASSERT_FALSE(loadSuccess);
267
268 loadSuccess =
269 !readPngImage(&tensor, "tests/images/other/ghost_missing.png", range);
270 ASSERT_FALSE(loadSuccess);
271}
272
273TEST_F(ImageTest, readPngPpmImageAndPreprocessWithAndWithoutInputTensor) {
274 auto image1 = readPngPpmImageAndPreprocess(
275 "tests/images/imagenet/cat_285.png", ImageNormalizationMode::k0to1,
276 ImageChannelOrder::RGB, ImageLayout::NHWC, imagenetNormMean,
277 imagenetNormStd);
278
279 Tensor image2;
280 std::vector<float> meanBGR(llvm::makeArrayRef(imagenetNormMean));
281 std::vector<float> stddevBGR(llvm::makeArrayRef(imagenetNormStd));
282 std::reverse(meanBGR.begin(), meanBGR.end());
283 std::reverse(stddevBGR.begin(), stddevBGR.end());
284 readPngPpmImageAndPreprocess(image2, "tests/images/imagenet/cat_285.png",
285 ImageNormalizationMode::k0to1,
286 ImageChannelOrder::BGR, ImageLayout::NCHW,
287 meanBGR, stddevBGR);
288
289 // Test if the preprocess actually happened.
290 dim_t imgHeight = image1.dims()[0];
291 dim_t imgWidth = image1.dims()[1];
292 dim_t numChannels = image1.dims()[2];
293
294 Tensor transposed;
295 image2.transpose(&transposed, {1u, 2u, 0u});
296 image2 = std::move(transposed);
297
298 Tensor swizzled(image1.getType());
299 auto IH = image1.getHandle();
300 auto SH = swizzled.getHandle();
301 for (dim_t z = 0; z < numChannels; z++) {
302 for (dim_t y = 0; y < imgHeight; y++) {
303 for (dim_t x = 0; x < imgWidth; x++) {
304 SH.at({x, y, numChannels - 1 - z}) = IH.at({x, y, z});
305 }
306 }
307 }
308 image1 = std::move(swizzled);
309 EXPECT_TRUE(image1.isEqual(image2, 0.01));
310}
311
312TEST_F(ImageTest, readPpmImageAndPreprocess) {
313 std::vector<float> imagenetNormMeanBGR(imagenetNormMean,
314 imagenetNormMean + 3);
315 std::vector<float> imagenetNormStdBGR(imagenetNormStd, imagenetNormStd + 3);
316 std::reverse(imagenetNormMeanBGR.begin(), imagenetNormMeanBGR.end());
317 std::reverse(imagenetNormStdBGR.begin(), imagenetNormStdBGR.end());
318
319 // Use PNG image as reference.
320 auto pngRef = readPngPpmImageAndPreprocess(
321 "tests/images/imagenet/cat_285.png", ImageNormalizationMode::k0to1,
322 ImageChannelOrder::RGB, ImageLayout::NHWC, imagenetNormMean,
323 imagenetNormStd);
324 auto ppmExp = readPngPpmImageAndPreprocess(
325 "tests/images/ppm/cat_285.ppm", ImageNormalizationMode::k0to1,
326 ImageChannelOrder::RGB, ImageLayout::NHWC, imagenetNormMean,
327 imagenetNormStd);
328
329 EXPECT_TRUE(ppmExp.isEqual(pngRef, 0.01));
330}
331
332TEST_F(ImageTest, writePngImage) {
333 auto range = std::make_pair(0.f, 1.f);
334 Tensor localCopy;
335 bool loadSuccess =
336 !readPngImage(&localCopy, "tests/images/imagenet/cat_285.png", range);
337 ASSERT_TRUE(loadSuccess);
338
339 llvm::SmallVector<char, 10> resultPath;
340 llvm::sys::fs::createTemporaryFile("prefix", "suffix", resultPath);
341 std::string outfilename(resultPath.begin(), resultPath.end());
342
343 bool storeSuccess = !writePngImage(&localCopy, outfilename.c_str(), range);
344 ASSERT_TRUE(storeSuccess);
345
346 Tensor secondLocalCopy;
347 loadSuccess = !readPngImage(&secondLocalCopy, outfilename.c_str(), range);
348 ASSERT_TRUE(loadSuccess);
349 EXPECT_TRUE(secondLocalCopy.isEqual(localCopy, 0.01));
350
351 // Delete the temporary file.
352 std::remove(outfilename.c_str());
353}
354
355TEST_F(ImageTest, readMultipleInputsOpt) {
356 imageLayoutOpt = {ImageLayout::NCHW, ImageLayout::NCHW};
357 meanValuesOpt = {{127.5, 127.5, 127.5}, {0, 0, 0}};
358 stddevValuesOpt = {{2, 2, 2}, {1, 1, 1}};
359 imageChannelOrderOpt = {ImageChannelOrder::RGB, ImageChannelOrder::RGB};
360 imageNormMode = {ImageNormalizationMode::k0to255,
361 ImageNormalizationMode::k0to255};
362
363 std::vector<std::vector<std::string>> filenamesList = {
364 {"tests/images/imagenet/cat_285.png"},
365 {"tests/images/imagenet/cat_285.png"}};
366 Tensor image1;
367 Tensor image2;
368 loadImagesAndPreprocess(filenamesList, {&image1, &image2});
369
370 auto H1 = image1.getHandle();
371 auto H2 = image2.getHandle();
372 EXPECT_EQ(H1.size(), H2.size());
373 for (dim_t i = 0; i < H1.size(); i++) {
374 EXPECT_FLOAT_EQ((H2.raw(i) - 127.5) / 2, H1.raw(i));
375 }
376}
377
378TEST_F(ImageTest, readMultipleInputsApi) {
379 std::vector<ImageLayout> layout = {ImageLayout::NHWC, ImageLayout::NHWC};
380 std::vector<std::vector<float>> mean = {{100, 100, 100}, {0, 0, 0}};
381 std::vector<std::vector<float>> stddev = {{1.5, 1.5, 1.5}, {1, 1, 1}};
382 std::vector<ImageChannelOrder> chOrder = {ImageChannelOrder::BGR,
383 ImageChannelOrder::BGR};
384 std::vector<ImageNormalizationMode> norm = {ImageNormalizationMode::k0to1,
385 ImageNormalizationMode::k0to1};
386
387 std::vector<std::vector<std::string>> filenamesList = {
388 {"tests/images/imagenet/cat_285.png"},
389 {"tests/images/imagenet/cat_285.png"}};
390 Tensor image1;
391 Tensor image2;
392 loadImagesAndPreprocess(filenamesList, {&image1, &image2}, norm, chOrder,
393 layout, {}, mean, stddev);
394
395 auto H1 = image1.getHandle();
396 auto H2 = image2.getHandle();
397 EXPECT_EQ(H1.size(), H2.size());
398 for (dim_t i = 0; i < H1.size(); i++) {
399 EXPECT_NEAR((H2.raw(i) - (100 / 255.)) / 1.5, H1.raw(i), 0.0000001);
400 }
401}
402
403/// Test writing a png image along with using the standard Imagenet
404/// normalization when reading the image.
405TEST_F(ImageTest, writePngImageWithImagenetNormalization) {
406 auto range = std::make_pair(0.f, 1.f);
407 Tensor localCopy;
408 bool loadSuccess =
409 !readPngImage(&localCopy, "tests/images/imagenet/cat_285.png", range,
410 imagenetNormMean, imagenetNormStd);
411 ASSERT_TRUE(loadSuccess);
412
413 llvm::SmallVector<char, 10> resultPath;
414 llvm::sys::fs::createTemporaryFile("prefix", "suffix", resultPath);
415 std::string outfilename(resultPath.begin(), resultPath.end());
416
417 bool storeSuccess = !writePngImage(&localCopy, outfilename.c_str(), range,
418 imagenetNormMean, imagenetNormStd);
419 ASSERT_TRUE(storeSuccess);
420
421 Tensor secondLocalCopy;
422 loadSuccess = !readPngImage(&secondLocalCopy, outfilename.c_str(), range,
423 imagenetNormMean, imagenetNormStd);
424 ASSERT_TRUE(loadSuccess);
425 EXPECT_TRUE(secondLocalCopy.isEqual(localCopy, 0.02));
426
427 // Delete the temporary file.
428 std::remove(outfilename.c_str());
429}
430
431/// Test PNG w/ order and layout transposes, and different mean/stddev per
432/// channel.
433TEST_F(ImageTest, readNonSquarePngBGRNCHWTest) {
434 auto image = readPngPpmImageAndPreprocess(
435 "tests/images/other/tensor_2x4x3.png", ImageNormalizationMode::k0to255,
436 ImageChannelOrder::BGR, ImageLayout::NCHW, {0, 1, 2}, {3, 4, 5});
437
438 std::vector<float> expected = {1., 2.0, 3., 4., 5., 6.0, 7., 8.,
439 0.25, 1., 1.75, 2.5, 3.25, 4., 4.75, 5.5,
440 -0.2, 0.4, 1., 1.6, 2.2, 2.8, 3.4, 4.};
441
442 auto H = image.getHandle();
443 for (dim_t i = 0; i < H.size(); i++) {
444 EXPECT_FLOAT_EQ(expected[i], H.raw(i));
445 }
446}
447