1 | // Copyright 2011 Google Inc. All Rights Reserved. |
2 | // |
3 | // Redistribution and use in source and binary forms, with or without |
4 | // modification, are permitted provided that the following conditions are |
5 | // met: |
6 | // |
7 | // * Redistributions of source code must retain the above copyright |
8 | // notice, this list of conditions and the following disclaimer. |
9 | // * Redistributions in binary form must reproduce the above |
10 | // copyright notice, this list of conditions and the following disclaimer |
11 | // in the documentation and/or other materials provided with the |
12 | // distribution. |
13 | // * Neither the name of Google Inc. nor the names of its |
14 | // contributors may be used to endorse or promote products derived from |
15 | // this software without specific prior written permission. |
16 | // |
17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
28 | // |
29 | // Various stubs for the unit tests for the open-source version of Snappy. |
30 | |
31 | #include "snappy-test.h" |
32 | |
33 | #include <algorithm> |
34 | #include <cstdarg> |
35 | #include <cstdio> |
36 | #include <cstdlib> |
37 | #include <iostream> |
38 | #include <string> |
39 | |
40 | namespace file { |
41 | |
42 | OptionsStub::OptionsStub() = default; |
43 | OptionsStub::~OptionsStub() = default; |
44 | |
45 | const OptionsStub &Defaults() { |
46 | static OptionsStub defaults; |
47 | return defaults; |
48 | } |
49 | |
50 | StatusStub::StatusStub() = default; |
51 | StatusStub::StatusStub(const StatusStub &) = default; |
52 | StatusStub &StatusStub::operator=(const StatusStub &) = default; |
53 | StatusStub::~StatusStub() = default; |
54 | |
55 | bool StatusStub::ok() { return true; } |
56 | |
57 | StatusStub GetContents(const std::string &filename, std::string *output, |
58 | const OptionsStub & /* options */) { |
59 | std::FILE *fp = std::fopen(filename.c_str(), "rb" ); |
60 | if (fp == nullptr) { |
61 | std::perror(filename.c_str()); |
62 | std::exit(1); |
63 | } |
64 | |
65 | output->clear(); |
66 | while (!std::feof(fp)) { |
67 | char buffer[4096]; |
68 | size_t bytes_read = std::fread(buffer, 1, sizeof(buffer), fp); |
69 | if (bytes_read == 0 && std::ferror(fp)) { |
70 | std::perror("fread" ); |
71 | std::exit(1); |
72 | } |
73 | output->append(buffer, bytes_read); |
74 | } |
75 | |
76 | std::fclose(fp); |
77 | return StatusStub(); |
78 | } |
79 | |
80 | StatusStub SetContents(const std::string &file_name, const std::string &content, |
81 | const OptionsStub & /* options */) { |
82 | std::FILE *fp = std::fopen(file_name.c_str(), "wb" ); |
83 | if (fp == nullptr) { |
84 | std::perror(file_name.c_str()); |
85 | std::exit(1); |
86 | } |
87 | |
88 | size_t bytes_written = std::fwrite(content.data(), 1, content.size(), fp); |
89 | if (bytes_written != content.size()) { |
90 | std::perror("fwrite" ); |
91 | std::exit(1); |
92 | } |
93 | |
94 | std::fclose(fp); |
95 | return StatusStub(); |
96 | } |
97 | |
98 | } // namespace file |
99 | |
100 | namespace snappy { |
101 | |
102 | std::string ReadTestDataFile(const std::string& base, size_t size_limit) { |
103 | std::string contents; |
104 | const char* srcdir = getenv("srcdir" ); // This is set by Automake. |
105 | std::string prefix; |
106 | if (srcdir) { |
107 | prefix = std::string(srcdir) + "/" ; |
108 | } |
109 | file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults() |
110 | ).ok(); |
111 | if (size_limit > 0) { |
112 | contents = contents.substr(0, size_limit); |
113 | } |
114 | return contents; |
115 | } |
116 | |
117 | std::string StrFormat(const char* format, ...) { |
118 | char buffer[4096]; |
119 | std::va_list ap; |
120 | va_start(ap, format); |
121 | std::vsnprintf(buffer, sizeof(buffer), format, ap); |
122 | va_end(ap); |
123 | return buffer; |
124 | } |
125 | |
126 | LogMessage::~LogMessage() { std::cerr << std::endl; } |
127 | |
128 | LogMessage &LogMessage::operator<<(const std::string &message) { |
129 | std::cerr << message; |
130 | return *this; |
131 | } |
132 | |
133 | LogMessage &LogMessage::operator<<(int number) { |
134 | std::cerr << number; |
135 | return *this; |
136 | } |
137 | |
138 | #ifdef _MSC_VER |
139 | // ~LogMessageCrash calls std::abort() and therefore never exits. This is by |
140 | // design, so temporarily disable warning C4722. |
141 | #pragma warning(push) |
142 | #pragma warning(disable : 4722) |
143 | #endif |
144 | |
145 | LogMessageCrash::~LogMessageCrash() { |
146 | std::cerr << std::endl; |
147 | std::abort(); |
148 | } |
149 | |
150 | #ifdef _MSC_VER |
151 | #pragma warning(pop) |
152 | #endif |
153 | |
154 | #if HAVE_LIBZ |
155 | |
156 | ZLib::ZLib() |
157 | : comp_init_(false), |
158 | uncomp_init_(false) { |
159 | Reinit(); |
160 | } |
161 | |
162 | ZLib::~ZLib() { |
163 | if (comp_init_) { deflateEnd(&comp_stream_); } |
164 | if (uncomp_init_) { inflateEnd(&uncomp_stream_); } |
165 | } |
166 | |
167 | void ZLib::Reinit() { |
168 | compression_level_ = Z_DEFAULT_COMPRESSION; |
169 | window_bits_ = MAX_WBITS; |
170 | mem_level_ = 8; // DEF_MEM_LEVEL |
171 | if (comp_init_) { |
172 | deflateEnd(&comp_stream_); |
173 | comp_init_ = false; |
174 | } |
175 | if (uncomp_init_) { |
176 | inflateEnd(&uncomp_stream_); |
177 | uncomp_init_ = false; |
178 | } |
179 | first_chunk_ = true; |
180 | } |
181 | |
182 | void ZLib::Reset() { |
183 | first_chunk_ = true; |
184 | } |
185 | |
186 | // --------- COMPRESS MODE |
187 | |
188 | // Initialization method to be called if we hit an error while |
189 | // compressing. On hitting an error, call this method before returning |
190 | // the error. |
191 | void ZLib::CompressErrorInit() { |
192 | deflateEnd(&comp_stream_); |
193 | comp_init_ = false; |
194 | Reset(); |
195 | } |
196 | |
197 | int ZLib::DeflateInit() { |
198 | return deflateInit2(&comp_stream_, |
199 | compression_level_, |
200 | Z_DEFLATED, |
201 | window_bits_, |
202 | mem_level_, |
203 | Z_DEFAULT_STRATEGY); |
204 | } |
205 | |
206 | int ZLib::CompressInit(Bytef *dest, uLongf *destLen, |
207 | const Bytef *source, uLong *sourceLen) { |
208 | int err; |
209 | |
210 | comp_stream_.next_in = (Bytef*)source; |
211 | comp_stream_.avail_in = (uInt)*sourceLen; |
212 | if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR; |
213 | comp_stream_.next_out = dest; |
214 | comp_stream_.avail_out = (uInt)*destLen; |
215 | if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR; |
216 | |
217 | if ( !first_chunk_ ) // only need to set up stream the first time through |
218 | return Z_OK; |
219 | |
220 | if (comp_init_) { // we've already initted it |
221 | err = deflateReset(&comp_stream_); |
222 | if (err != Z_OK) { |
223 | LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one" ; |
224 | deflateEnd(&comp_stream_); |
225 | comp_init_ = false; |
226 | } |
227 | } |
228 | if (!comp_init_) { // first use |
229 | comp_stream_.zalloc = (alloc_func)0; |
230 | comp_stream_.zfree = (free_func)0; |
231 | comp_stream_.opaque = (voidpf)0; |
232 | err = DeflateInit(); |
233 | if (err != Z_OK) return err; |
234 | comp_init_ = true; |
235 | } |
236 | return Z_OK; |
237 | } |
238 | |
239 | // In a perfect world we'd always have the full buffer to compress |
240 | // when the time came, and we could just call Compress(). Alas, we |
241 | // want to do chunked compression on our webserver. In this |
242 | // application, we compress the header, send it off, then compress the |
243 | // results, send them off, then compress the footer. Thus we need to |
244 | // use the chunked compression features of zlib. |
245 | int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen, |
246 | const Bytef *source, uLong *sourceLen, |
247 | int flush_mode) { // Z_FULL_FLUSH or Z_FINISH |
248 | int err; |
249 | |
250 | if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK ) |
251 | return err; |
252 | |
253 | // This is used to figure out how many bytes we wrote *this chunk* |
254 | int compressed_size = comp_stream_.total_out; |
255 | |
256 | // Some setup happens only for the first chunk we compress in a run |
257 | if ( first_chunk_ ) { |
258 | first_chunk_ = false; |
259 | } |
260 | |
261 | // flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental |
262 | // compression. |
263 | err = deflate(&comp_stream_, flush_mode); |
264 | |
265 | *sourceLen = comp_stream_.avail_in; |
266 | |
267 | if ((err == Z_STREAM_END || err == Z_OK) |
268 | && comp_stream_.avail_in == 0 |
269 | && comp_stream_.avail_out != 0 ) { |
270 | // we processed everything ok and the output buffer was large enough. |
271 | ; |
272 | } else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) { |
273 | return Z_BUF_ERROR; // should never happen |
274 | } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) { |
275 | // an error happened |
276 | CompressErrorInit(); |
277 | return err; |
278 | } else if (comp_stream_.avail_out == 0) { // not enough space |
279 | err = Z_BUF_ERROR; |
280 | } |
281 | |
282 | assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR); |
283 | if (err == Z_STREAM_END) |
284 | err = Z_OK; |
285 | |
286 | // update the crc and other metadata |
287 | compressed_size = comp_stream_.total_out - compressed_size; // delta |
288 | *destLen = compressed_size; |
289 | |
290 | return err; |
291 | } |
292 | |
293 | int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen, |
294 | const Bytef *source, uLong sourceLen, |
295 | int flush_mode) { // Z_FULL_FLUSH or Z_FINISH |
296 | const int ret = |
297 | CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode); |
298 | if (ret == Z_BUF_ERROR) |
299 | CompressErrorInit(); |
300 | return ret; |
301 | } |
302 | |
303 | // This routine only initializes the compression stream once. Thereafter, it |
304 | // just does a deflateReset on the stream, which should be faster. |
305 | int ZLib::Compress(Bytef *dest, uLongf *destLen, |
306 | const Bytef *source, uLong sourceLen) { |
307 | int err; |
308 | if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen, |
309 | Z_FINISH)) != Z_OK ) |
310 | return err; |
311 | Reset(); // reset for next call to Compress |
312 | |
313 | return Z_OK; |
314 | } |
315 | |
316 | |
317 | // --------- UNCOMPRESS MODE |
318 | |
319 | int ZLib::InflateInit() { |
320 | return inflateInit2(&uncomp_stream_, MAX_WBITS); |
321 | } |
322 | |
323 | // Initialization method to be called if we hit an error while |
324 | // uncompressing. On hitting an error, call this method before |
325 | // returning the error. |
326 | void ZLib::UncompressErrorInit() { |
327 | inflateEnd(&uncomp_stream_); |
328 | uncomp_init_ = false; |
329 | Reset(); |
330 | } |
331 | |
332 | int ZLib::UncompressInit(Bytef *dest, uLongf *destLen, |
333 | const Bytef *source, uLong *sourceLen) { |
334 | int err; |
335 | |
336 | uncomp_stream_.next_in = (Bytef*)source; |
337 | uncomp_stream_.avail_in = (uInt)*sourceLen; |
338 | // Check for source > 64K on 16-bit machine: |
339 | if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR; |
340 | |
341 | uncomp_stream_.next_out = dest; |
342 | uncomp_stream_.avail_out = (uInt)*destLen; |
343 | if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR; |
344 | |
345 | if ( !first_chunk_ ) // only need to set up stream the first time through |
346 | return Z_OK; |
347 | |
348 | if (uncomp_init_) { // we've already initted it |
349 | err = inflateReset(&uncomp_stream_); |
350 | if (err != Z_OK) { |
351 | LOG(WARNING) |
352 | << "ERROR: Can't reset uncompress object; creating a new one" ; |
353 | UncompressErrorInit(); |
354 | } |
355 | } |
356 | if (!uncomp_init_) { |
357 | uncomp_stream_.zalloc = (alloc_func)0; |
358 | uncomp_stream_.zfree = (free_func)0; |
359 | uncomp_stream_.opaque = (voidpf)0; |
360 | err = InflateInit(); |
361 | if (err != Z_OK) return err; |
362 | uncomp_init_ = true; |
363 | } |
364 | return Z_OK; |
365 | } |
366 | |
367 | // If you compressed your data a chunk at a time, with CompressChunk, |
368 | // you can uncompress it a chunk at a time with UncompressChunk. |
369 | // Only difference bewteen chunked and unchunked uncompression |
370 | // is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked). |
371 | int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen, |
372 | const Bytef *source, uLong *sourceLen, |
373 | int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH |
374 | int err = Z_OK; |
375 | |
376 | if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) { |
377 | LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: " |
378 | << *sourceLen; |
379 | return err; |
380 | } |
381 | |
382 | // This is used to figure out how many output bytes we wrote *this chunk*: |
383 | const uLong old_total_out = uncomp_stream_.total_out; |
384 | |
385 | // This is used to figure out how many input bytes we read *this chunk*: |
386 | const uLong old_total_in = uncomp_stream_.total_in; |
387 | |
388 | // Some setup happens only for the first chunk we compress in a run |
389 | if ( first_chunk_ ) { |
390 | first_chunk_ = false; // so we don't do this again |
391 | |
392 | // For the first chunk *only* (to avoid infinite troubles), we let |
393 | // there be no actual data to uncompress. This sometimes triggers |
394 | // when the input is only the gzip header, say. |
395 | if ( *sourceLen == 0 ) { |
396 | *destLen = 0; |
397 | return Z_OK; |
398 | } |
399 | } |
400 | |
401 | // We'll uncompress as much as we can. If we end OK great, otherwise |
402 | // if we get an error that seems to be the gzip footer, we store the |
403 | // gzip footer and return OK, otherwise we return the error. |
404 | |
405 | // flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode. |
406 | err = inflate(&uncomp_stream_, flush_mode); |
407 | |
408 | // Figure out how many bytes of the input zlib slurped up: |
409 | const uLong bytes_read = uncomp_stream_.total_in - old_total_in; |
410 | CHECK_LE(source + bytes_read, source + *sourceLen); |
411 | *sourceLen = uncomp_stream_.avail_in; |
412 | |
413 | if ((err == Z_STREAM_END || err == Z_OK) // everything went ok |
414 | && uncomp_stream_.avail_in == 0) { // and we read it all |
415 | ; |
416 | } else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) { |
417 | LOG(WARNING) |
418 | << "UncompressChunkOrAll: Received some extra data, bytes total: " |
419 | << uncomp_stream_.avail_in << " bytes: " |
420 | << std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in), |
421 | std::min(int(uncomp_stream_.avail_in), 20)); |
422 | UncompressErrorInit(); |
423 | return Z_DATA_ERROR; // what's the extra data for? |
424 | } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) { |
425 | // an error happened |
426 | LOG(WARNING) << "UncompressChunkOrAll: Error: " << err |
427 | << " avail_out: " << uncomp_stream_.avail_out; |
428 | UncompressErrorInit(); |
429 | return err; |
430 | } else if (uncomp_stream_.avail_out == 0) { |
431 | err = Z_BUF_ERROR; |
432 | } |
433 | |
434 | assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END); |
435 | if (err == Z_STREAM_END) |
436 | err = Z_OK; |
437 | |
438 | *destLen = uncomp_stream_.total_out - old_total_out; // size for this call |
439 | |
440 | return err; |
441 | } |
442 | |
443 | int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen, |
444 | const Bytef *source, uLong sourceLen, |
445 | int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH |
446 | const int ret = |
447 | UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode); |
448 | if (ret == Z_BUF_ERROR) |
449 | UncompressErrorInit(); |
450 | return ret; |
451 | } |
452 | |
453 | int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen, |
454 | const Bytef *source, uLong *sourceLen) { |
455 | return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH); |
456 | } |
457 | |
458 | // We make sure we've uncompressed everything, that is, the current |
459 | // uncompress stream is at a compressed-buffer-EOF boundary. In gzip |
460 | // mode, we also check the gzip footer to make sure we pass the gzip |
461 | // consistency checks. We RETURN true iff both types of checks pass. |
462 | bool ZLib::UncompressChunkDone() { |
463 | assert(!first_chunk_ && uncomp_init_); |
464 | // Make sure we're at the end-of-compressed-data point. This means |
465 | // if we call inflate with Z_FINISH we won't consume any input or |
466 | // write any output |
467 | Bytef dummyin, dummyout; |
468 | uLongf dummylen = 0; |
469 | if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH) |
470 | != Z_OK ) { |
471 | return false; |
472 | } |
473 | |
474 | // Make sure that when we exit, we can start a new round of chunks later |
475 | Reset(); |
476 | |
477 | return true; |
478 | } |
479 | |
480 | // Uncompresses the source buffer into the destination buffer. |
481 | // The destination buffer must be long enough to hold the entire |
482 | // decompressed contents. |
483 | // |
484 | // We only initialize the uncomp_stream once. Thereafter, we use |
485 | // inflateReset, which should be faster. |
486 | // |
487 | // Returns Z_OK on success, otherwise, it returns a zlib error code. |
488 | int ZLib::Uncompress(Bytef *dest, uLongf *destLen, |
489 | const Bytef *source, uLong sourceLen) { |
490 | int err; |
491 | if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen, |
492 | Z_FINISH)) != Z_OK ) { |
493 | Reset(); // let us try to compress again |
494 | return err; |
495 | } |
496 | if ( !UncompressChunkDone() ) // calls Reset() |
497 | return Z_DATA_ERROR; |
498 | return Z_OK; // stream_end is ok |
499 | } |
500 | |
501 | #endif // HAVE_LIBZ |
502 | |
503 | } // namespace snappy |
504 | |