1// Copyright 2011 Google Inc. All Rights Reserved.
2//
3// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are
5// met:
6//
7// * Redistributions of source code must retain the above copyright
8// notice, this list of conditions and the following disclaimer.
9// * Redistributions in binary form must reproduce the above
10// copyright notice, this list of conditions and the following disclaimer
11// in the documentation and/or other materials provided with the
12// distribution.
13// * Neither the name of Google Inc. nor the names of its
14// contributors may be used to endorse or promote products derived from
15// this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29// Various stubs for the unit tests for the open-source version of Snappy.
30
31#include "snappy-test.h"
32
33#include <algorithm>
34#include <cstdarg>
35#include <cstdio>
36#include <cstdlib>
37#include <iostream>
38#include <string>
39
40namespace file {
41
42OptionsStub::OptionsStub() = default;
43OptionsStub::~OptionsStub() = default;
44
45const OptionsStub &Defaults() {
46 static OptionsStub defaults;
47 return defaults;
48}
49
50StatusStub::StatusStub() = default;
51StatusStub::StatusStub(const StatusStub &) = default;
52StatusStub &StatusStub::operator=(const StatusStub &) = default;
53StatusStub::~StatusStub() = default;
54
55bool StatusStub::ok() { return true; }
56
57StatusStub GetContents(const std::string &filename, std::string *output,
58 const OptionsStub & /* options */) {
59 std::FILE *fp = std::fopen(filename.c_str(), "rb");
60 if (fp == nullptr) {
61 std::perror(filename.c_str());
62 std::exit(1);
63 }
64
65 output->clear();
66 while (!std::feof(fp)) {
67 char buffer[4096];
68 size_t bytes_read = std::fread(buffer, 1, sizeof(buffer), fp);
69 if (bytes_read == 0 && std::ferror(fp)) {
70 std::perror("fread");
71 std::exit(1);
72 }
73 output->append(buffer, bytes_read);
74 }
75
76 std::fclose(fp);
77 return StatusStub();
78}
79
80StatusStub SetContents(const std::string &file_name, const std::string &content,
81 const OptionsStub & /* options */) {
82 std::FILE *fp = std::fopen(file_name.c_str(), "wb");
83 if (fp == nullptr) {
84 std::perror(file_name.c_str());
85 std::exit(1);
86 }
87
88 size_t bytes_written = std::fwrite(content.data(), 1, content.size(), fp);
89 if (bytes_written != content.size()) {
90 std::perror("fwrite");
91 std::exit(1);
92 }
93
94 std::fclose(fp);
95 return StatusStub();
96}
97
98} // namespace file
99
100namespace snappy {
101
102std::string ReadTestDataFile(const std::string& base, size_t size_limit) {
103 std::string contents;
104 const char* srcdir = getenv("srcdir"); // This is set by Automake.
105 std::string prefix;
106 if (srcdir) {
107 prefix = std::string(srcdir) + "/";
108 }
109 file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
110 ).ok();
111 if (size_limit > 0) {
112 contents = contents.substr(0, size_limit);
113 }
114 return contents;
115}
116
117std::string StrFormat(const char* format, ...) {
118 char buffer[4096];
119 std::va_list ap;
120 va_start(ap, format);
121 std::vsnprintf(buffer, sizeof(buffer), format, ap);
122 va_end(ap);
123 return buffer;
124}
125
126LogMessage::~LogMessage() { std::cerr << std::endl; }
127
128LogMessage &LogMessage::operator<<(const std::string &message) {
129 std::cerr << message;
130 return *this;
131}
132
133LogMessage &LogMessage::operator<<(int number) {
134 std::cerr << number;
135 return *this;
136}
137
138#ifdef _MSC_VER
139// ~LogMessageCrash calls std::abort() and therefore never exits. This is by
140// design, so temporarily disable warning C4722.
141#pragma warning(push)
142#pragma warning(disable : 4722)
143#endif
144
145LogMessageCrash::~LogMessageCrash() {
146 std::cerr << std::endl;
147 std::abort();
148}
149
150#ifdef _MSC_VER
151#pragma warning(pop)
152#endif
153
154#if HAVE_LIBZ
155
156ZLib::ZLib()
157 : comp_init_(false),
158 uncomp_init_(false) {
159 Reinit();
160}
161
162ZLib::~ZLib() {
163 if (comp_init_) { deflateEnd(&comp_stream_); }
164 if (uncomp_init_) { inflateEnd(&uncomp_stream_); }
165}
166
167void ZLib::Reinit() {
168 compression_level_ = Z_DEFAULT_COMPRESSION;
169 window_bits_ = MAX_WBITS;
170 mem_level_ = 8; // DEF_MEM_LEVEL
171 if (comp_init_) {
172 deflateEnd(&comp_stream_);
173 comp_init_ = false;
174 }
175 if (uncomp_init_) {
176 inflateEnd(&uncomp_stream_);
177 uncomp_init_ = false;
178 }
179 first_chunk_ = true;
180}
181
182void ZLib::Reset() {
183 first_chunk_ = true;
184}
185
186// --------- COMPRESS MODE
187
188// Initialization method to be called if we hit an error while
189// compressing. On hitting an error, call this method before returning
190// the error.
191void ZLib::CompressErrorInit() {
192 deflateEnd(&comp_stream_);
193 comp_init_ = false;
194 Reset();
195}
196
197int ZLib::DeflateInit() {
198 return deflateInit2(&comp_stream_,
199 compression_level_,
200 Z_DEFLATED,
201 window_bits_,
202 mem_level_,
203 Z_DEFAULT_STRATEGY);
204}
205
206int ZLib::CompressInit(Bytef *dest, uLongf *destLen,
207 const Bytef *source, uLong *sourceLen) {
208 int err;
209
210 comp_stream_.next_in = (Bytef*)source;
211 comp_stream_.avail_in = (uInt)*sourceLen;
212 if ((uLong)comp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
213 comp_stream_.next_out = dest;
214 comp_stream_.avail_out = (uInt)*destLen;
215 if ((uLong)comp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
216
217 if ( !first_chunk_ ) // only need to set up stream the first time through
218 return Z_OK;
219
220 if (comp_init_) { // we've already initted it
221 err = deflateReset(&comp_stream_);
222 if (err != Z_OK) {
223 LOG(WARNING) << "ERROR: Can't reset compress object; creating a new one";
224 deflateEnd(&comp_stream_);
225 comp_init_ = false;
226 }
227 }
228 if (!comp_init_) { // first use
229 comp_stream_.zalloc = (alloc_func)0;
230 comp_stream_.zfree = (free_func)0;
231 comp_stream_.opaque = (voidpf)0;
232 err = DeflateInit();
233 if (err != Z_OK) return err;
234 comp_init_ = true;
235 }
236 return Z_OK;
237}
238
239// In a perfect world we'd always have the full buffer to compress
240// when the time came, and we could just call Compress(). Alas, we
241// want to do chunked compression on our webserver. In this
242// application, we compress the header, send it off, then compress the
243// results, send them off, then compress the footer. Thus we need to
244// use the chunked compression features of zlib.
245int ZLib::CompressAtMostOrAll(Bytef *dest, uLongf *destLen,
246 const Bytef *source, uLong *sourceLen,
247 int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
248 int err;
249
250 if ( (err=CompressInit(dest, destLen, source, sourceLen)) != Z_OK )
251 return err;
252
253 // This is used to figure out how many bytes we wrote *this chunk*
254 int compressed_size = comp_stream_.total_out;
255
256 // Some setup happens only for the first chunk we compress in a run
257 if ( first_chunk_ ) {
258 first_chunk_ = false;
259 }
260
261 // flush_mode is Z_FINISH for all mode, Z_SYNC_FLUSH for incremental
262 // compression.
263 err = deflate(&comp_stream_, flush_mode);
264
265 *sourceLen = comp_stream_.avail_in;
266
267 if ((err == Z_STREAM_END || err == Z_OK)
268 && comp_stream_.avail_in == 0
269 && comp_stream_.avail_out != 0 ) {
270 // we processed everything ok and the output buffer was large enough.
271 ;
272 } else if (err == Z_STREAM_END && comp_stream_.avail_in > 0) {
273 return Z_BUF_ERROR; // should never happen
274 } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
275 // an error happened
276 CompressErrorInit();
277 return err;
278 } else if (comp_stream_.avail_out == 0) { // not enough space
279 err = Z_BUF_ERROR;
280 }
281
282 assert(err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR);
283 if (err == Z_STREAM_END)
284 err = Z_OK;
285
286 // update the crc and other metadata
287 compressed_size = comp_stream_.total_out - compressed_size; // delta
288 *destLen = compressed_size;
289
290 return err;
291}
292
293int ZLib::CompressChunkOrAll(Bytef *dest, uLongf *destLen,
294 const Bytef *source, uLong sourceLen,
295 int flush_mode) { // Z_FULL_FLUSH or Z_FINISH
296 const int ret =
297 CompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
298 if (ret == Z_BUF_ERROR)
299 CompressErrorInit();
300 return ret;
301}
302
303// This routine only initializes the compression stream once. Thereafter, it
304// just does a deflateReset on the stream, which should be faster.
305int ZLib::Compress(Bytef *dest, uLongf *destLen,
306 const Bytef *source, uLong sourceLen) {
307 int err;
308 if ( (err=CompressChunkOrAll(dest, destLen, source, sourceLen,
309 Z_FINISH)) != Z_OK )
310 return err;
311 Reset(); // reset for next call to Compress
312
313 return Z_OK;
314}
315
316
317// --------- UNCOMPRESS MODE
318
319int ZLib::InflateInit() {
320 return inflateInit2(&uncomp_stream_, MAX_WBITS);
321}
322
323// Initialization method to be called if we hit an error while
324// uncompressing. On hitting an error, call this method before
325// returning the error.
326void ZLib::UncompressErrorInit() {
327 inflateEnd(&uncomp_stream_);
328 uncomp_init_ = false;
329 Reset();
330}
331
332int ZLib::UncompressInit(Bytef *dest, uLongf *destLen,
333 const Bytef *source, uLong *sourceLen) {
334 int err;
335
336 uncomp_stream_.next_in = (Bytef*)source;
337 uncomp_stream_.avail_in = (uInt)*sourceLen;
338 // Check for source > 64K on 16-bit machine:
339 if ((uLong)uncomp_stream_.avail_in != *sourceLen) return Z_BUF_ERROR;
340
341 uncomp_stream_.next_out = dest;
342 uncomp_stream_.avail_out = (uInt)*destLen;
343 if ((uLong)uncomp_stream_.avail_out != *destLen) return Z_BUF_ERROR;
344
345 if ( !first_chunk_ ) // only need to set up stream the first time through
346 return Z_OK;
347
348 if (uncomp_init_) { // we've already initted it
349 err = inflateReset(&uncomp_stream_);
350 if (err != Z_OK) {
351 LOG(WARNING)
352 << "ERROR: Can't reset uncompress object; creating a new one";
353 UncompressErrorInit();
354 }
355 }
356 if (!uncomp_init_) {
357 uncomp_stream_.zalloc = (alloc_func)0;
358 uncomp_stream_.zfree = (free_func)0;
359 uncomp_stream_.opaque = (voidpf)0;
360 err = InflateInit();
361 if (err != Z_OK) return err;
362 uncomp_init_ = true;
363 }
364 return Z_OK;
365}
366
367// If you compressed your data a chunk at a time, with CompressChunk,
368// you can uncompress it a chunk at a time with UncompressChunk.
369// Only difference bewteen chunked and unchunked uncompression
370// is the flush mode we use: Z_SYNC_FLUSH (chunked) or Z_FINISH (unchunked).
371int ZLib::UncompressAtMostOrAll(Bytef *dest, uLongf *destLen,
372 const Bytef *source, uLong *sourceLen,
373 int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
374 int err = Z_OK;
375
376 if ( (err=UncompressInit(dest, destLen, source, sourceLen)) != Z_OK ) {
377 LOG(WARNING) << "UncompressInit: Error: " << err << " SourceLen: "
378 << *sourceLen;
379 return err;
380 }
381
382 // This is used to figure out how many output bytes we wrote *this chunk*:
383 const uLong old_total_out = uncomp_stream_.total_out;
384
385 // This is used to figure out how many input bytes we read *this chunk*:
386 const uLong old_total_in = uncomp_stream_.total_in;
387
388 // Some setup happens only for the first chunk we compress in a run
389 if ( first_chunk_ ) {
390 first_chunk_ = false; // so we don't do this again
391
392 // For the first chunk *only* (to avoid infinite troubles), we let
393 // there be no actual data to uncompress. This sometimes triggers
394 // when the input is only the gzip header, say.
395 if ( *sourceLen == 0 ) {
396 *destLen = 0;
397 return Z_OK;
398 }
399 }
400
401 // We'll uncompress as much as we can. If we end OK great, otherwise
402 // if we get an error that seems to be the gzip footer, we store the
403 // gzip footer and return OK, otherwise we return the error.
404
405 // flush_mode is Z_SYNC_FLUSH for chunked mode, Z_FINISH for all mode.
406 err = inflate(&uncomp_stream_, flush_mode);
407
408 // Figure out how many bytes of the input zlib slurped up:
409 const uLong bytes_read = uncomp_stream_.total_in - old_total_in;
410 CHECK_LE(source + bytes_read, source + *sourceLen);
411 *sourceLen = uncomp_stream_.avail_in;
412
413 if ((err == Z_STREAM_END || err == Z_OK) // everything went ok
414 && uncomp_stream_.avail_in == 0) { // and we read it all
415 ;
416 } else if (err == Z_STREAM_END && uncomp_stream_.avail_in > 0) {
417 LOG(WARNING)
418 << "UncompressChunkOrAll: Received some extra data, bytes total: "
419 << uncomp_stream_.avail_in << " bytes: "
420 << std::string(reinterpret_cast<const char *>(uncomp_stream_.next_in),
421 std::min(int(uncomp_stream_.avail_in), 20));
422 UncompressErrorInit();
423 return Z_DATA_ERROR; // what's the extra data for?
424 } else if (err != Z_OK && err != Z_STREAM_END && err != Z_BUF_ERROR) {
425 // an error happened
426 LOG(WARNING) << "UncompressChunkOrAll: Error: " << err
427 << " avail_out: " << uncomp_stream_.avail_out;
428 UncompressErrorInit();
429 return err;
430 } else if (uncomp_stream_.avail_out == 0) {
431 err = Z_BUF_ERROR;
432 }
433
434 assert(err == Z_OK || err == Z_BUF_ERROR || err == Z_STREAM_END);
435 if (err == Z_STREAM_END)
436 err = Z_OK;
437
438 *destLen = uncomp_stream_.total_out - old_total_out; // size for this call
439
440 return err;
441}
442
443int ZLib::UncompressChunkOrAll(Bytef *dest, uLongf *destLen,
444 const Bytef *source, uLong sourceLen,
445 int flush_mode) { // Z_SYNC_FLUSH or Z_FINISH
446 const int ret =
447 UncompressAtMostOrAll(dest, destLen, source, &sourceLen, flush_mode);
448 if (ret == Z_BUF_ERROR)
449 UncompressErrorInit();
450 return ret;
451}
452
453int ZLib::UncompressAtMost(Bytef *dest, uLongf *destLen,
454 const Bytef *source, uLong *sourceLen) {
455 return UncompressAtMostOrAll(dest, destLen, source, sourceLen, Z_SYNC_FLUSH);
456}
457
458// We make sure we've uncompressed everything, that is, the current
459// uncompress stream is at a compressed-buffer-EOF boundary. In gzip
460// mode, we also check the gzip footer to make sure we pass the gzip
461// consistency checks. We RETURN true iff both types of checks pass.
462bool ZLib::UncompressChunkDone() {
463 assert(!first_chunk_ && uncomp_init_);
464 // Make sure we're at the end-of-compressed-data point. This means
465 // if we call inflate with Z_FINISH we won't consume any input or
466 // write any output
467 Bytef dummyin, dummyout;
468 uLongf dummylen = 0;
469 if ( UncompressChunkOrAll(&dummyout, &dummylen, &dummyin, 0, Z_FINISH)
470 != Z_OK ) {
471 return false;
472 }
473
474 // Make sure that when we exit, we can start a new round of chunks later
475 Reset();
476
477 return true;
478}
479
480// Uncompresses the source buffer into the destination buffer.
481// The destination buffer must be long enough to hold the entire
482// decompressed contents.
483//
484// We only initialize the uncomp_stream once. Thereafter, we use
485// inflateReset, which should be faster.
486//
487// Returns Z_OK on success, otherwise, it returns a zlib error code.
488int ZLib::Uncompress(Bytef *dest, uLongf *destLen,
489 const Bytef *source, uLong sourceLen) {
490 int err;
491 if ( (err=UncompressChunkOrAll(dest, destLen, source, sourceLen,
492 Z_FINISH)) != Z_OK ) {
493 Reset(); // let us try to compress again
494 return err;
495 }
496 if ( !UncompressChunkDone() ) // calls Reset()
497 return Z_DATA_ERROR;
498 return Z_OK; // stream_end is ok
499}
500
501#endif // HAVE_LIBZ
502
503} // namespace snappy
504