1// Copyright 2005 and onwards Google Inc.
2//
3// Redistribution and use in source and binary forms, with or without
4// modification, are permitted provided that the following conditions are
5// met:
6//
7// * Redistributions of source code must retain the above copyright
8// notice, this list of conditions and the following disclaimer.
9// * Redistributions in binary form must reproduce the above
10// copyright notice, this list of conditions and the following disclaimer
11// in the documentation and/or other materials provided with the
12// distribution.
13// * Neither the name of Google Inc. nor the names of its
14// contributors may be used to endorse or promote products derived from
15// this software without specific prior written permission.
16//
17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29// A light-weight compression algorithm. It is designed for speed of
30// compression and decompression, rather than for the utmost in space
31// savings.
32//
33// For getting better compression ratios when you are compressing data
34// with long repeated sequences or compressing data that is similar to
35// other data, while still compressing fast, you might look at first
36// using BMDiff and then compressing the output of BMDiff with
37// Snappy.
38
39#ifndef THIRD_PARTY_SNAPPY_SNAPPY_H__
40#define THIRD_PARTY_SNAPPY_SNAPPY_H__
41
42#include <stddef.h>
43#include <stdint.h>
44
45#include <string>
46
47#include "snappy-stubs-public.h"
48
49namespace snappy {
50 class Source;
51 class Sink;
52
53 // ------------------------------------------------------------------------
54 // Generic compression/decompression routines.
55 // ------------------------------------------------------------------------
56
57 // Compress the bytes read from "*source" and append to "*sink". Return the
58 // number of bytes written.
59 size_t Compress(Source* source, Sink* sink);
60
61 // Find the uncompressed length of the given stream, as given by the header.
62 // Note that the true length could deviate from this; the stream could e.g.
63 // be truncated.
64 //
65 // Also note that this leaves "*source" in a state that is unsuitable for
66 // further operations, such as RawUncompress(). You will need to rewind
67 // or recreate the source yourself before attempting any further calls.
68 bool GetUncompressedLength(Source* source, uint32_t* result);
69
70 // ------------------------------------------------------------------------
71 // Higher-level string based routines (should be sufficient for most users)
72 // ------------------------------------------------------------------------
73
74 // Sets "*compressed" to the compressed version of "input[0..input_length-1]".
75 // Original contents of *compressed are lost.
76 //
77 // REQUIRES: "input[]" is not an alias of "*compressed".
78 size_t Compress(const char* input, size_t input_length,
79 std::string* compressed);
80
81 // Same as `Compress` above but taking an `iovec` array as input. Note that
82 // this function preprocesses the inputs to compute the sum of
83 // `iov[0..iov_cnt-1].iov_len` before reading. To avoid this, use
84 // `RawCompressFromIOVec` below.
85 size_t CompressFromIOVec(const struct iovec* iov, size_t iov_cnt,
86 std::string* compressed);
87
88 // Decompresses "compressed[0..compressed_length-1]" to "*uncompressed".
89 // Original contents of "*uncompressed" are lost.
90 //
91 // REQUIRES: "compressed[]" is not an alias of "*uncompressed".
92 //
93 // returns false if the message is corrupted and could not be decompressed
94 bool Uncompress(const char* compressed, size_t compressed_length,
95 std::string* uncompressed);
96
97 // Decompresses "compressed" to "*uncompressed".
98 //
99 // returns false if the message is corrupted and could not be decompressed
100 bool Uncompress(Source* compressed, Sink* uncompressed);
101
102 // This routine uncompresses as much of the "compressed" as possible
103 // into sink. It returns the number of valid bytes added to sink
104 // (extra invalid bytes may have been added due to errors; the caller
105 // should ignore those). The emitted data typically has length
106 // GetUncompressedLength(), but may be shorter if an error is
107 // encountered.
108 size_t UncompressAsMuchAsPossible(Source* compressed, Sink* uncompressed);
109
110 // ------------------------------------------------------------------------
111 // Lower-level character array based routines. May be useful for
112 // efficiency reasons in certain circumstances.
113 // ------------------------------------------------------------------------
114
115 // REQUIRES: "compressed" must point to an area of memory that is at
116 // least "MaxCompressedLength(input_length)" bytes in length.
117 //
118 // Takes the data stored in "input[0..input_length]" and stores
119 // it in the array pointed to by "compressed".
120 //
121 // "*compressed_length" is set to the length of the compressed output.
122 //
123 // Example:
124 // char* output = new char[snappy::MaxCompressedLength(input_length)];
125 // size_t output_length;
126 // RawCompress(input, input_length, output, &output_length);
127 // ... Process(output, output_length) ...
128 // delete [] output;
129 void RawCompress(const char* input,
130 size_t input_length,
131 char* compressed,
132 size_t* compressed_length);
133
134 // Same as `RawCompress` above but taking an `iovec` array as input. Note that
135 // `uncompressed_length` is the total number of bytes to be read from the
136 // elements of `iov` (_not_ the number of elements in `iov`).
137 void RawCompressFromIOVec(const struct iovec* iov, size_t uncompressed_length,
138 char* compressed, size_t* compressed_length);
139
140 // Given data in "compressed[0..compressed_length-1]" generated by
141 // calling the Snappy::Compress routine, this routine
142 // stores the uncompressed data to
143 // uncompressed[0..GetUncompressedLength(compressed)-1]
144 // returns false if the message is corrupted and could not be decrypted
145 bool RawUncompress(const char* compressed, size_t compressed_length,
146 char* uncompressed);
147
148 // Given data from the byte source 'compressed' generated by calling
149 // the Snappy::Compress routine, this routine stores the uncompressed
150 // data to
151 // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1]
152 // returns false if the message is corrupted and could not be decrypted
153 bool RawUncompress(Source* compressed, char* uncompressed);
154
155 // Given data in "compressed[0..compressed_length-1]" generated by
156 // calling the Snappy::Compress routine, this routine
157 // stores the uncompressed data to the iovec "iov". The number of physical
158 // buffers in "iov" is given by iov_cnt and their cumulative size
159 // must be at least GetUncompressedLength(compressed). The individual buffers
160 // in "iov" must not overlap with each other.
161 //
162 // returns false if the message is corrupted and could not be decrypted
163 bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
164 const struct iovec* iov, size_t iov_cnt);
165
166 // Given data from the byte source 'compressed' generated by calling
167 // the Snappy::Compress routine, this routine stores the uncompressed
168 // data to the iovec "iov". The number of physical
169 // buffers in "iov" is given by iov_cnt and their cumulative size
170 // must be at least GetUncompressedLength(compressed). The individual buffers
171 // in "iov" must not overlap with each other.
172 //
173 // returns false if the message is corrupted and could not be decrypted
174 bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
175 size_t iov_cnt);
176
177 // Returns the maximal size of the compressed representation of
178 // input data that is "source_bytes" bytes in length;
179 size_t MaxCompressedLength(size_t source_bytes);
180
181 // REQUIRES: "compressed[]" was produced by RawCompress() or Compress()
182 // Returns true and stores the length of the uncompressed data in
183 // *result normally. Returns false on parsing error.
184 // This operation takes O(1) time.
185 bool GetUncompressedLength(const char* compressed, size_t compressed_length,
186 size_t* result);
187
188 // Returns true iff the contents of "compressed[]" can be uncompressed
189 // successfully. Does not return the uncompressed data. Takes
190 // time proportional to compressed_length, but is usually at least
191 // a factor of four faster than actual decompression.
192 bool IsValidCompressedBuffer(const char* compressed,
193 size_t compressed_length);
194
195 // Returns true iff the contents of "compressed" can be uncompressed
196 // successfully. Does not return the uncompressed data. Takes
197 // time proportional to *compressed length, but is usually at least
198 // a factor of four faster than actual decompression.
199 // On success, consumes all of *compressed. On failure, consumes an
200 // unspecified prefix of *compressed.
201 bool IsValidCompressed(Source* compressed);
202
203 // The size of a compression block. Note that many parts of the compression
204 // code assumes that kBlockSize <= 65536; in particular, the hash table
205 // can only store 16-bit offsets, and EmitCopy() also assumes the offset
206 // is 65535 bytes or less. Note also that if you change this, it will
207 // affect the framing format (see framing_format.txt).
208 //
209 // Note that there might be older data around that is compressed with larger
210 // block sizes, so the decompression code should not rely on the
211 // non-existence of long backreferences.
212 static constexpr int kBlockLog = 16;
213 static constexpr size_t kBlockSize = 1 << kBlockLog;
214
215 static constexpr int kMinHashTableBits = 8;
216 static constexpr size_t kMinHashTableSize = 1 << kMinHashTableBits;
217
218 static constexpr int kMaxHashTableBits = 14;
219 static constexpr size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
220} // end namespace snappy
221
222#endif // THIRD_PARTY_SNAPPY_SNAPPY_H__
223