diff options
author | android-build-team Robot <android-build-team-robot@google.com> | 2018-03-04 08:21:55 +0000 |
---|---|---|
committer | android-build-team Robot <android-build-team-robot@google.com> | 2018-03-04 08:21:55 +0000 |
commit | 7c492ae6e97dadf21d9282e26711d318ccddfc9b (patch) | |
tree | 49ce7bafe0791566fb81b49291df98d9e7c1c329 | |
parent | 6c4897b199b42266165b67b95575fde3a4f87ce1 (diff) | |
parent | 7f8f0b1698dad1846a8ae3953b133b83617175bc (diff) | |
download | platform_external_puffin-android-wear-9.0.0_r4.tar.gz platform_external_puffin-android-wear-9.0.0_r4.tar.bz2 platform_external_puffin-android-wear-9.0.0_r4.zip |
Snap for 4632767 from 7f8f0b1698dad1846a8ae3953b133b83617175bc to pi-releaseandroid-wear-9.0.0_r9android-wear-9.0.0_r8android-wear-9.0.0_r7android-wear-9.0.0_r6android-wear-9.0.0_r5android-wear-9.0.0_r4android-wear-9.0.0_r3android-wear-9.0.0_r23android-wear-9.0.0_r22android-wear-9.0.0_r21android-wear-9.0.0_r20android-wear-9.0.0_r2android-wear-9.0.0_r19android-wear-9.0.0_r18android-wear-9.0.0_r17android-wear-9.0.0_r16android-wear-9.0.0_r15android-wear-9.0.0_r14android-wear-9.0.0_r13android-wear-9.0.0_r12android-wear-9.0.0_r11android-wear-9.0.0_r10android-wear-9.0.0_r1android-vts-9.0_r9android-vts-9.0_r8android-vts-9.0_r7android-vts-9.0_r6android-vts-9.0_r5android-vts-9.0_r4android-vts-9.0_r14android-vts-9.0_r13android-vts-9.0_r12android-vts-9.0_r11android-vts-9.0_r10android-cts-9.0_r9android-cts-9.0_r8android-cts-9.0_r7android-cts-9.0_r6android-cts-9.0_r5android-cts-9.0_r4android-cts-9.0_r3android-cts-9.0_r2android-cts-9.0_r13android-cts-9.0_r12android-cts-9.0_r11android-cts-9.0_r10android-cts-9.0_r1android-9.0.0_r9android-9.0.0_r8android-9.0.0_r7android-9.0.0_r60android-9.0.0_r6android-9.0.0_r59android-9.0.0_r58android-9.0.0_r57android-9.0.0_r56android-9.0.0_r55android-9.0.0_r54android-9.0.0_r53android-9.0.0_r52android-9.0.0_r51android-9.0.0_r50android-9.0.0_r5android-9.0.0_r49android-9.0.0_r48android-9.0.0_r3android-9.0.0_r2android-9.0.0_r18android-9.0.0_r17android-9.0.0_r10android-9.0.0_r1security-pi-releasepie-vts-releasepie-security-releasepie-s2-releasepie-release-2pie-releasepie-r2-s2-releasepie-r2-s1-releasepie-r2-releasepie-platform-releasepie-gsipie-cuttlefish-testingpie-cts-release
Change-Id: I105f712386d84661940a56284636bc6d63476b5f
-rw-r--r-- | src/bit_reader.cc | 2 | ||||
-rw-r--r-- | src/bit_reader.h | 8 | ||||
-rw-r--r-- | src/bit_writer.h | 4 | ||||
-rw-r--r-- | src/extent_stream.cc | 8 | ||||
-rw-r--r-- | src/extent_stream.h | 12 | ||||
-rw-r--r-- | src/file_stream.cc | 6 | ||||
-rw-r--r-- | src/file_stream.h | 6 | ||||
-rw-r--r-- | src/huffman_table.cc | 7 | ||||
-rw-r--r-- | src/include/puffin/stream.h | 6 | ||||
-rw-r--r-- | src/include/puffin/utils.h | 4 | ||||
-rw-r--r-- | src/main.cc | 26 | ||||
-rw-r--r-- | src/memory_stream.cc | 8 | ||||
-rw-r--r-- | src/memory_stream.h | 8 | ||||
-rw-r--r-- | src/puff_io_unittest.cc | 102 | ||||
-rw-r--r-- | src/puff_reader.cc | 14 | ||||
-rw-r--r-- | src/puff_writer.cc | 12 | ||||
-rw-r--r-- | src/puffdiff.cc | 10 | ||||
-rw-r--r-- | src/puffin_stream.cc | 62 | ||||
-rw-r--r-- | src/puffin_stream.h | 26 | ||||
-rw-r--r-- | src/puffin_unittest.cc | 2 | ||||
-rw-r--r-- | src/puffpatch.cc | 8 | ||||
-rw-r--r-- | src/stream_unittest.cc | 12 | ||||
-rw-r--r-- | src/unittest_common.h | 64 | ||||
-rw-r--r-- | src/utils.cc | 38 | ||||
-rw-r--r-- | src/utils_unittest.cc | 4 |
25 files changed, 264 insertions, 195 deletions
diff --git a/src/bit_reader.cc b/src/bit_reader.cc index 006870d..d6f6abb 100644 --- a/src/bit_reader.cc +++ b/src/bit_reader.cc @@ -64,7 +64,7 @@ size_t BufferBitReader::Offset() const { return index_ - in_cache_bits_ / 8; } -size_t BufferBitReader::OffsetInBits() const { +uint64_t BufferBitReader::OffsetInBits() const { return (index_ * 8) - in_cache_bits_; } diff --git a/src/bit_reader.h b/src/bit_reader.h index ee2124b..2d0c805 100644 --- a/src/bit_reader.h +++ b/src/bit_reader.h @@ -67,7 +67,7 @@ class BitReaderInterface { virtual size_t Offset() const = 0; // Returns the number of bits read (dropped) till now. - virtual size_t OffsetInBits() const = 0; + virtual uint64_t OffsetInBits() const = 0; }; // A raw buffer implementation of |BitReaderInterface|. @@ -96,12 +96,12 @@ class BufferBitReader : public BitReaderInterface { size_t length, std::function<bool(uint8_t* buffer, size_t count)>* read_fn) override; size_t Offset() const override; - size_t OffsetInBits() const override; + uint64_t OffsetInBits() const override; private: const uint8_t* in_buf_; // The input buffer. - size_t in_size_; // The number of bytes in |in_buf_|. - size_t index_; // The index to the next byte to be read. + uint64_t in_size_; // The number of bytes in |in_buf_|. + uint64_t index_; // The index to the next byte to be read. uint32_t in_cache_; // The temporary buffer to put input data into. size_t in_cache_bits_; // The number of bits available in |in_cache_|. diff --git a/src/bit_writer.h b/src/bit_writer.h index d7a608b..9e9bdb9 100644 --- a/src/bit_writer.h +++ b/src/bit_writer.h @@ -88,10 +88,10 @@ class BufferBitWriter : public BitWriterInterface { uint8_t* out_buf_; // The number of bytes in |out_buf_|. - size_t out_size_; + uint64_t out_size_; // The index to the next byte to write into. - size_t index_; + uint64_t index_; // A temporary buffer to keep the bits going out. uint32_t out_holder_; diff --git a/src/extent_stream.cc b/src/extent_stream.cc index e296ec1..d6d6060 100644 --- a/src/extent_stream.cc +++ b/src/extent_stream.cc @@ -34,7 +34,7 @@ ExtentStream::ExtentStream(UniqueStreamPtr stream, extents_upper_bounds_.reserve(extents_.size() + 1); extents_upper_bounds_.emplace_back(0); uint64_t total_size = 0; - size_t extent_end = 0; + uint64_t extent_end = 0; for (const auto& extent : extents_) { total_size += extent.length; extents_upper_bounds_.emplace_back(total_size); @@ -49,17 +49,17 @@ ExtentStream::ExtentStream(UniqueStreamPtr stream, cur_extent_ = extents_.begin(); } -bool ExtentStream::GetSize(size_t* size) const { +bool ExtentStream::GetSize(uint64_t* size) const { *size = size_; return true; } -bool ExtentStream::GetOffset(size_t* offset) const { +bool ExtentStream::GetOffset(uint64_t* offset) const { *offset = offset_; return true; } -bool ExtentStream::Seek(size_t offset) { +bool ExtentStream::Seek(uint64_t offset) { TEST_AND_RETURN_FALSE(offset <= size_); // The first item is zero and upper_bound never returns it because it always diff --git a/src/extent_stream.h b/src/extent_stream.h index 61a556b..455f800 100644 --- a/src/extent_stream.h +++ b/src/extent_stream.h @@ -26,9 +26,9 @@ class ExtentStream : public StreamInterface { const std::vector<ByteExtent>& extents); ~ExtentStream() override = default; - bool GetSize(size_t* size) const override; - bool GetOffset(size_t* offset) const override; - bool Seek(size_t offset) override; + bool GetSize(uint64_t* size) const override; + bool GetOffset(uint64_t* offset) const override; + bool Seek(uint64_t offset) override; bool Read(void* buffer, size_t length) override; bool Write(const void* buffer, size_t length) override; bool Close() override; @@ -54,17 +54,17 @@ class ExtentStream : public StreamInterface { std::vector<ByteExtent>::iterator cur_extent_; // The current offset in the current |ByteExtent| |cur_extent_|. - size_t cur_extent_offset_; + uint64_t cur_extent_offset_; // |True| if the stream is write only. |False| if the stream is read only. bool is_for_write_; // The size of the stream. It is actually the cumulative size of all the bytes // in |extents_|. - size_t size_; + uint64_t size_; // The current offset. - size_t offset_; + uint64_t offset_; // Used for proper and faster seeking. std::vector<uint64_t> extents_upper_bounds_; diff --git a/src/file_stream.cc b/src/file_stream.cc index 0192a98..2089078 100644 --- a/src/file_stream.cc +++ b/src/file_stream.cc @@ -34,7 +34,7 @@ UniqueStreamPtr FileStream::Open(const string& path, bool read, bool write) { return UniqueStreamPtr(new FileStream(fd)); } -bool FileStream::GetSize(size_t* size) const { +bool FileStream::GetSize(uint64_t* size) const { auto cur_off = lseek(fd_, 0, SEEK_CUR); TEST_AND_RETURN_FALSE(cur_off >= 0); auto fsize = lseek(fd_, 0, SEEK_END); @@ -45,14 +45,14 @@ bool FileStream::GetSize(size_t* size) const { return true; } -bool FileStream::GetOffset(size_t* offset) const { +bool FileStream::GetOffset(uint64_t* offset) const { auto off = lseek(fd_, 0, SEEK_CUR); TEST_AND_RETURN_FALSE(off >= 0); *offset = off; return true; } -bool FileStream::Seek(size_t offset) { +bool FileStream::Seek(uint64_t offset) { auto off = lseek(fd_, offset, SEEK_SET); TEST_AND_RETURN_FALSE(off == static_cast<off_t>(offset)); return true; diff --git a/src/file_stream.h b/src/file_stream.h index 156266b..626d7c6 100644 --- a/src/file_stream.h +++ b/src/file_stream.h @@ -21,9 +21,9 @@ class FileStream : public StreamInterface { static UniqueStreamPtr Open(const std::string& path, bool read, bool write); - bool GetSize(size_t* size) const override; - bool GetOffset(size_t* offset) const override; - bool Seek(size_t offset) override; + bool GetSize(uint64_t* size) const override; + bool GetOffset(uint64_t* offset) const override; + bool Seek(uint64_t offset) override; bool Read(void* buffer, size_t length) override; bool Write(const void* buffer, size_t length) override; bool Close() override; diff --git a/src/huffman_table.cc b/src/huffman_table.cc index 1edb268..6e711c6 100644 --- a/src/huffman_table.cc +++ b/src/huffman_table.cc @@ -65,13 +65,6 @@ bool HuffmanTable::InitHuffmanCodes(const Buffer& lens, size_t* max_bits) { } } - // No codes found! However, This is not invalid because you can have no - // length/distance codes in a block (all literals). - if (lens.size() == len_count_[0]) { - LOG(WARNING) - << "No non-zero lengths are given in the Huffman code Length array."; - } - // Check for oversubscribed code lengths. (A code with length 'L' cannot have // more than 2^L items. for (size_t idx = 1; idx <= *max_bits; idx++) { diff --git a/src/include/puffin/stream.h b/src/include/puffin/stream.h index a5062ad..ada9d06 100644 --- a/src/include/puffin/stream.h +++ b/src/include/puffin/stream.h @@ -18,15 +18,15 @@ class StreamInterface { virtual ~StreamInterface() = default; // Returns the size of the stream. - virtual bool GetSize(size_t* size) const = 0; + virtual bool GetSize(uint64_t* size) const = 0; // Returns the current offset in the stream where next read or write will // happen. - virtual bool GetOffset(size_t* offset) const = 0; + virtual bool GetOffset(uint64_t* offset) const = 0; // Sets the offset in the stream for the next read or write. On error // returns |false|. - virtual bool Seek(size_t offset) = 0; + virtual bool Seek(uint64_t offset) = 0; // Reads |length| bytes of data into |buffer|. On error, returns |false|. virtual bool Read(void* buffer, size_t length) = 0; diff --git a/src/include/puffin/utils.h b/src/include/puffin/utils.h index 65f8e63..4c81996 100644 --- a/src/include/puffin/utils.h +++ b/src/include/puffin/utils.h @@ -15,7 +15,7 @@ namespace puffin { // Counts the number of bytes in a list of |ByteExtent|s. PUFFIN_EXPORT -size_t BytesInByteExtents(const std::vector<ByteExtent>& extents); +uint64_t BytesInByteExtents(const std::vector<ByteExtent>& extents); // Converts an array of |ByteExtens| or |BitExtents| to a string. Each extent // has format "offset:length" and are comma separated. @@ -72,7 +72,7 @@ bool FindDeflateSubBlocks(const UniqueStreamPtr& src, bool FindPuffLocations(const UniqueStreamPtr& src, const std::vector<BitExtent>& deflates, std::vector<ByteExtent>* puffs, - size_t* out_puff_size); + uint64_t* out_puff_size); } // namespace puffin diff --git a/src/main.cc b/src/main.cc index dd57c05..c610764 100644 --- a/src/main.cc +++ b/src/main.cc @@ -61,7 +61,7 @@ vector<T> StringToExtents(const string& str) { return extents; } -const size_t kDefaultPuffCacheSize = 50 * 1024 * 1024; // 50 MB +const uint64_t kDefaultPuffCacheSize = 50 * 1024 * 1024; // 50 MB // An enum representing the type of compressed files. enum class FileType { kDeflate, kZlib, kGzip, kZip, kRaw, kUnknown }; @@ -123,7 +123,7 @@ bool LocateDeflatesBasedOnFileType(const UniqueStreamPtr& stream, return true; } - size_t stream_size; + uint64_t stream_size; TEST_AND_RETURN_FALSE(stream->GetSize(&stream_size)); if (file_type == FileType::kDeflate) { // Assume the whole stream is a deflate block. @@ -244,7 +244,7 @@ int main(int argc, char** argv) { -1); } TEST_AND_RETURN_VALUE(dst_puffs.empty(), -1); - size_t dst_puff_size; + uint64_t dst_puff_size; TEST_AND_RETURN_VALUE(FindPuffLocations(src_stream, src_deflates_bit, &dst_puffs, &dst_puff_size), -1); @@ -262,10 +262,10 @@ int main(int argc, char** argv) { : std::move(dst_stream); Buffer buffer(1024 * 1024); - size_t bytes_wrote = 0; + uint64_t bytes_wrote = 0; while (bytes_wrote < dst_puff_size) { - auto write_size = std::min( - buffer.size(), static_cast<size_t>(dst_puff_size - bytes_wrote)); + auto write_size = std::min(static_cast<uint64_t>(buffer.size()), + dst_puff_size - bytes_wrote); TEST_AND_RETURN_VALUE(reader->Read(buffer.data(), write_size), -1); TEST_AND_RETURN_VALUE(writer->Write(buffer.data(), write_size), -1); bytes_wrote += write_size; @@ -284,9 +284,10 @@ int main(int argc, char** argv) { std::move(dst_stream), huffer, dst_puff_size, dst_deflates_bit, src_puffs); - size_t bytes_read = 0; + uint64_t bytes_read = 0; while (bytes_read < dst_puff_size) { - auto read_size = std::min(buffer.size(), dst_puff_size - bytes_read); + auto read_size = std::min(static_cast<uint64_t>(buffer.size()), + dst_puff_size - bytes_read); TEST_AND_RETURN_VALUE(read_puff_stream->Read(buffer.data(), read_size), -1); TEST_AND_RETURN_VALUE(huff_writer->Write(buffer.data(), read_size), -1); @@ -299,7 +300,7 @@ int main(int argc, char** argv) { << ", is this intentional?"; } TEST_AND_RETURN_VALUE(src_puffs.size() == dst_deflates_bit.size(), -1); - size_t src_stream_size; + uint64_t src_stream_size; TEST_AND_RETURN_VALUE(src_stream->GetSize(&src_stream_size), -1); auto dst_file = FileStream::Open(FLAGS_dst_file, false, true); TEST_AND_RETURN_VALUE(dst_file, -1); @@ -310,9 +311,10 @@ int main(int argc, char** argv) { dst_deflates_bit, src_puffs); Buffer buffer(1024 * 1024); - size_t bytes_read = 0; + uint64_t bytes_read = 0; while (bytes_read < src_stream_size) { - auto read_size = std::min(buffer.size(), src_stream_size - bytes_read); + auto read_size = std::min(static_cast<uint64_t>(buffer.size()), + src_stream_size - bytes_read); TEST_AND_RETURN_VALUE(src_stream->Read(buffer.data(), read_size), -1); TEST_AND_RETURN_VALUE(dst_stream->Write(buffer.data(), read_size), -1); bytes_read += read_size; @@ -370,7 +372,7 @@ int main(int argc, char** argv) { } else if (FLAGS_operation == "puffpatch") { auto patch_stream = FileStream::Open(FLAGS_patch_file, true, false); TEST_AND_RETURN_VALUE(patch_stream, -1); - size_t patch_size; + uint64_t patch_size; TEST_AND_RETURN_VALUE(patch_stream->GetSize(&patch_size), -1); Buffer puffdiff_delta(patch_size); diff --git a/src/memory_stream.cc b/src/memory_stream.cc index bd3aa2d..5f32a90 100644 --- a/src/memory_stream.cc +++ b/src/memory_stream.cc @@ -29,20 +29,20 @@ MemoryStream::MemoryStream(const Buffer* read_memory, Buffer* write_memory) offset_(0), open_(true) {} -bool MemoryStream::GetSize(size_t* size) const { +bool MemoryStream::GetSize(uint64_t* size) const { *size = read_memory_ != nullptr ? read_memory_->size() : write_memory_->size(); return true; } -bool MemoryStream::GetOffset(size_t* offset) const { +bool MemoryStream::GetOffset(uint64_t* offset) const { *offset = offset_; return true; } -bool MemoryStream::Seek(size_t offset) { +bool MemoryStream::Seek(uint64_t offset) { TEST_AND_RETURN_FALSE(open_); - size_t size; + uint64_t size; GetSize(&size); TEST_AND_RETURN_FALSE(offset <= size); offset_ = offset; diff --git a/src/memory_stream.h b/src/memory_stream.h index 406d38d..a338e7f 100644 --- a/src/memory_stream.h +++ b/src/memory_stream.h @@ -26,9 +26,9 @@ class MemoryStream : public StreamInterface { // the |memory|. static UniqueStreamPtr CreateForWrite(Buffer* memory); - bool GetSize(size_t* size) const override; - bool GetOffset(size_t* offset) const override; - bool Seek(size_t offset) override; + bool GetSize(uint64_t* size) const override; + bool GetOffset(uint64_t* offset) const override; + bool Seek(uint64_t offset) override; bool Read(void* buffer, size_t length) override; bool Write(const void* buffer, size_t length) override; bool Close() override; @@ -44,7 +44,7 @@ class MemoryStream : public StreamInterface { Buffer* write_memory_; // The current offset. - size_t offset_; + uint64_t offset_; // True if the stream is open. bool open_; diff --git a/src/puff_io_unittest.cc b/src/puff_io_unittest.cc index 628fd87..64976bd 100644 --- a/src/puff_io_unittest.cc +++ b/src/puff_io_unittest.cc @@ -224,35 +224,99 @@ TEST(PuffIOTest, InputOutputTest) { ASSERT_EQ(buf.size() - pr.BytesLeft(), epw.Size()); } -// Testing boundary -TEST(PuffIOTest, BoundaryTest) { - Buffer buf(5); +// Testing metadata boundary. +TEST(PuffIOTest, MetadataBoundaryTest) { PuffData pd; Error error; - + Buffer buf(3); BufferPuffWriter pw(buf.data(), buf.size()); - uint8_t block[] = {10, 11, 12}; + + // Block metadata takes two + varied bytes, so on a thre byte buffer, only one + // bytes is left for the varied part of metadata. pd.type = PuffData::Type::kBlockMetadata; - memcpy(pd.block_metadata, block, sizeof(block)); - pd.length = sizeof(block) + 1; + pd.length = 2; ASSERT_FALSE(pw.Insert(pd, &error)); ASSERT_EQ(error, Error::kInsufficientOutput); + pd.length = 0; // length should be at least 1. + ASSERT_FALSE(pw.Insert(pd, &error)); + pd.length = 1; + ASSERT_TRUE(pw.Insert(pd, &error)); - BufferPuffWriter pw2(buf.data(), buf.size()); - pd.length = sizeof(block); - ASSERT_TRUE(pw2.Insert(pd, &error)); + Buffer puff_buffer = {0x00, 0x03, 0x02, 0x00, 0x00}; + BufferPuffReader pr(puff_buffer.data(), puff_buffer.size()); + ASSERT_FALSE(pr.GetNext(&pd, &error)); +} - BufferPuffReader pr(buf.data(), buf.size()); - ASSERT_TRUE(pr.GetNext(&pd, &error)); - ASSERT_EQ(pd.type, PuffData::Type::kBlockMetadata); - ASSERT_EQ(pd.length, sizeof(block)); - ASSERT_EQ(pd.block_metadata[0], 10); +TEST(PuffIOTest, InvalidCopyLengthsDistanceTest) { + PuffData pd; + Error error; + Buffer puff_buffer(20); + BufferPuffWriter pw(puff_buffer.data(), puff_buffer.size()); + + // Invalid Lenght values. + pd.type = PuffData::Type::kLenDist; + pd.distance = 1; + pd.length = 0; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.length = 1; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.length = 2; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.length = 3; + EXPECT_TRUE(pw.Insert(pd, &error)); + pd.length = 259; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.length = 258; + EXPECT_TRUE(pw.Insert(pd, &error)); + + // Invalid distance values. + pd.length = 3; + pd.distance = 0; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.distance = 1; + EXPECT_TRUE(pw.Insert(pd, &error)); + pd.distance = 32769; + EXPECT_FALSE(pw.Insert(pd, &error)); + pd.distance = 32768; + EXPECT_TRUE(pw.Insert(pd, &error)); + + // First three bytes header, four bytes value lit/len, and four bytes + // invalid lit/len. + puff_buffer = {0x00, 0x00, 0xFF, 0xFF, 0x80, 0x00, + 0x00, 0xFF, 0x82, 0x00, 0x00}; + BufferPuffReader pr(puff_buffer.data(), puff_buffer.size()); + EXPECT_TRUE(pr.GetNext(&pd, &error)); + EXPECT_EQ(pd.type, PuffData::Type::kBlockMetadata); + EXPECT_TRUE(pr.GetNext(&pd, &error)); + EXPECT_EQ(pd.type, PuffData::Type::kLenDist); + EXPECT_FALSE(pr.GetNext(&pd, &error)); +} - BufferPuffReader pr2(buf.data(), sizeof(block)); - ASSERT_FALSE(pr2.GetNext(&pd, &error)); - ASSERT_EQ(error, Error::kInsufficientInput); +TEST(PuffIOTest, InvalidCopyLenghtDistanceBoundaryTest) { + PuffData pd; + Error error; + Buffer puff_buffer(5); + + pd.type = PuffData::Type::kLenDist; + pd.distance = 1; + pd.length = 129; + for (size_t i = 1; i < 2; i++) { + BufferPuffWriter pw(puff_buffer.data(), i); + EXPECT_FALSE(pw.Insert(pd, &error)); + } + + pd.length = 130; + for (size_t i = 1; i < 3; i++) { + BufferPuffWriter pw(puff_buffer.data(), i); + EXPECT_FALSE(pw.Insert(pd, &error)); + } - // TODO(ahassani): Boundary check for literals and lendist. + // First three bytes header, three bytes value lit/len. + puff_buffer = {0x00, 0x00, 0xFF, 0xFF, 0x80, 0x00}; + BufferPuffReader pr(puff_buffer.data(), puff_buffer.size()); + EXPECT_TRUE(pr.GetNext(&pd, &error)); + EXPECT_EQ(pd.type, PuffData::Type::kBlockMetadata); + EXPECT_FALSE(pr.GetNext(&pd, &error)); } TEST(PuffIOTest, LiteralsTest) { diff --git a/src/puff_reader.cc b/src/puff_reader.cc index 1365e54..34ef5c3 100644 --- a/src/puff_reader.cc +++ b/src/puff_reader.cc @@ -24,10 +24,10 @@ bool BufferPuffReader::GetNext(PuffData* data, Error* error) { PuffData& pd = *data; size_t length = 0; if (state_ == State::kReadingLenDist) { + // Boundary check + TEST_AND_RETURN_FALSE_SET_ERROR(index_ < puff_size_, + Error::kInsufficientInput); if (puff_buf_in_[index_] & 0x80) { // Reading length/distance. - // Boundary check - TEST_AND_RETURN_FALSE_SET_ERROR(index_ < puff_size_, - Error::kInsufficientInput); if ((puff_buf_in_[index_] & 0x7F) < 127) { length = puff_buf_in_[index_] & 0x7F; } else { @@ -38,6 +38,8 @@ bool BufferPuffReader::GetNext(PuffData* data, Error* error) { length = puff_buf_in_[index_] + 127; } length += 3; + TEST_AND_RETURN_FALSE(length <= 259); + index_++; // End of block. End of block is similar to length/distance but without @@ -53,9 +55,13 @@ bool BufferPuffReader::GetNext(PuffData* data, Error* error) { TEST_AND_RETURN_FALSE_SET_ERROR(index_ + 1 < puff_size_, Error::kInsufficientInput); auto distance = ReadByteArrayToUint16(&puff_buf_in_[index_]); + // The distance in RFC is in the range [1..32768], but in the puff spec, + // we write zero-based distance in the puff stream. + TEST_AND_RETURN_FALSE_SET_ERROR(distance < (1 << 15), + Error::kInsufficientInput); + distance++; index_ += 2; - TEST_AND_RETURN_FALSE(length < 259); pd.type = PuffData::Type::kLenDist; pd.length = length; pd.distance = distance; diff --git a/src/puff_writer.cc b/src/puff_writer.cc index 648bbe5..f39dad9 100644 --- a/src/puff_writer.cc +++ b/src/puff_writer.cc @@ -85,7 +85,10 @@ bool BufferPuffWriter::Insert(const PuffData& pd, Error* error) { case PuffData::Type::kLenDist: DVLOG(2) << "Write length: " << pd.length << " distance: " << pd.distance; TEST_AND_RETURN_FALSE(FlushLiterals(error)); - TEST_AND_RETURN_FALSE_SET_ERROR(pd.length < 259, Error::kInvalidInput); + TEST_AND_RETURN_FALSE_SET_ERROR(pd.length <= 258 && pd.length >= 3, + Error::kInvalidInput); + TEST_AND_RETURN_FALSE_SET_ERROR(pd.distance <= 32768 && pd.distance >= 1, + Error::kInvalidInput); if (pd.length < 130) { if (puff_buf_out_ != nullptr) { // Boundary check @@ -111,7 +114,8 @@ bool BufferPuffWriter::Insert(const PuffData& pd, Error* error) { } if (puff_buf_out_ != nullptr) { - WriteUint16ToByteArray(pd.distance, &puff_buf_out_[index_]); + // Write the distance in the range [1..32768] zero-based. + WriteUint16ToByteArray(pd.distance - 1, &puff_buf_out_[index_]); } index_ += 2; len_index_ = index_; @@ -121,6 +125,9 @@ bool BufferPuffWriter::Insert(const PuffData& pd, Error* error) { case PuffData::Type::kBlockMetadata: DVLOG(2) << "Write block metadata length: " << pd.length; TEST_AND_RETURN_FALSE(FlushLiterals(error)); + TEST_AND_RETURN_FALSE_SET_ERROR( + pd.length <= sizeof(pd.block_metadata) && pd.length > 0, + Error::kInvalidInput); if (puff_buf_out_ != nullptr) { // Boundary check TEST_AND_RETURN_FALSE_SET_ERROR(index_ + pd.length + 2 <= puff_size_, @@ -130,7 +137,6 @@ bool BufferPuffWriter::Insert(const PuffData& pd, Error* error) { } index_ += 2; - TEST_AND_RETURN_FALSE(pd.length <= sizeof(pd.block_metadata)); if (puff_buf_out_ != nullptr) { memcpy(&puff_buf_out_[index_], pd.block_metadata, pd.length); } diff --git a/src/puffdiff.cc b/src/puffdiff.cc index fd3c4c8..7d02085 100644 --- a/src/puffdiff.cc +++ b/src/puffdiff.cc @@ -52,8 +52,8 @@ bool CreatePatch(const Buffer& bsdiff_patch, const vector<BitExtent>& dst_deflates, const vector<ByteExtent>& src_puffs, const vector<ByteExtent>& dst_puffs, - size_t src_puff_size, - size_t dst_puff_size, + uint64_t src_puff_size, + uint64_t dst_puff_size, Buffer* patch) { metadata::PatchHeader header; header.set_version(1); @@ -68,7 +68,7 @@ bool CreatePatch(const Buffer& bsdiff_patch, const uint32_t header_size = header.ByteSize(); - size_t offset = 0; + uint64_t offset = 0; patch->resize(kMagicLength + sizeof(header_size) + header_size + bsdiff_patch.size()); @@ -104,7 +104,7 @@ bool PuffDiff(UniqueStreamPtr src, auto puff_deflate_stream = [&puffer](UniqueStreamPtr stream, const vector<BitExtent>& deflates, Buffer* puff_buffer, vector<ByteExtent>* puffs) { - size_t puff_size; + uint64_t puff_size; TEST_AND_RETURN_FALSE(stream->Seek(0)); TEST_AND_RETURN_FALSE( FindPuffLocations(stream, deflates, puffs, &puff_size)); @@ -132,7 +132,7 @@ bool PuffDiff(UniqueStreamPtr src, auto bsdiff_patch = FileStream::Open(tmp_filepath, true, false); TEST_AND_RETURN_FALSE(bsdiff_patch); - size_t patch_size; + uint64_t patch_size; TEST_AND_RETURN_FALSE(bsdiff_patch->GetSize(&patch_size)); Buffer bsdiff_patch_buf(patch_size); TEST_AND_RETURN_FALSE( diff --git a/src/puffin_stream.cc b/src/puffin_stream.cc index 3fb54a1..05dd7f6 100644 --- a/src/puffin_stream.cc +++ b/src/puffin_stream.cc @@ -28,7 +28,7 @@ using std::shared_ptr; namespace { -bool CheckArgsIntegrity(size_t puff_size, +bool CheckArgsIntegrity(uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs) { TEST_AND_RETURN_FALSE(puffs.size() == deflates.size()); @@ -57,7 +57,7 @@ bool CheckArgsIntegrity(size_t puff_size, UniqueStreamPtr PuffinStream::CreateForPuff( UniqueStreamPtr stream, std::shared_ptr<Puffer> puffer, - size_t puff_size, + uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs, size_t max_cache_size) { @@ -75,7 +75,7 @@ UniqueStreamPtr PuffinStream::CreateForPuff( UniqueStreamPtr PuffinStream::CreateForHuff( UniqueStreamPtr stream, std::shared_ptr<Huffer> huffer, - size_t puff_size, + uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs) { TEST_AND_RETURN_VALUE(CheckArgsIntegrity(puff_size, deflates, puffs), @@ -91,7 +91,7 @@ UniqueStreamPtr PuffinStream::CreateForHuff( PuffinStream::PuffinStream(UniqueStreamPtr stream, shared_ptr<Puffer> puffer, shared_ptr<Huffer> huffer, - size_t puff_size, + uint64_t puff_size, const vector<BitExtent>& deflates, const vector<ByteExtent>& puffs, size_t max_cache_size) @@ -120,7 +120,7 @@ PuffinStream::PuffinStream(UniqueStreamPtr stream, // We can pass the size of the deflate stream too, but it is not necessary // yet. We cannot get the size of stream from itself, because we might be // writing into it and its size is not defined yet. - size_t deflate_stream_size = puff_stream_size_; + uint64_t deflate_stream_size = puff_stream_size_; if (!puffs.empty()) { deflate_stream_size = ((deflates.back().offset + deflates.back().length) / 8) + @@ -131,35 +131,33 @@ PuffinStream::PuffinStream(UniqueStreamPtr stream, puffs_.emplace_back(puff_stream_size_, 0); // Look for the largest puff and deflate extents and get proper size buffers. - size_t max_puff_length = 0; + uint64_t max_puff_length = 0; for (const auto& puff : puffs) { - max_puff_length = - std::max(max_puff_length, static_cast<size_t>(puff.length)); + max_puff_length = std::max(max_puff_length, puff.length); } puff_buffer_.reset(new Buffer(max_puff_length + 1)); if (max_cache_size_ < max_puff_length) { max_cache_size_ = 0; // It means we are not caching puffs. } - size_t max_deflate_length = 0; + uint64_t max_deflate_length = 0; for (const auto& deflate : deflates) { - max_deflate_length = - std::max(max_deflate_length, static_cast<size_t>(deflate.length * 8)); + max_deflate_length = std::max(max_deflate_length, deflate.length * 8); } deflate_buffer_.reset(new Buffer(max_deflate_length + 2)); } -bool PuffinStream::GetSize(size_t* size) const { +bool PuffinStream::GetSize(uint64_t* size) const { *size = puff_stream_size_; return true; } -bool PuffinStream::GetOffset(size_t* offset) const { +bool PuffinStream::GetOffset(uint64_t* offset) const { *offset = puff_pos_ + skip_bytes_; return true; } -bool PuffinStream::Seek(size_t offset) { +bool PuffinStream::Seek(uint64_t offset) { TEST_AND_RETURN_FALSE(!closed_); if (!is_for_puff_) { // For huffing we should not seek, only seek to zero is accepted. @@ -206,22 +204,23 @@ bool PuffinStream::Close() { return stream_->Close(); } -bool PuffinStream::Read(void* buffer, size_t length) { +bool PuffinStream::Read(void* buffer, size_t count) { TEST_AND_RETURN_FALSE(!closed_); TEST_AND_RETURN_FALSE(is_for_puff_); if (cur_puff_ == puffs_.end()) { - TEST_AND_RETURN_FALSE(length == 0); + TEST_AND_RETURN_FALSE(count == 0); } auto bytes = static_cast<uint8_t*>(buffer); - size_t bytes_read = 0; + uint64_t length = count; + uint64_t bytes_read = 0; while (bytes_read < length) { if (puff_pos_ < cur_puff_->offset) { // Reading between two deflates. We also read bytes that have at least one // bit of a deflate bit stream. The byte which has both deflate and raw // data will be shifted or masked off the deflate bits and the remaining // value will be saved in the puff stream as an byte integer. - size_t start_byte = (deflate_bit_pos_ / 8); - size_t end_byte = (cur_deflate_->offset + 7) / 8; + uint64_t start_byte = (deflate_bit_pos_ / 8); + uint64_t end_byte = (cur_deflate_->offset + 7) / 8; auto bytes_to_read = std::min(length - bytes_read, end_byte - start_byte); TEST_AND_RETURN_FALSE(bytes_to_read >= 1); @@ -300,8 +299,7 @@ bool PuffinStream::Read(void* buffer, size_t length) { } // Copy from puff buffer to output if needed. auto bytes_to_copy = - std::min(length - bytes_read, - static_cast<size_t>(cur_puff_->length) - skip_bytes_); + std::min(length - bytes_read, cur_puff_->length - skip_bytes_); if (!puff_directly_into_buffer) { memcpy(bytes + bytes_read, puff_buffer_->data() + skip_bytes_, bytes_to_copy); @@ -328,11 +326,12 @@ bool PuffinStream::Read(void* buffer, size_t length) { return true; } -bool PuffinStream::Write(const void* buffer, size_t length) { +bool PuffinStream::Write(const void* buffer, size_t count) { TEST_AND_RETURN_FALSE(!closed_); TEST_AND_RETURN_FALSE(!is_for_puff_); auto bytes = static_cast<const uint8_t*>(buffer); - size_t bytes_wrote = 0; + uint64_t length = count; + uint64_t bytes_wrote = 0; while (bytes_wrote < length) { if (deflate_bit_pos_ < (cur_deflate_->offset & ~7ull)) { // Between two puffs or before the first puff. We know that we are @@ -340,9 +339,9 @@ bool PuffinStream::Write(const void* buffer, size_t length) { // non-deflate bits of the last byte of the last deflate. Here we don't // process any byte that has deflate bit. TEST_AND_RETURN_FALSE((deflate_bit_pos_ & 7) == 0); - auto copy_len = std::min((static_cast<size_t>(cur_deflate_->offset) / 8) - - (deflate_bit_pos_ / 8), - length - bytes_wrote); + auto copy_len = + std::min((cur_deflate_->offset / 8) - (deflate_bit_pos_ / 8), + length - bytes_wrote); TEST_AND_RETURN_FALSE(stream_->Write(bytes + bytes_wrote, copy_len)); bytes_wrote += copy_len; puff_pos_ += copy_len; @@ -364,9 +363,8 @@ bool PuffinStream::Write(const void* buffer, size_t length) { TEST_AND_RETURN_FALSE(puff_pos_ == cur_puff_->offset); } - auto copy_len = std::min( - length - bytes_wrote, - static_cast<size_t>(cur_puff_->length) + extra_byte_ - skip_bytes_); + auto copy_len = std::min(length - bytes_wrote, + cur_puff_->length + extra_byte_ - skip_bytes_); TEST_AND_RETURN_FALSE(puff_buffer_->size() >= skip_bytes_ + copy_len); memcpy(puff_buffer_->data() + skip_bytes_, bytes + bytes_wrote, copy_len); skip_bytes_ += copy_len; @@ -435,7 +433,7 @@ bool PuffinStream::SetExtraByte() { extra_byte_ = 0; return true; } - size_t end_bit = cur_deflate_->offset + cur_deflate_->length; + uint64_t end_bit = cur_deflate_->offset + cur_deflate_->length; if ((end_bit & 7) && ((end_bit + 7) & ~7ull) <= (cur_deflate_ + 1)->offset) { extra_byte_ = 1; } else { @@ -445,7 +443,7 @@ bool PuffinStream::SetExtraByte() { } bool PuffinStream::GetPuffCache(int puff_id, - size_t puff_size, + uint64_t puff_size, SharedBufferPtr* buffer) { bool found = false; // Search for it. @@ -477,7 +475,7 @@ bool PuffinStream::GetPuffCache(int puff_id, } cache.second->resize(puff_size); - constexpr size_t kMaxSizeDifference = 20 * 1024; + constexpr uint64_t kMaxSizeDifference = 20 * 1024; if (puff_size + kMaxSizeDifference < cache.second->capacity()) { cache.second->shrink_to_fit(); } diff --git a/src/puffin_stream.h b/src/puffin_stream.h index f791ac5..47b8ace 100644 --- a/src/puffin_stream.h +++ b/src/puffin_stream.h @@ -45,7 +45,7 @@ class PuffinStream : public StreamInterface { // and no puff will be cached. static UniqueStreamPtr CreateForPuff(UniqueStreamPtr stream, std::shared_ptr<Puffer> puffer, - size_t puff_size, + uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs, size_t max_cache_size = 0); @@ -59,17 +59,17 @@ class PuffinStream : public StreamInterface { // |puffs| IN The location of puffs into the input puff stream. static UniqueStreamPtr CreateForHuff(UniqueStreamPtr stream, std::shared_ptr<Huffer> huffer, - size_t puff_size, + uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs); - bool GetSize(size_t* size) const override; + bool GetSize(uint64_t* size) const override; // Returns the current offset in the imaginary puff stream. - bool GetOffset(size_t* offset) const override; + bool GetOffset(uint64_t* offset) const override; // Sets the current offset in the imaginary puff stream. - bool Seek(size_t offset) override; + bool Seek(uint64_t offset) override; // Reads from the deflate stream |stream_| and writes the puff stream into // |buffer|. @@ -90,7 +90,7 @@ class PuffinStream : public StreamInterface { PuffinStream(UniqueStreamPtr stream, std::shared_ptr<Puffer> puffer, std::shared_ptr<Huffer> huffer, - size_t puff_size, + uint64_t puff_size, const std::vector<BitExtent>& deflates, const std::vector<ByteExtent>& puffs, size_t max_cache_size); @@ -102,7 +102,7 @@ class PuffinStream : public StreamInterface { // Returns the cache for the |puff_id|th puff. If it does not find it, either // returns the least accessed cached (if cache is full) or creates a new empty // buffer. It returns false if it cannot find the |puff_id|th puff cache. - bool GetPuffCache(int puff_id, size_t puff_size, SharedBufferPtr* buffer); + bool GetPuffCache(int puff_id, uint64_t puff_size, SharedBufferPtr* buffer); UniqueStreamPtr stream_; @@ -110,7 +110,7 @@ class PuffinStream : public StreamInterface { std::shared_ptr<Huffer> huffer_; // The size of the imaginary puff stream. - size_t puff_stream_size_; + uint64_t puff_stream_size_; std::vector<BitExtent> deflates_; // The current deflate is being processed. @@ -120,15 +120,15 @@ class PuffinStream : public StreamInterface { // The current puff is being processed. std::vector<ByteExtent>::iterator cur_puff_; - std::vector<size_t> upper_bounds_; + std::vector<uint64_t> upper_bounds_; // The current offset in the imaginary puff stream is |puff_pos_| + // |skip_bytes_| - size_t puff_pos_; - size_t skip_bytes_; + uint64_t puff_pos_; + uint64_t skip_bytes_; // The current bit offset in |stream_|. - size_t deflate_bit_pos_; + uint64_t deflate_bit_pos_; // This value caches the first or last byte of a deflate stream. This is // needed when two deflate stream end on the same byte (with greater than zero @@ -158,7 +158,7 @@ class PuffinStream : public StreamInterface { // this class. size_t max_cache_size_; // The current amount of memory (in bytes) used for caching puff buffers. - size_t cur_cache_size_; + uint64_t cur_cache_size_; DISALLOW_COPY_AND_ASSIGN(PuffinStream); }; diff --git a/src/puffin_unittest.cc b/src/puffin_unittest.cc index ff785ec..fd035bc 100644 --- a/src/puffin_unittest.cc +++ b/src/puffin_unittest.cc @@ -199,7 +199,7 @@ class PuffinTest : public ::testing::Test { auto deflate_stream = MemoryStream::CreateForRead(deflate_buffer); ASSERT_TRUE(deflate_stream->Seek(0)); vector<ByteExtent> out_puff_extents; - size_t puff_size; + uint64_t puff_size; ASSERT_TRUE(FindPuffLocations(deflate_stream, deflate_extents, &out_puff_extents, &puff_size)); EXPECT_EQ(puff_size, puff_buffer.size()); diff --git a/src/puffpatch.cc b/src/puffpatch.cc index 1284196..185b1dd 100644 --- a/src/puffpatch.cc +++ b/src/puffpatch.cc @@ -52,8 +52,8 @@ bool DecodePatch(const uint8_t* patch, vector<BitExtent>* dst_deflates, vector<ByteExtent>* src_puffs, vector<ByteExtent>* dst_puffs, - size_t* src_puff_size, - size_t* dst_puff_size) { + uint64_t* src_puff_size, + uint64_t* dst_puff_size) { size_t offset = 0; uint32_t header_size; TEST_AND_RETURN_FALSE(patch_length >= (kMagicLength + sizeof(header_size))); @@ -116,7 +116,7 @@ class BsdiffStream : public bsdiff::FileInterface { bool Close() override { return stream_->Close(); } bool GetSize(uint64_t* size) override { - size_t my_size; + uint64_t my_size; TEST_AND_RETURN_FALSE(stream_->GetSize(&my_size)); *size = my_size; return true; @@ -139,7 +139,7 @@ bool PuffPatch(UniqueStreamPtr src, size_t bsdiff_patch_size = 0; vector<BitExtent> src_deflates, dst_deflates; vector<ByteExtent> src_puffs, dst_puffs; - size_t src_puff_size, dst_puff_size; + uint64_t src_puff_size, dst_puff_size; // Decode the patch and get the bsdiff_patch. TEST_AND_RETURN_FALSE(DecodePatch(patch, patch_length, &bsdiff_patch_offset, diff --git a/src/stream_unittest.cc b/src/stream_unittest.cc index 32614bc..0a7c7b3 100644 --- a/src/stream_unittest.cc +++ b/src/stream_unittest.cc @@ -35,7 +35,7 @@ class StreamTest : public ::testing::Test { // No reading out of data boundary. Buffer tmp(100); - size_t size; + uint64_t size; ASSERT_TRUE(stream->GetSize(&size)); ASSERT_TRUE(stream->Seek(size)); ASSERT_TRUE(stream->Read(tmp.data(), 0)); @@ -76,8 +76,8 @@ class StreamTest : public ::testing::Test { for (size_t idx = 0; idx < 10000; idx++) { // zero to full size available. size_t size = rand_r(&rand_seed) % (buf.size() + 1); - size_t max_start = buf.size() - size; - size_t start = rand_r(&rand_seed) % (max_start + 1); + uint64_t max_start = buf.size() - size; + uint64_t start = rand_r(&rand_seed) % (max_start + 1); ASSERT_TRUE(stream->Seek(start)); ASSERT_TRUE(stream->Read(tmp.data(), size)); for (size_t idx = 0; idx < size; idx++) { @@ -89,7 +89,7 @@ class StreamTest : public ::testing::Test { void TestWriteBoundary(StreamInterface* stream) { Buffer buf(10); // Writing out of boundary is fine. - size_t size; + uint64_t size; ASSERT_TRUE(stream->GetSize(&size)); ASSERT_TRUE(stream->Seek(size)); ASSERT_TRUE(stream->Write(buf.data(), 0)); @@ -111,7 +111,7 @@ class StreamTest : public ::testing::Test { } void TestWrite(StreamInterface* write_stream, StreamInterface* read_stream) { - size_t size; + uint64_t size; ASSERT_TRUE(read_stream->GetSize(&size)); Buffer buf1(size); Buffer buf2(size); @@ -139,7 +139,7 @@ class StreamTest : public ::testing::Test { // Call this at the end before |TestClose|. void TestSeek(StreamInterface* stream, bool seek_end_is_fine) { - size_t size, offset; + uint64_t size, offset; ASSERT_TRUE(stream->GetSize(&size)); ASSERT_TRUE(stream->Seek(size)); ASSERT_TRUE(stream->GetOffset(&offset)); diff --git a/src/unittest_common.h b/src/unittest_common.h index d0bbb02..c03aea4 100644 --- a/src/unittest_common.h +++ b/src/unittest_common.h @@ -335,47 +335,47 @@ const Buffer kPuff10 = { 0x37, 0x20, 0x54, 0x68, 0x65, 0x20, 0x41, 0x6E, 0x64, 0x72, 0x6F, 0x69, 0x64, 0x20, 0x4F, 0x70, 0x65, 0x6E, 0x20, 0x53, 0x6F, 0x75, 0x72, 0x63, 0x65, 0x20, 0x50, 0x72, 0x6F, 0x6A, 0x65, 0x63, 0x74, 0x0A, 0x83, 0x00, - 0x39, 0x0F, 0x4C, 0x69, 0x63, 0x65, 0x6E, 0x73, 0x65, 0x64, 0x20, 0x75, - 0x6E, 0x64, 0x65, 0x72, 0x20, 0x74, 0x81, 0x00, 0x35, 0x02, 0x70, 0x61, - 0x63, 0x80, 0x00, 0x07, 0x84, 0x00, 0x1A, 0x0E, 0x2C, 0x20, 0x56, 0x65, + 0x38, 0x0F, 0x4C, 0x69, 0x63, 0x65, 0x6E, 0x73, 0x65, 0x64, 0x20, 0x75, + 0x6E, 0x64, 0x65, 0x72, 0x20, 0x74, 0x81, 0x00, 0x34, 0x02, 0x70, 0x61, + 0x63, 0x80, 0x00, 0x06, 0x84, 0x00, 0x19, 0x0E, 0x2C, 0x20, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x20, 0x32, 0x2E, 0x30, 0x20, 0x28, 0x81, - 0x00, 0x21, 0x00, 0x22, 0x84, 0x00, 0x1B, 0x02, 0x22, 0x29, 0x3B, 0x81, - 0x00, 0x43, 0x0E, 0x79, 0x6F, 0x75, 0x20, 0x6D, 0x61, 0x79, 0x20, 0x6E, - 0x6F, 0x74, 0x20, 0x75, 0x73, 0x65, 0x80, 0x00, 0x44, 0x19, 0x69, 0x73, + 0x00, 0x20, 0x00, 0x22, 0x84, 0x00, 0x1A, 0x02, 0x22, 0x29, 0x3B, 0x81, + 0x00, 0x42, 0x0E, 0x79, 0x6F, 0x75, 0x20, 0x6D, 0x61, 0x79, 0x20, 0x6E, + 0x6F, 0x74, 0x20, 0x75, 0x73, 0x65, 0x80, 0x00, 0x43, 0x19, 0x69, 0x73, 0x20, 0x66, 0x69, 0x6C, 0x65, 0x20, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x20, 0x69, 0x6E, 0x20, 0x63, 0x6F, 0x6D, 0x70, 0x6C, 0x69, 0x61, 0x6E, - 0x80, 0x00, 0x80, 0x03, 0x77, 0x69, 0x74, 0x68, 0x82, 0x00, 0x68, 0x84, - 0x00, 0x46, 0x00, 0x2E, 0x81, 0x00, 0x44, 0x00, 0x59, 0x84, 0x00, 0x44, - 0x03, 0x6F, 0x62, 0x74, 0x61, 0x80, 0x00, 0x2F, 0x00, 0x61, 0x80, 0x00, - 0x31, 0x00, 0x70, 0x80, 0x00, 0x0E, 0x00, 0x66, 0x89, 0x00, 0x29, 0x01, - 0x20, 0x61, 0x85, 0x00, 0xB5, 0x82, 0x00, 0x01, 0x0B, 0x68, 0x74, 0x74, - 0x70, 0x3A, 0x2F, 0x2F, 0x77, 0x77, 0x77, 0x2E, 0x61, 0x82, 0x00, 0xB2, - 0x05, 0x2E, 0x6F, 0x72, 0x67, 0x2F, 0x6C, 0x83, 0x00, 0x2C, 0x09, 0x73, - 0x2F, 0x4C, 0x49, 0x43, 0x45, 0x4E, 0x53, 0x45, 0x2D, 0x80, 0x00, 0xB6, - 0x84, 0x00, 0x36, 0x0C, 0x55, 0x6E, 0x6C, 0x65, 0x73, 0x73, 0x20, 0x72, - 0x65, 0x71, 0x75, 0x69, 0x72, 0x80, 0x00, 0xF2, 0x04, 0x62, 0x79, 0x20, - 0x61, 0x70, 0x80, 0x00, 0x96, 0x02, 0x63, 0x61, 0x62, 0x80, 0x00, 0xAC, + 0x80, 0x00, 0x7F, 0x03, 0x77, 0x69, 0x74, 0x68, 0x82, 0x00, 0x67, 0x84, + 0x00, 0x45, 0x00, 0x2E, 0x81, 0x00, 0x43, 0x00, 0x59, 0x84, 0x00, 0x43, + 0x03, 0x6F, 0x62, 0x74, 0x61, 0x80, 0x00, 0x2E, 0x00, 0x61, 0x80, 0x00, + 0x30, 0x00, 0x70, 0x80, 0x00, 0x0D, 0x00, 0x66, 0x89, 0x00, 0x28, 0x01, + 0x20, 0x61, 0x85, 0x00, 0xB4, 0x82, 0x00, 0x00, 0x0B, 0x68, 0x74, 0x74, + 0x70, 0x3A, 0x2F, 0x2F, 0x77, 0x77, 0x77, 0x2E, 0x61, 0x82, 0x00, 0xB1, + 0x05, 0x2E, 0x6F, 0x72, 0x67, 0x2F, 0x6C, 0x83, 0x00, 0x2B, 0x09, 0x73, + 0x2F, 0x4C, 0x49, 0x43, 0x45, 0x4E, 0x53, 0x45, 0x2D, 0x80, 0x00, 0xB5, + 0x84, 0x00, 0x35, 0x0C, 0x55, 0x6E, 0x6C, 0x65, 0x73, 0x73, 0x20, 0x72, + 0x65, 0x71, 0x75, 0x69, 0x72, 0x80, 0x00, 0xF1, 0x04, 0x62, 0x79, 0x20, + 0x61, 0x70, 0x80, 0x00, 0x95, 0x02, 0x63, 0x61, 0x62, 0x80, 0x00, 0xAB, 0x0A, 0x6C, 0x61, 0x77, 0x20, 0x6F, 0x72, 0x20, 0x61, 0x67, 0x72, 0x65, - 0x80, 0x00, 0x1C, 0x01, 0x74, 0x6F, 0x81, 0x00, 0xB6, 0x10, 0x77, 0x72, + 0x80, 0x00, 0x1B, 0x01, 0x74, 0x6F, 0x81, 0x00, 0xB5, 0x10, 0x77, 0x72, 0x69, 0x74, 0x69, 0x6E, 0x67, 0x2C, 0x20, 0x73, 0x6F, 0x66, 0x74, 0x77, - 0x61, 0x72, 0x65, 0x81, 0x00, 0x47, 0x08, 0x64, 0x69, 0x73, 0x74, 0x72, - 0x69, 0x62, 0x75, 0x74, 0x8A, 0x01, 0x35, 0x85, 0x00, 0xA4, 0x80, 0x00, - 0xFB, 0x89, 0x00, 0x21, 0x80, 0x01, 0x37, 0x10, 0x61, 0x6E, 0x20, 0x22, + 0x61, 0x72, 0x65, 0x81, 0x00, 0x46, 0x08, 0x64, 0x69, 0x73, 0x74, 0x72, + 0x69, 0x62, 0x75, 0x74, 0x8A, 0x01, 0x34, 0x85, 0x00, 0xA3, 0x80, 0x00, + 0xFA, 0x89, 0x00, 0x20, 0x80, 0x01, 0x36, 0x10, 0x61, 0x6E, 0x20, 0x22, 0x41, 0x53, 0x20, 0x49, 0x53, 0x22, 0x20, 0x42, 0x41, 0x53, 0x49, 0x53, - 0x2C, 0x81, 0x00, 0x45, 0x1E, 0x57, 0x49, 0x54, 0x48, 0x4F, 0x55, 0x54, + 0x2C, 0x81, 0x00, 0x44, 0x1E, 0x57, 0x49, 0x54, 0x48, 0x4F, 0x55, 0x54, 0x20, 0x57, 0x41, 0x52, 0x52, 0x41, 0x4E, 0x54, 0x49, 0x45, 0x53, 0x20, 0x4F, 0x52, 0x20, 0x43, 0x4F, 0x4E, 0x44, 0x49, 0x54, 0x49, 0x4F, 0x4E, - 0x80, 0x00, 0x0E, 0x0C, 0x46, 0x20, 0x41, 0x4E, 0x59, 0x20, 0x4B, 0x49, - 0x4E, 0x44, 0x2C, 0x20, 0x65, 0x80, 0x01, 0x33, 0x80, 0x00, 0x68, 0x03, - 0x65, 0x78, 0x70, 0x72, 0x81, 0x00, 0xC2, 0x80, 0x00, 0xA7, 0x00, 0x69, - 0x81, 0x01, 0x4F, 0x01, 0x65, 0x64, 0x82, 0x01, 0x3C, 0x02, 0x53, 0x65, - 0x65, 0x8A, 0x00, 0x83, 0x01, 0x66, 0x6F, 0x83, 0x00, 0x93, 0x07, 0x73, - 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x80, 0x00, 0xDB, 0x0C, 0x6E, + 0x80, 0x00, 0x0D, 0x0C, 0x46, 0x20, 0x41, 0x4E, 0x59, 0x20, 0x4B, 0x49, + 0x4E, 0x44, 0x2C, 0x20, 0x65, 0x80, 0x01, 0x32, 0x80, 0x00, 0x67, 0x03, + 0x65, 0x78, 0x70, 0x72, 0x81, 0x00, 0xC1, 0x80, 0x00, 0xA6, 0x00, 0x69, + 0x81, 0x01, 0x4E, 0x01, 0x65, 0x64, 0x82, 0x01, 0x3B, 0x02, 0x53, 0x65, + 0x65, 0x8A, 0x00, 0x82, 0x01, 0x66, 0x6F, 0x83, 0x00, 0x92, 0x07, 0x73, + 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x80, 0x00, 0xDA, 0x0C, 0x6E, 0x67, 0x75, 0x61, 0x67, 0x65, 0x20, 0x67, 0x6F, 0x76, 0x65, 0x72, 0x6E, - 0x80, 0x00, 0xD2, 0x06, 0x20, 0x70, 0x65, 0x72, 0x6D, 0x69, 0x73, 0x81, - 0x01, 0xD7, 0x00, 0x73, 0x80, 0x00, 0xA1, 0x00, 0x64, 0x81, 0x00, 0x47, - 0x06, 0x6C, 0x69, 0x6D, 0x69, 0x74, 0x61, 0x74, 0x82, 0x00, 0x13, 0x8E, - 0x00, 0xD8, 0x01, 0x2E, 0x0A, 0xFF, 0x81}; + 0x80, 0x00, 0xD1, 0x06, 0x20, 0x70, 0x65, 0x72, 0x6D, 0x69, 0x73, 0x81, + 0x01, 0xD6, 0x00, 0x73, 0x80, 0x00, 0xA0, 0x00, 0x64, 0x81, 0x00, 0x46, + 0x06, 0x6C, 0x69, 0x6D, 0x69, 0x74, 0x61, 0x74, 0x82, 0x00, 0x12, 0x8E, + 0x00, 0xD7, 0x01, 0x2E, 0x0A, 0xFF, 0x81}; // The following is a sequence of bits starting from the top right and ends in // bottom left. It represents the bits in |kDeflate11|. Bits inside the brackets diff --git a/src/utils.cc b/src/utils.cc index 2b41220..5019d0b 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -32,9 +32,9 @@ inline T get_unaligned(const void* address) { // Calculate both the compressed size and uncompressed size of the deflate // block that starts from the offset |start| of buffer |data|. bool CalculateSizeOfDeflateBlock(const puffin::Buffer& data, - size_t start, - size_t* compressed_size, - size_t* uncompressed_size) { + uint64_t start, + uint64_t* compressed_size, + uint64_t* uncompressed_size) { TEST_AND_RETURN_FALSE(compressed_size != nullptr && uncompressed_size != nullptr); @@ -79,8 +79,8 @@ namespace puffin { using std::string; using std::vector; -size_t BytesInByteExtents(const vector<ByteExtent>& extents) { - size_t bytes = 0; +uint64_t BytesInByteExtents(const vector<ByteExtent>& extents) { + uint64_t bytes = 0; for (const auto& extent : extents) { bytes += extent.length; } @@ -113,7 +113,7 @@ bool LocateDeflatesInZlib(const Buffer& data, auto flag = data[1]; TEST_AND_RETURN_FALSE(((cmf << 8) + flag) % 31 == 0); - size_t header_len = 2; + uint64_t header_len = 2; if (flag & 0x20) { header_len += 4; // 4 bytes for the preset dictionary. } @@ -182,7 +182,7 @@ bool LocateDeflatesInZlibBlocks(const string& file_path, // https://www.ietf.org/rfc/rfc1952.txt bool LocateDeflatesInGzip(const Buffer& data, vector<ByteExtent>* deflate_blocks) { - size_t member_start = 0; + uint64_t member_start = 0; while (member_start < data.size()) { // Each member entry has the following format // 0 1 0x1F @@ -197,7 +197,7 @@ bool LocateDeflatesInGzip(const Buffer& data, TEST_AND_RETURN_FALSE(data[member_start + 1] == 0x8B); TEST_AND_RETURN_FALSE(data[member_start + 2] == 8); - size_t offset = member_start + 10; + uint64_t offset = member_start + 10; int flag = data[member_start + 3]; // Extra field if (flag & 4) { @@ -230,7 +230,7 @@ bool LocateDeflatesInGzip(const Buffer& data, offset += 2; } - size_t compressed_size, uncompressed_size; + uint64_t compressed_size, uncompressed_size; TEST_AND_RETURN_FALSE(CalculateSizeOfDeflateBlock( data, offset, &compressed_size, &uncompressed_size)); TEST_AND_RETURN_FALSE(offset + compressed_size <= data.size()); @@ -254,7 +254,7 @@ bool LocateDeflatesInGzip(const Buffer& data, // https://support.pkware.com/display/PKZIP/APPNOTE bool LocateDeflatesInZipArchive(const Buffer& data, vector<ByteExtent>* deflate_blocks) { - size_t pos = 0; + uint64_t pos = 0; while (pos <= data.size() - 30) { // TODO(xunchang) add support for big endian system when searching for // magic numbers. @@ -295,8 +295,8 @@ bool LocateDeflatesInZipArchive(const Buffer& data, continue; } - size_t calculated_compressed_size; - size_t calculated_uncompressed_size; + uint64_t calculated_compressed_size; + uint64_t calculated_uncompressed_size; if (!CalculateSizeOfDeflateBlock(data, pos + header_size, &calculated_compressed_size, &calculated_uncompressed_size)) { @@ -342,7 +342,7 @@ bool LocateDeflateSubBlocksInZipArchive(const Buffer& data, bool FindPuffLocations(const UniqueStreamPtr& src, const vector<BitExtent>& deflates, vector<ByteExtent>* puffs, - size_t* out_puff_size) { + uint64_t* out_puff_size) { Puffer puffer; Buffer deflate_buffer; @@ -350,7 +350,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src, // puff. At the end we add this cummulative size difference to the size of the // deflate stream to get the size of the puff stream. We use signed size // because puff size could be smaller than deflate size. - ssize_t total_size_difference = 0; + int64_t total_size_difference = 0; for (auto deflate = deflates.begin(); deflate != deflates.end(); ++deflate) { // Read from src into deflate_buffer. auto start_byte = deflate->offset / 8; @@ -361,7 +361,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src, src->Read(deflate_buffer.data(), deflate_buffer.size())); // Find the size of the puff. BufferBitReader bit_reader(deflate_buffer.data(), deflate_buffer.size()); - size_t bits_to_skip = deflate->offset % 8; + uint64_t bits_to_skip = deflate->offset % 8; TEST_AND_RETURN_FALSE(bit_reader.CacheBits(bits_to_skip)); bit_reader.DropBits(bits_to_skip); @@ -389,7 +389,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src, start_byte = ((deflate->offset + 7) / 8); end_byte = (deflate->offset + deflate->length) / 8; - ssize_t deflate_length_in_bytes = end_byte - start_byte; + int64_t deflate_length_in_bytes = end_byte - start_byte; // If there was no gap bits between the current and previous deflates, there // will be no extra gap byte, so the offset will be shifted one byte back. @@ -398,12 +398,12 @@ bool FindPuffLocations(const UniqueStreamPtr& src, // Add the location into puff. puffs->emplace_back(puff_offset, puff_size); total_size_difference += - static_cast<ssize_t>(puff_size) - deflate_length_in_bytes - gap; + static_cast<int64_t>(puff_size) - deflate_length_in_bytes - gap; } - size_t src_size; + uint64_t src_size; TEST_AND_RETURN_FALSE(src->GetSize(&src_size)); - auto final_size = static_cast<ssize_t>(src_size) + total_size_difference; + auto final_size = static_cast<int64_t>(src_size) + total_size_difference; TEST_AND_RETURN_FALSE(final_size >= 0); *out_puff_size = final_size; return true; diff --git a/src/utils_unittest.cc b/src/utils_unittest.cc index 8d972d2..e1fbb25 100644 --- a/src/utils_unittest.cc +++ b/src/utils_unittest.cc @@ -102,10 +102,10 @@ void FindDeflatesInZlibBlocks(const Buffer& src, void CheckFindPuffLocation(const Buffer& compressed, const vector<BitExtent>& deflates, const vector<ByteExtent>& expected_puffs, - size_t expected_puff_size) { + uint64_t expected_puff_size) { auto src = MemoryStream::CreateForRead(compressed); vector<ByteExtent> puffs; - size_t puff_size; + uint64_t puff_size; ASSERT_TRUE(FindPuffLocations(src, deflates, &puffs, &puff_size)); EXPECT_EQ(puffs, expected_puffs); EXPECT_EQ(puff_size, expected_puff_size); |