aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAmin Hassani <ahassani@google.com>2018-02-28 15:34:21 -0800
committerAmin Hassani <ahassani@google.com>2018-02-28 15:34:21 -0800
commitd7768d5c29e747020531b5f71e7c7cfe980ea861 (patch)
tree83f781a80d7a306dd487e0fa2ccce74768abf258
parent20bdc7cbf417b5cc93fb6abe110c233cde4b7cd0 (diff)
downloadplatform_external_puffin-d7768d5c29e747020531b5f71e7c7cfe980ea861.tar.gz
platform_external_puffin-d7768d5c29e747020531b5f71e7c7cfe980ea861.tar.bz2
platform_external_puffin-d7768d5c29e747020531b5f71e7c7cfe980ea861.zip
Convert majority of size_t to uint64_t for 32-bit compatibility
We need to get puffin to work correctly on 32-bit machines. This patch converts appropriate size_t types to uint64_t to make sure when we are working with large files in 32bit machines, nothing breaks. The 32 bit tests actually found some errors. For example BitReader and BitWriter should have their size and indices kept in 64 bit integers otherwise a multiply by 8 (to figure out the bit-based length) will overflow. Although it has not been tested for puffdiff and puffpatch. Bug: 73781483 Test: unittests pass Test: puffhuff on a >5G zlib file passes Test: build with 32-bit compiler, and puffhuff on >1GB passes Change-Id: I0234a787e095a7ebf0489966a7b7565b6688496c
-rw-r--r--src/bit_reader.cc2
-rw-r--r--src/bit_reader.h8
-rw-r--r--src/bit_writer.h4
-rw-r--r--src/extent_stream.cc8
-rw-r--r--src/extent_stream.h12
-rw-r--r--src/file_stream.cc6
-rw-r--r--src/file_stream.h6
-rw-r--r--src/include/puffin/stream.h6
-rw-r--r--src/include/puffin/utils.h4
-rw-r--r--src/main.cc26
-rw-r--r--src/memory_stream.cc8
-rw-r--r--src/memory_stream.h8
-rw-r--r--src/puffdiff.cc10
-rw-r--r--src/puffin_stream.cc62
-rw-r--r--src/puffin_stream.h26
-rw-r--r--src/puffin_unittest.cc2
-rw-r--r--src/puffpatch.cc8
-rw-r--r--src/stream_unittest.cc12
-rw-r--r--src/utils.cc38
-rw-r--r--src/utils_unittest.cc4
20 files changed, 130 insertions, 130 deletions
diff --git a/src/bit_reader.cc b/src/bit_reader.cc
index 006870d..d6f6abb 100644
--- a/src/bit_reader.cc
+++ b/src/bit_reader.cc
@@ -64,7 +64,7 @@ size_t BufferBitReader::Offset() const {
return index_ - in_cache_bits_ / 8;
}
-size_t BufferBitReader::OffsetInBits() const {
+uint64_t BufferBitReader::OffsetInBits() const {
return (index_ * 8) - in_cache_bits_;
}
diff --git a/src/bit_reader.h b/src/bit_reader.h
index ee2124b..2d0c805 100644
--- a/src/bit_reader.h
+++ b/src/bit_reader.h
@@ -67,7 +67,7 @@ class BitReaderInterface {
virtual size_t Offset() const = 0;
// Returns the number of bits read (dropped) till now.
- virtual size_t OffsetInBits() const = 0;
+ virtual uint64_t OffsetInBits() const = 0;
};
// A raw buffer implementation of |BitReaderInterface|.
@@ -96,12 +96,12 @@ class BufferBitReader : public BitReaderInterface {
size_t length,
std::function<bool(uint8_t* buffer, size_t count)>* read_fn) override;
size_t Offset() const override;
- size_t OffsetInBits() const override;
+ uint64_t OffsetInBits() const override;
private:
const uint8_t* in_buf_; // The input buffer.
- size_t in_size_; // The number of bytes in |in_buf_|.
- size_t index_; // The index to the next byte to be read.
+ uint64_t in_size_; // The number of bytes in |in_buf_|.
+ uint64_t index_; // The index to the next byte to be read.
uint32_t in_cache_; // The temporary buffer to put input data into.
size_t in_cache_bits_; // The number of bits available in |in_cache_|.
diff --git a/src/bit_writer.h b/src/bit_writer.h
index d7a608b..9e9bdb9 100644
--- a/src/bit_writer.h
+++ b/src/bit_writer.h
@@ -88,10 +88,10 @@ class BufferBitWriter : public BitWriterInterface {
uint8_t* out_buf_;
// The number of bytes in |out_buf_|.
- size_t out_size_;
+ uint64_t out_size_;
// The index to the next byte to write into.
- size_t index_;
+ uint64_t index_;
// A temporary buffer to keep the bits going out.
uint32_t out_holder_;
diff --git a/src/extent_stream.cc b/src/extent_stream.cc
index e296ec1..d6d6060 100644
--- a/src/extent_stream.cc
+++ b/src/extent_stream.cc
@@ -34,7 +34,7 @@ ExtentStream::ExtentStream(UniqueStreamPtr stream,
extents_upper_bounds_.reserve(extents_.size() + 1);
extents_upper_bounds_.emplace_back(0);
uint64_t total_size = 0;
- size_t extent_end = 0;
+ uint64_t extent_end = 0;
for (const auto& extent : extents_) {
total_size += extent.length;
extents_upper_bounds_.emplace_back(total_size);
@@ -49,17 +49,17 @@ ExtentStream::ExtentStream(UniqueStreamPtr stream,
cur_extent_ = extents_.begin();
}
-bool ExtentStream::GetSize(size_t* size) const {
+bool ExtentStream::GetSize(uint64_t* size) const {
*size = size_;
return true;
}
-bool ExtentStream::GetOffset(size_t* offset) const {
+bool ExtentStream::GetOffset(uint64_t* offset) const {
*offset = offset_;
return true;
}
-bool ExtentStream::Seek(size_t offset) {
+bool ExtentStream::Seek(uint64_t offset) {
TEST_AND_RETURN_FALSE(offset <= size_);
// The first item is zero and upper_bound never returns it because it always
diff --git a/src/extent_stream.h b/src/extent_stream.h
index 61a556b..455f800 100644
--- a/src/extent_stream.h
+++ b/src/extent_stream.h
@@ -26,9 +26,9 @@ class ExtentStream : public StreamInterface {
const std::vector<ByteExtent>& extents);
~ExtentStream() override = default;
- bool GetSize(size_t* size) const override;
- bool GetOffset(size_t* offset) const override;
- bool Seek(size_t offset) override;
+ bool GetSize(uint64_t* size) const override;
+ bool GetOffset(uint64_t* offset) const override;
+ bool Seek(uint64_t offset) override;
bool Read(void* buffer, size_t length) override;
bool Write(const void* buffer, size_t length) override;
bool Close() override;
@@ -54,17 +54,17 @@ class ExtentStream : public StreamInterface {
std::vector<ByteExtent>::iterator cur_extent_;
// The current offset in the current |ByteExtent| |cur_extent_|.
- size_t cur_extent_offset_;
+ uint64_t cur_extent_offset_;
// |True| if the stream is write only. |False| if the stream is read only.
bool is_for_write_;
// The size of the stream. It is actually the cumulative size of all the bytes
// in |extents_|.
- size_t size_;
+ uint64_t size_;
// The current offset.
- size_t offset_;
+ uint64_t offset_;
// Used for proper and faster seeking.
std::vector<uint64_t> extents_upper_bounds_;
diff --git a/src/file_stream.cc b/src/file_stream.cc
index 0192a98..2089078 100644
--- a/src/file_stream.cc
+++ b/src/file_stream.cc
@@ -34,7 +34,7 @@ UniqueStreamPtr FileStream::Open(const string& path, bool read, bool write) {
return UniqueStreamPtr(new FileStream(fd));
}
-bool FileStream::GetSize(size_t* size) const {
+bool FileStream::GetSize(uint64_t* size) const {
auto cur_off = lseek(fd_, 0, SEEK_CUR);
TEST_AND_RETURN_FALSE(cur_off >= 0);
auto fsize = lseek(fd_, 0, SEEK_END);
@@ -45,14 +45,14 @@ bool FileStream::GetSize(size_t* size) const {
return true;
}
-bool FileStream::GetOffset(size_t* offset) const {
+bool FileStream::GetOffset(uint64_t* offset) const {
auto off = lseek(fd_, 0, SEEK_CUR);
TEST_AND_RETURN_FALSE(off >= 0);
*offset = off;
return true;
}
-bool FileStream::Seek(size_t offset) {
+bool FileStream::Seek(uint64_t offset) {
auto off = lseek(fd_, offset, SEEK_SET);
TEST_AND_RETURN_FALSE(off == static_cast<off_t>(offset));
return true;
diff --git a/src/file_stream.h b/src/file_stream.h
index 156266b..626d7c6 100644
--- a/src/file_stream.h
+++ b/src/file_stream.h
@@ -21,9 +21,9 @@ class FileStream : public StreamInterface {
static UniqueStreamPtr Open(const std::string& path, bool read, bool write);
- bool GetSize(size_t* size) const override;
- bool GetOffset(size_t* offset) const override;
- bool Seek(size_t offset) override;
+ bool GetSize(uint64_t* size) const override;
+ bool GetOffset(uint64_t* offset) const override;
+ bool Seek(uint64_t offset) override;
bool Read(void* buffer, size_t length) override;
bool Write(const void* buffer, size_t length) override;
bool Close() override;
diff --git a/src/include/puffin/stream.h b/src/include/puffin/stream.h
index a5062ad..ada9d06 100644
--- a/src/include/puffin/stream.h
+++ b/src/include/puffin/stream.h
@@ -18,15 +18,15 @@ class StreamInterface {
virtual ~StreamInterface() = default;
// Returns the size of the stream.
- virtual bool GetSize(size_t* size) const = 0;
+ virtual bool GetSize(uint64_t* size) const = 0;
// Returns the current offset in the stream where next read or write will
// happen.
- virtual bool GetOffset(size_t* offset) const = 0;
+ virtual bool GetOffset(uint64_t* offset) const = 0;
// Sets the offset in the stream for the next read or write. On error
// returns |false|.
- virtual bool Seek(size_t offset) = 0;
+ virtual bool Seek(uint64_t offset) = 0;
// Reads |length| bytes of data into |buffer|. On error, returns |false|.
virtual bool Read(void* buffer, size_t length) = 0;
diff --git a/src/include/puffin/utils.h b/src/include/puffin/utils.h
index 65f8e63..4c81996 100644
--- a/src/include/puffin/utils.h
+++ b/src/include/puffin/utils.h
@@ -15,7 +15,7 @@ namespace puffin {
// Counts the number of bytes in a list of |ByteExtent|s.
PUFFIN_EXPORT
-size_t BytesInByteExtents(const std::vector<ByteExtent>& extents);
+uint64_t BytesInByteExtents(const std::vector<ByteExtent>& extents);
// Converts an array of |ByteExtens| or |BitExtents| to a string. Each extent
// has format "offset:length" and are comma separated.
@@ -72,7 +72,7 @@ bool FindDeflateSubBlocks(const UniqueStreamPtr& src,
bool FindPuffLocations(const UniqueStreamPtr& src,
const std::vector<BitExtent>& deflates,
std::vector<ByteExtent>* puffs,
- size_t* out_puff_size);
+ uint64_t* out_puff_size);
} // namespace puffin
diff --git a/src/main.cc b/src/main.cc
index dd57c05..c610764 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -61,7 +61,7 @@ vector<T> StringToExtents(const string& str) {
return extents;
}
-const size_t kDefaultPuffCacheSize = 50 * 1024 * 1024; // 50 MB
+const uint64_t kDefaultPuffCacheSize = 50 * 1024 * 1024; // 50 MB
// An enum representing the type of compressed files.
enum class FileType { kDeflate, kZlib, kGzip, kZip, kRaw, kUnknown };
@@ -123,7 +123,7 @@ bool LocateDeflatesBasedOnFileType(const UniqueStreamPtr& stream,
return true;
}
- size_t stream_size;
+ uint64_t stream_size;
TEST_AND_RETURN_FALSE(stream->GetSize(&stream_size));
if (file_type == FileType::kDeflate) {
// Assume the whole stream is a deflate block.
@@ -244,7 +244,7 @@ int main(int argc, char** argv) {
-1);
}
TEST_AND_RETURN_VALUE(dst_puffs.empty(), -1);
- size_t dst_puff_size;
+ uint64_t dst_puff_size;
TEST_AND_RETURN_VALUE(FindPuffLocations(src_stream, src_deflates_bit,
&dst_puffs, &dst_puff_size),
-1);
@@ -262,10 +262,10 @@ int main(int argc, char** argv) {
: std::move(dst_stream);
Buffer buffer(1024 * 1024);
- size_t bytes_wrote = 0;
+ uint64_t bytes_wrote = 0;
while (bytes_wrote < dst_puff_size) {
- auto write_size = std::min(
- buffer.size(), static_cast<size_t>(dst_puff_size - bytes_wrote));
+ auto write_size = std::min(static_cast<uint64_t>(buffer.size()),
+ dst_puff_size - bytes_wrote);
TEST_AND_RETURN_VALUE(reader->Read(buffer.data(), write_size), -1);
TEST_AND_RETURN_VALUE(writer->Write(buffer.data(), write_size), -1);
bytes_wrote += write_size;
@@ -284,9 +284,10 @@ int main(int argc, char** argv) {
std::move(dst_stream), huffer, dst_puff_size, dst_deflates_bit,
src_puffs);
- size_t bytes_read = 0;
+ uint64_t bytes_read = 0;
while (bytes_read < dst_puff_size) {
- auto read_size = std::min(buffer.size(), dst_puff_size - bytes_read);
+ auto read_size = std::min(static_cast<uint64_t>(buffer.size()),
+ dst_puff_size - bytes_read);
TEST_AND_RETURN_VALUE(read_puff_stream->Read(buffer.data(), read_size),
-1);
TEST_AND_RETURN_VALUE(huff_writer->Write(buffer.data(), read_size), -1);
@@ -299,7 +300,7 @@ int main(int argc, char** argv) {
<< ", is this intentional?";
}
TEST_AND_RETURN_VALUE(src_puffs.size() == dst_deflates_bit.size(), -1);
- size_t src_stream_size;
+ uint64_t src_stream_size;
TEST_AND_RETURN_VALUE(src_stream->GetSize(&src_stream_size), -1);
auto dst_file = FileStream::Open(FLAGS_dst_file, false, true);
TEST_AND_RETURN_VALUE(dst_file, -1);
@@ -310,9 +311,10 @@ int main(int argc, char** argv) {
dst_deflates_bit, src_puffs);
Buffer buffer(1024 * 1024);
- size_t bytes_read = 0;
+ uint64_t bytes_read = 0;
while (bytes_read < src_stream_size) {
- auto read_size = std::min(buffer.size(), src_stream_size - bytes_read);
+ auto read_size = std::min(static_cast<uint64_t>(buffer.size()),
+ src_stream_size - bytes_read);
TEST_AND_RETURN_VALUE(src_stream->Read(buffer.data(), read_size), -1);
TEST_AND_RETURN_VALUE(dst_stream->Write(buffer.data(), read_size), -1);
bytes_read += read_size;
@@ -370,7 +372,7 @@ int main(int argc, char** argv) {
} else if (FLAGS_operation == "puffpatch") {
auto patch_stream = FileStream::Open(FLAGS_patch_file, true, false);
TEST_AND_RETURN_VALUE(patch_stream, -1);
- size_t patch_size;
+ uint64_t patch_size;
TEST_AND_RETURN_VALUE(patch_stream->GetSize(&patch_size), -1);
Buffer puffdiff_delta(patch_size);
diff --git a/src/memory_stream.cc b/src/memory_stream.cc
index bd3aa2d..5f32a90 100644
--- a/src/memory_stream.cc
+++ b/src/memory_stream.cc
@@ -29,20 +29,20 @@ MemoryStream::MemoryStream(const Buffer* read_memory, Buffer* write_memory)
offset_(0),
open_(true) {}
-bool MemoryStream::GetSize(size_t* size) const {
+bool MemoryStream::GetSize(uint64_t* size) const {
*size =
read_memory_ != nullptr ? read_memory_->size() : write_memory_->size();
return true;
}
-bool MemoryStream::GetOffset(size_t* offset) const {
+bool MemoryStream::GetOffset(uint64_t* offset) const {
*offset = offset_;
return true;
}
-bool MemoryStream::Seek(size_t offset) {
+bool MemoryStream::Seek(uint64_t offset) {
TEST_AND_RETURN_FALSE(open_);
- size_t size;
+ uint64_t size;
GetSize(&size);
TEST_AND_RETURN_FALSE(offset <= size);
offset_ = offset;
diff --git a/src/memory_stream.h b/src/memory_stream.h
index 406d38d..a338e7f 100644
--- a/src/memory_stream.h
+++ b/src/memory_stream.h
@@ -26,9 +26,9 @@ class MemoryStream : public StreamInterface {
// the |memory|.
static UniqueStreamPtr CreateForWrite(Buffer* memory);
- bool GetSize(size_t* size) const override;
- bool GetOffset(size_t* offset) const override;
- bool Seek(size_t offset) override;
+ bool GetSize(uint64_t* size) const override;
+ bool GetOffset(uint64_t* offset) const override;
+ bool Seek(uint64_t offset) override;
bool Read(void* buffer, size_t length) override;
bool Write(const void* buffer, size_t length) override;
bool Close() override;
@@ -44,7 +44,7 @@ class MemoryStream : public StreamInterface {
Buffer* write_memory_;
// The current offset.
- size_t offset_;
+ uint64_t offset_;
// True if the stream is open.
bool open_;
diff --git a/src/puffdiff.cc b/src/puffdiff.cc
index fd3c4c8..7d02085 100644
--- a/src/puffdiff.cc
+++ b/src/puffdiff.cc
@@ -52,8 +52,8 @@ bool CreatePatch(const Buffer& bsdiff_patch,
const vector<BitExtent>& dst_deflates,
const vector<ByteExtent>& src_puffs,
const vector<ByteExtent>& dst_puffs,
- size_t src_puff_size,
- size_t dst_puff_size,
+ uint64_t src_puff_size,
+ uint64_t dst_puff_size,
Buffer* patch) {
metadata::PatchHeader header;
header.set_version(1);
@@ -68,7 +68,7 @@ bool CreatePatch(const Buffer& bsdiff_patch,
const uint32_t header_size = header.ByteSize();
- size_t offset = 0;
+ uint64_t offset = 0;
patch->resize(kMagicLength + sizeof(header_size) + header_size +
bsdiff_patch.size());
@@ -104,7 +104,7 @@ bool PuffDiff(UniqueStreamPtr src,
auto puff_deflate_stream =
[&puffer](UniqueStreamPtr stream, const vector<BitExtent>& deflates,
Buffer* puff_buffer, vector<ByteExtent>* puffs) {
- size_t puff_size;
+ uint64_t puff_size;
TEST_AND_RETURN_FALSE(stream->Seek(0));
TEST_AND_RETURN_FALSE(
FindPuffLocations(stream, deflates, puffs, &puff_size));
@@ -132,7 +132,7 @@ bool PuffDiff(UniqueStreamPtr src,
auto bsdiff_patch = FileStream::Open(tmp_filepath, true, false);
TEST_AND_RETURN_FALSE(bsdiff_patch);
- size_t patch_size;
+ uint64_t patch_size;
TEST_AND_RETURN_FALSE(bsdiff_patch->GetSize(&patch_size));
Buffer bsdiff_patch_buf(patch_size);
TEST_AND_RETURN_FALSE(
diff --git a/src/puffin_stream.cc b/src/puffin_stream.cc
index 3fb54a1..05dd7f6 100644
--- a/src/puffin_stream.cc
+++ b/src/puffin_stream.cc
@@ -28,7 +28,7 @@ using std::shared_ptr;
namespace {
-bool CheckArgsIntegrity(size_t puff_size,
+bool CheckArgsIntegrity(uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs) {
TEST_AND_RETURN_FALSE(puffs.size() == deflates.size());
@@ -57,7 +57,7 @@ bool CheckArgsIntegrity(size_t puff_size,
UniqueStreamPtr PuffinStream::CreateForPuff(
UniqueStreamPtr stream,
std::shared_ptr<Puffer> puffer,
- size_t puff_size,
+ uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs,
size_t max_cache_size) {
@@ -75,7 +75,7 @@ UniqueStreamPtr PuffinStream::CreateForPuff(
UniqueStreamPtr PuffinStream::CreateForHuff(
UniqueStreamPtr stream,
std::shared_ptr<Huffer> huffer,
- size_t puff_size,
+ uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs) {
TEST_AND_RETURN_VALUE(CheckArgsIntegrity(puff_size, deflates, puffs),
@@ -91,7 +91,7 @@ UniqueStreamPtr PuffinStream::CreateForHuff(
PuffinStream::PuffinStream(UniqueStreamPtr stream,
shared_ptr<Puffer> puffer,
shared_ptr<Huffer> huffer,
- size_t puff_size,
+ uint64_t puff_size,
const vector<BitExtent>& deflates,
const vector<ByteExtent>& puffs,
size_t max_cache_size)
@@ -120,7 +120,7 @@ PuffinStream::PuffinStream(UniqueStreamPtr stream,
// We can pass the size of the deflate stream too, but it is not necessary
// yet. We cannot get the size of stream from itself, because we might be
// writing into it and its size is not defined yet.
- size_t deflate_stream_size = puff_stream_size_;
+ uint64_t deflate_stream_size = puff_stream_size_;
if (!puffs.empty()) {
deflate_stream_size =
((deflates.back().offset + deflates.back().length) / 8) +
@@ -131,35 +131,33 @@ PuffinStream::PuffinStream(UniqueStreamPtr stream,
puffs_.emplace_back(puff_stream_size_, 0);
// Look for the largest puff and deflate extents and get proper size buffers.
- size_t max_puff_length = 0;
+ uint64_t max_puff_length = 0;
for (const auto& puff : puffs) {
- max_puff_length =
- std::max(max_puff_length, static_cast<size_t>(puff.length));
+ max_puff_length = std::max(max_puff_length, puff.length);
}
puff_buffer_.reset(new Buffer(max_puff_length + 1));
if (max_cache_size_ < max_puff_length) {
max_cache_size_ = 0; // It means we are not caching puffs.
}
- size_t max_deflate_length = 0;
+ uint64_t max_deflate_length = 0;
for (const auto& deflate : deflates) {
- max_deflate_length =
- std::max(max_deflate_length, static_cast<size_t>(deflate.length * 8));
+ max_deflate_length = std::max(max_deflate_length, deflate.length * 8);
}
deflate_buffer_.reset(new Buffer(max_deflate_length + 2));
}
-bool PuffinStream::GetSize(size_t* size) const {
+bool PuffinStream::GetSize(uint64_t* size) const {
*size = puff_stream_size_;
return true;
}
-bool PuffinStream::GetOffset(size_t* offset) const {
+bool PuffinStream::GetOffset(uint64_t* offset) const {
*offset = puff_pos_ + skip_bytes_;
return true;
}
-bool PuffinStream::Seek(size_t offset) {
+bool PuffinStream::Seek(uint64_t offset) {
TEST_AND_RETURN_FALSE(!closed_);
if (!is_for_puff_) {
// For huffing we should not seek, only seek to zero is accepted.
@@ -206,22 +204,23 @@ bool PuffinStream::Close() {
return stream_->Close();
}
-bool PuffinStream::Read(void* buffer, size_t length) {
+bool PuffinStream::Read(void* buffer, size_t count) {
TEST_AND_RETURN_FALSE(!closed_);
TEST_AND_RETURN_FALSE(is_for_puff_);
if (cur_puff_ == puffs_.end()) {
- TEST_AND_RETURN_FALSE(length == 0);
+ TEST_AND_RETURN_FALSE(count == 0);
}
auto bytes = static_cast<uint8_t*>(buffer);
- size_t bytes_read = 0;
+ uint64_t length = count;
+ uint64_t bytes_read = 0;
while (bytes_read < length) {
if (puff_pos_ < cur_puff_->offset) {
// Reading between two deflates. We also read bytes that have at least one
// bit of a deflate bit stream. The byte which has both deflate and raw
// data will be shifted or masked off the deflate bits and the remaining
// value will be saved in the puff stream as an byte integer.
- size_t start_byte = (deflate_bit_pos_ / 8);
- size_t end_byte = (cur_deflate_->offset + 7) / 8;
+ uint64_t start_byte = (deflate_bit_pos_ / 8);
+ uint64_t end_byte = (cur_deflate_->offset + 7) / 8;
auto bytes_to_read = std::min(length - bytes_read, end_byte - start_byte);
TEST_AND_RETURN_FALSE(bytes_to_read >= 1);
@@ -300,8 +299,7 @@ bool PuffinStream::Read(void* buffer, size_t length) {
}
// Copy from puff buffer to output if needed.
auto bytes_to_copy =
- std::min(length - bytes_read,
- static_cast<size_t>(cur_puff_->length) - skip_bytes_);
+ std::min(length - bytes_read, cur_puff_->length - skip_bytes_);
if (!puff_directly_into_buffer) {
memcpy(bytes + bytes_read, puff_buffer_->data() + skip_bytes_,
bytes_to_copy);
@@ -328,11 +326,12 @@ bool PuffinStream::Read(void* buffer, size_t length) {
return true;
}
-bool PuffinStream::Write(const void* buffer, size_t length) {
+bool PuffinStream::Write(const void* buffer, size_t count) {
TEST_AND_RETURN_FALSE(!closed_);
TEST_AND_RETURN_FALSE(!is_for_puff_);
auto bytes = static_cast<const uint8_t*>(buffer);
- size_t bytes_wrote = 0;
+ uint64_t length = count;
+ uint64_t bytes_wrote = 0;
while (bytes_wrote < length) {
if (deflate_bit_pos_ < (cur_deflate_->offset & ~7ull)) {
// Between two puffs or before the first puff. We know that we are
@@ -340,9 +339,9 @@ bool PuffinStream::Write(const void* buffer, size_t length) {
// non-deflate bits of the last byte of the last deflate. Here we don't
// process any byte that has deflate bit.
TEST_AND_RETURN_FALSE((deflate_bit_pos_ & 7) == 0);
- auto copy_len = std::min((static_cast<size_t>(cur_deflate_->offset) / 8) -
- (deflate_bit_pos_ / 8),
- length - bytes_wrote);
+ auto copy_len =
+ std::min((cur_deflate_->offset / 8) - (deflate_bit_pos_ / 8),
+ length - bytes_wrote);
TEST_AND_RETURN_FALSE(stream_->Write(bytes + bytes_wrote, copy_len));
bytes_wrote += copy_len;
puff_pos_ += copy_len;
@@ -364,9 +363,8 @@ bool PuffinStream::Write(const void* buffer, size_t length) {
TEST_AND_RETURN_FALSE(puff_pos_ == cur_puff_->offset);
}
- auto copy_len = std::min(
- length - bytes_wrote,
- static_cast<size_t>(cur_puff_->length) + extra_byte_ - skip_bytes_);
+ auto copy_len = std::min(length - bytes_wrote,
+ cur_puff_->length + extra_byte_ - skip_bytes_);
TEST_AND_RETURN_FALSE(puff_buffer_->size() >= skip_bytes_ + copy_len);
memcpy(puff_buffer_->data() + skip_bytes_, bytes + bytes_wrote, copy_len);
skip_bytes_ += copy_len;
@@ -435,7 +433,7 @@ bool PuffinStream::SetExtraByte() {
extra_byte_ = 0;
return true;
}
- size_t end_bit = cur_deflate_->offset + cur_deflate_->length;
+ uint64_t end_bit = cur_deflate_->offset + cur_deflate_->length;
if ((end_bit & 7) && ((end_bit + 7) & ~7ull) <= (cur_deflate_ + 1)->offset) {
extra_byte_ = 1;
} else {
@@ -445,7 +443,7 @@ bool PuffinStream::SetExtraByte() {
}
bool PuffinStream::GetPuffCache(int puff_id,
- size_t puff_size,
+ uint64_t puff_size,
SharedBufferPtr* buffer) {
bool found = false;
// Search for it.
@@ -477,7 +475,7 @@ bool PuffinStream::GetPuffCache(int puff_id,
}
cache.second->resize(puff_size);
- constexpr size_t kMaxSizeDifference = 20 * 1024;
+ constexpr uint64_t kMaxSizeDifference = 20 * 1024;
if (puff_size + kMaxSizeDifference < cache.second->capacity()) {
cache.second->shrink_to_fit();
}
diff --git a/src/puffin_stream.h b/src/puffin_stream.h
index f791ac5..47b8ace 100644
--- a/src/puffin_stream.h
+++ b/src/puffin_stream.h
@@ -45,7 +45,7 @@ class PuffinStream : public StreamInterface {
// and no puff will be cached.
static UniqueStreamPtr CreateForPuff(UniqueStreamPtr stream,
std::shared_ptr<Puffer> puffer,
- size_t puff_size,
+ uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs,
size_t max_cache_size = 0);
@@ -59,17 +59,17 @@ class PuffinStream : public StreamInterface {
// |puffs| IN The location of puffs into the input puff stream.
static UniqueStreamPtr CreateForHuff(UniqueStreamPtr stream,
std::shared_ptr<Huffer> huffer,
- size_t puff_size,
+ uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs);
- bool GetSize(size_t* size) const override;
+ bool GetSize(uint64_t* size) const override;
// Returns the current offset in the imaginary puff stream.
- bool GetOffset(size_t* offset) const override;
+ bool GetOffset(uint64_t* offset) const override;
// Sets the current offset in the imaginary puff stream.
- bool Seek(size_t offset) override;
+ bool Seek(uint64_t offset) override;
// Reads from the deflate stream |stream_| and writes the puff stream into
// |buffer|.
@@ -90,7 +90,7 @@ class PuffinStream : public StreamInterface {
PuffinStream(UniqueStreamPtr stream,
std::shared_ptr<Puffer> puffer,
std::shared_ptr<Huffer> huffer,
- size_t puff_size,
+ uint64_t puff_size,
const std::vector<BitExtent>& deflates,
const std::vector<ByteExtent>& puffs,
size_t max_cache_size);
@@ -102,7 +102,7 @@ class PuffinStream : public StreamInterface {
// Returns the cache for the |puff_id|th puff. If it does not find it, either
// returns the least accessed cached (if cache is full) or creates a new empty
// buffer. It returns false if it cannot find the |puff_id|th puff cache.
- bool GetPuffCache(int puff_id, size_t puff_size, SharedBufferPtr* buffer);
+ bool GetPuffCache(int puff_id, uint64_t puff_size, SharedBufferPtr* buffer);
UniqueStreamPtr stream_;
@@ -110,7 +110,7 @@ class PuffinStream : public StreamInterface {
std::shared_ptr<Huffer> huffer_;
// The size of the imaginary puff stream.
- size_t puff_stream_size_;
+ uint64_t puff_stream_size_;
std::vector<BitExtent> deflates_;
// The current deflate is being processed.
@@ -120,15 +120,15 @@ class PuffinStream : public StreamInterface {
// The current puff is being processed.
std::vector<ByteExtent>::iterator cur_puff_;
- std::vector<size_t> upper_bounds_;
+ std::vector<uint64_t> upper_bounds_;
// The current offset in the imaginary puff stream is |puff_pos_| +
// |skip_bytes_|
- size_t puff_pos_;
- size_t skip_bytes_;
+ uint64_t puff_pos_;
+ uint64_t skip_bytes_;
// The current bit offset in |stream_|.
- size_t deflate_bit_pos_;
+ uint64_t deflate_bit_pos_;
// This value caches the first or last byte of a deflate stream. This is
// needed when two deflate stream end on the same byte (with greater than zero
@@ -158,7 +158,7 @@ class PuffinStream : public StreamInterface {
// this class.
size_t max_cache_size_;
// The current amount of memory (in bytes) used for caching puff buffers.
- size_t cur_cache_size_;
+ uint64_t cur_cache_size_;
DISALLOW_COPY_AND_ASSIGN(PuffinStream);
};
diff --git a/src/puffin_unittest.cc b/src/puffin_unittest.cc
index ff785ec..fd035bc 100644
--- a/src/puffin_unittest.cc
+++ b/src/puffin_unittest.cc
@@ -199,7 +199,7 @@ class PuffinTest : public ::testing::Test {
auto deflate_stream = MemoryStream::CreateForRead(deflate_buffer);
ASSERT_TRUE(deflate_stream->Seek(0));
vector<ByteExtent> out_puff_extents;
- size_t puff_size;
+ uint64_t puff_size;
ASSERT_TRUE(FindPuffLocations(deflate_stream, deflate_extents,
&out_puff_extents, &puff_size));
EXPECT_EQ(puff_size, puff_buffer.size());
diff --git a/src/puffpatch.cc b/src/puffpatch.cc
index 1284196..185b1dd 100644
--- a/src/puffpatch.cc
+++ b/src/puffpatch.cc
@@ -52,8 +52,8 @@ bool DecodePatch(const uint8_t* patch,
vector<BitExtent>* dst_deflates,
vector<ByteExtent>* src_puffs,
vector<ByteExtent>* dst_puffs,
- size_t* src_puff_size,
- size_t* dst_puff_size) {
+ uint64_t* src_puff_size,
+ uint64_t* dst_puff_size) {
size_t offset = 0;
uint32_t header_size;
TEST_AND_RETURN_FALSE(patch_length >= (kMagicLength + sizeof(header_size)));
@@ -116,7 +116,7 @@ class BsdiffStream : public bsdiff::FileInterface {
bool Close() override { return stream_->Close(); }
bool GetSize(uint64_t* size) override {
- size_t my_size;
+ uint64_t my_size;
TEST_AND_RETURN_FALSE(stream_->GetSize(&my_size));
*size = my_size;
return true;
@@ -139,7 +139,7 @@ bool PuffPatch(UniqueStreamPtr src,
size_t bsdiff_patch_size = 0;
vector<BitExtent> src_deflates, dst_deflates;
vector<ByteExtent> src_puffs, dst_puffs;
- size_t src_puff_size, dst_puff_size;
+ uint64_t src_puff_size, dst_puff_size;
// Decode the patch and get the bsdiff_patch.
TEST_AND_RETURN_FALSE(DecodePatch(patch, patch_length, &bsdiff_patch_offset,
diff --git a/src/stream_unittest.cc b/src/stream_unittest.cc
index 32614bc..0a7c7b3 100644
--- a/src/stream_unittest.cc
+++ b/src/stream_unittest.cc
@@ -35,7 +35,7 @@ class StreamTest : public ::testing::Test {
// No reading out of data boundary.
Buffer tmp(100);
- size_t size;
+ uint64_t size;
ASSERT_TRUE(stream->GetSize(&size));
ASSERT_TRUE(stream->Seek(size));
ASSERT_TRUE(stream->Read(tmp.data(), 0));
@@ -76,8 +76,8 @@ class StreamTest : public ::testing::Test {
for (size_t idx = 0; idx < 10000; idx++) {
// zero to full size available.
size_t size = rand_r(&rand_seed) % (buf.size() + 1);
- size_t max_start = buf.size() - size;
- size_t start = rand_r(&rand_seed) % (max_start + 1);
+ uint64_t max_start = buf.size() - size;
+ uint64_t start = rand_r(&rand_seed) % (max_start + 1);
ASSERT_TRUE(stream->Seek(start));
ASSERT_TRUE(stream->Read(tmp.data(), size));
for (size_t idx = 0; idx < size; idx++) {
@@ -89,7 +89,7 @@ class StreamTest : public ::testing::Test {
void TestWriteBoundary(StreamInterface* stream) {
Buffer buf(10);
// Writing out of boundary is fine.
- size_t size;
+ uint64_t size;
ASSERT_TRUE(stream->GetSize(&size));
ASSERT_TRUE(stream->Seek(size));
ASSERT_TRUE(stream->Write(buf.data(), 0));
@@ -111,7 +111,7 @@ class StreamTest : public ::testing::Test {
}
void TestWrite(StreamInterface* write_stream, StreamInterface* read_stream) {
- size_t size;
+ uint64_t size;
ASSERT_TRUE(read_stream->GetSize(&size));
Buffer buf1(size);
Buffer buf2(size);
@@ -139,7 +139,7 @@ class StreamTest : public ::testing::Test {
// Call this at the end before |TestClose|.
void TestSeek(StreamInterface* stream, bool seek_end_is_fine) {
- size_t size, offset;
+ uint64_t size, offset;
ASSERT_TRUE(stream->GetSize(&size));
ASSERT_TRUE(stream->Seek(size));
ASSERT_TRUE(stream->GetOffset(&offset));
diff --git a/src/utils.cc b/src/utils.cc
index 2b41220..5019d0b 100644
--- a/src/utils.cc
+++ b/src/utils.cc
@@ -32,9 +32,9 @@ inline T get_unaligned(const void* address) {
// Calculate both the compressed size and uncompressed size of the deflate
// block that starts from the offset |start| of buffer |data|.
bool CalculateSizeOfDeflateBlock(const puffin::Buffer& data,
- size_t start,
- size_t* compressed_size,
- size_t* uncompressed_size) {
+ uint64_t start,
+ uint64_t* compressed_size,
+ uint64_t* uncompressed_size) {
TEST_AND_RETURN_FALSE(compressed_size != nullptr &&
uncompressed_size != nullptr);
@@ -79,8 +79,8 @@ namespace puffin {
using std::string;
using std::vector;
-size_t BytesInByteExtents(const vector<ByteExtent>& extents) {
- size_t bytes = 0;
+uint64_t BytesInByteExtents(const vector<ByteExtent>& extents) {
+ uint64_t bytes = 0;
for (const auto& extent : extents) {
bytes += extent.length;
}
@@ -113,7 +113,7 @@ bool LocateDeflatesInZlib(const Buffer& data,
auto flag = data[1];
TEST_AND_RETURN_FALSE(((cmf << 8) + flag) % 31 == 0);
- size_t header_len = 2;
+ uint64_t header_len = 2;
if (flag & 0x20) {
header_len += 4; // 4 bytes for the preset dictionary.
}
@@ -182,7 +182,7 @@ bool LocateDeflatesInZlibBlocks(const string& file_path,
// https://www.ietf.org/rfc/rfc1952.txt
bool LocateDeflatesInGzip(const Buffer& data,
vector<ByteExtent>* deflate_blocks) {
- size_t member_start = 0;
+ uint64_t member_start = 0;
while (member_start < data.size()) {
// Each member entry has the following format
// 0 1 0x1F
@@ -197,7 +197,7 @@ bool LocateDeflatesInGzip(const Buffer& data,
TEST_AND_RETURN_FALSE(data[member_start + 1] == 0x8B);
TEST_AND_RETURN_FALSE(data[member_start + 2] == 8);
- size_t offset = member_start + 10;
+ uint64_t offset = member_start + 10;
int flag = data[member_start + 3];
// Extra field
if (flag & 4) {
@@ -230,7 +230,7 @@ bool LocateDeflatesInGzip(const Buffer& data,
offset += 2;
}
- size_t compressed_size, uncompressed_size;
+ uint64_t compressed_size, uncompressed_size;
TEST_AND_RETURN_FALSE(CalculateSizeOfDeflateBlock(
data, offset, &compressed_size, &uncompressed_size));
TEST_AND_RETURN_FALSE(offset + compressed_size <= data.size());
@@ -254,7 +254,7 @@ bool LocateDeflatesInGzip(const Buffer& data,
// https://support.pkware.com/display/PKZIP/APPNOTE
bool LocateDeflatesInZipArchive(const Buffer& data,
vector<ByteExtent>* deflate_blocks) {
- size_t pos = 0;
+ uint64_t pos = 0;
while (pos <= data.size() - 30) {
// TODO(xunchang) add support for big endian system when searching for
// magic numbers.
@@ -295,8 +295,8 @@ bool LocateDeflatesInZipArchive(const Buffer& data,
continue;
}
- size_t calculated_compressed_size;
- size_t calculated_uncompressed_size;
+ uint64_t calculated_compressed_size;
+ uint64_t calculated_uncompressed_size;
if (!CalculateSizeOfDeflateBlock(data, pos + header_size,
&calculated_compressed_size,
&calculated_uncompressed_size)) {
@@ -342,7 +342,7 @@ bool LocateDeflateSubBlocksInZipArchive(const Buffer& data,
bool FindPuffLocations(const UniqueStreamPtr& src,
const vector<BitExtent>& deflates,
vector<ByteExtent>* puffs,
- size_t* out_puff_size) {
+ uint64_t* out_puff_size) {
Puffer puffer;
Buffer deflate_buffer;
@@ -350,7 +350,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src,
// puff. At the end we add this cummulative size difference to the size of the
// deflate stream to get the size of the puff stream. We use signed size
// because puff size could be smaller than deflate size.
- ssize_t total_size_difference = 0;
+ int64_t total_size_difference = 0;
for (auto deflate = deflates.begin(); deflate != deflates.end(); ++deflate) {
// Read from src into deflate_buffer.
auto start_byte = deflate->offset / 8;
@@ -361,7 +361,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src,
src->Read(deflate_buffer.data(), deflate_buffer.size()));
// Find the size of the puff.
BufferBitReader bit_reader(deflate_buffer.data(), deflate_buffer.size());
- size_t bits_to_skip = deflate->offset % 8;
+ uint64_t bits_to_skip = deflate->offset % 8;
TEST_AND_RETURN_FALSE(bit_reader.CacheBits(bits_to_skip));
bit_reader.DropBits(bits_to_skip);
@@ -389,7 +389,7 @@ bool FindPuffLocations(const UniqueStreamPtr& src,
start_byte = ((deflate->offset + 7) / 8);
end_byte = (deflate->offset + deflate->length) / 8;
- ssize_t deflate_length_in_bytes = end_byte - start_byte;
+ int64_t deflate_length_in_bytes = end_byte - start_byte;
// If there was no gap bits between the current and previous deflates, there
// will be no extra gap byte, so the offset will be shifted one byte back.
@@ -398,12 +398,12 @@ bool FindPuffLocations(const UniqueStreamPtr& src,
// Add the location into puff.
puffs->emplace_back(puff_offset, puff_size);
total_size_difference +=
- static_cast<ssize_t>(puff_size) - deflate_length_in_bytes - gap;
+ static_cast<int64_t>(puff_size) - deflate_length_in_bytes - gap;
}
- size_t src_size;
+ uint64_t src_size;
TEST_AND_RETURN_FALSE(src->GetSize(&src_size));
- auto final_size = static_cast<ssize_t>(src_size) + total_size_difference;
+ auto final_size = static_cast<int64_t>(src_size) + total_size_difference;
TEST_AND_RETURN_FALSE(final_size >= 0);
*out_puff_size = final_size;
return true;
diff --git a/src/utils_unittest.cc b/src/utils_unittest.cc
index 8d972d2..e1fbb25 100644
--- a/src/utils_unittest.cc
+++ b/src/utils_unittest.cc
@@ -102,10 +102,10 @@ void FindDeflatesInZlibBlocks(const Buffer& src,
void CheckFindPuffLocation(const Buffer& compressed,
const vector<BitExtent>& deflates,
const vector<ByteExtent>& expected_puffs,
- size_t expected_puff_size) {
+ uint64_t expected_puff_size) {
auto src = MemoryStream::CreateForRead(compressed);
vector<ByteExtent> puffs;
- size_t puff_size;
+ uint64_t puff_size;
ASSERT_TRUE(FindPuffLocations(src, deflates, &puffs, &puff_size));
EXPECT_EQ(puffs, expected_puffs);
EXPECT_EQ(puff_size, expected_puff_size);