diff options
| -rw-r--r-- | compiler/image_test.cc | 3 | ||||
| -rw-r--r-- | compiler/image_writer.cc | 42 | ||||
| -rw-r--r-- | compiler/image_writer.h | 8 | ||||
| -rw-r--r-- | oatdump/oatdump.cc | 3 | ||||
| -rw-r--r-- | runtime/gc/accounting/space_bitmap.cc | 20 | ||||
| -rw-r--r-- | runtime/gc/accounting/space_bitmap.h | 8 | ||||
| -rw-r--r-- | runtime/gc/heap.cc | 8 | ||||
| -rw-r--r-- | runtime/gc/space/image_space.cc | 76 | ||||
| -rw-r--r-- | runtime/gc/space/image_space.h | 11 | ||||
| -rw-r--r-- | runtime/image.cc | 8 | ||||
| -rw-r--r-- | runtime/image.h | 29 |
11 files changed, 151 insertions, 65 deletions
diff --git a/compiler/image_test.cc b/compiler/image_test.cc index 106ef9ac86..60bc3cc4a7 100644 --- a/compiler/image_test.cc +++ b/compiler/image_test.cc @@ -85,6 +85,8 @@ TEST_F(ImageTest, WriteRead) { ImageHeader image_header; file->ReadFully(&image_header, sizeof(image_header)); ASSERT_TRUE(image_header.IsValid()); + ASSERT_GE(image_header.GetImageBitmapOffset(), sizeof(image_header)); + ASSERT_NE(0U, image_header.GetImageBitmapSize()); gc::Heap* heap = Runtime::Current()->GetHeap(); ASSERT_EQ(1U, heap->GetContinuousSpaces().size()); @@ -136,6 +138,7 @@ TEST_F(ImageTest, WriteRead) { ASSERT_TRUE(heap->GetContinuousSpaces()[1]->IsDlMallocSpace()); gc::space::ImageSpace* image_space = heap->GetImageSpace(); + image_space->VerifyImageAllocations(); byte* image_begin = image_space->Begin(); byte* image_end = image_space->End(); CHECK_EQ(requested_image_base, reinterpret_cast<uintptr_t>(image_begin)); diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 548ea9eee4..d1859e6f98 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -29,6 +29,7 @@ #include "elf_writer.h" #include "gc/accounting/card_table-inl.h" #include "gc/accounting/heap_bitmap.h" +#include "gc/accounting/space_bitmap-inl.h" #include "gc/heap.h" #include "gc/space/large_object_space.h" #include "gc/space/space-inl.h" @@ -136,9 +137,12 @@ bool ImageWriter::Write(const std::string& image_filename, CalculateNewObjectOffsets(oat_loaded_size, oat_data_offset); CopyAndFixupObjects(); PatchOatCodeAndMethods(); + // Record allocations into the image bitmap. + RecordImageAllocations(); Thread::Current()->TransitionFromRunnableToSuspended(kNative); UniquePtr<File> image_file(OS::CreateEmptyFile(image_filename.c_str())); + ImageHeader* image_header = reinterpret_cast<ImageHeader*>(image_->Begin()); if (image_file.get() == NULL) { LOG(ERROR) << "Failed to open image file " << image_filename; return false; @@ -147,14 +151,37 @@ bool ImageWriter::Write(const std::string& image_filename, PLOG(ERROR) << "Failed to make image file world readable: " << image_filename; return EXIT_FAILURE; } - bool success = image_file->WriteFully(image_->Begin(), image_end_); - if (!success) { + + // Write out the image. + CHECK_EQ(image_end_, image_header->GetImageSize()); + if (!image_file->WriteFully(image_->Begin(), image_end_)) { + PLOG(ERROR) << "Failed to write image file " << image_filename; + return false; + } + + // Write out the image bitmap at the page aligned start of the image end. + CHECK_ALIGNED(image_header->GetImageBitmapOffset(), kPageSize); + if (!image_file->Write(reinterpret_cast<char*>(image_bitmap_->Begin()), + image_header->GetImageBitmapSize(), + image_header->GetImageBitmapOffset())) { PLOG(ERROR) << "Failed to write image file " << image_filename; return false; } + return true; } +void ImageWriter::RecordImageAllocations() { + uint64_t start_time = NanoTime(); + CHECK(image_bitmap_.get() != nullptr); + for (const auto& it : offsets_) { + mirror::Object* obj = reinterpret_cast<mirror::Object*>(image_->Begin() + it.second); + DCHECK_ALIGNED(obj, kObjectAlignment); + image_bitmap_->Set(obj); + } + LOG(INFO) << "RecordImageAllocations took " << PrettyDuration(NanoTime() - start_time); +} + bool ImageWriter::AllocMemory() { size_t size = 0; for (const auto& space : Runtime::Current()->GetHeap()->GetContinuousSpaces()) { @@ -388,7 +415,7 @@ void ImageWriter::CalculateNewObjectOffsets(size_t oat_loaded_size, size_t oat_d DCHECK(!spaces.empty()); DCHECK_EQ(0U, image_end_); - // leave space for the header, but do not write it yet, we need to + // Leave space for the header, but do not write it yet, we need to // know where image_roots is going to end up image_end_ += RoundUp(sizeof(ImageHeader), 8); // 64-bit-alignment @@ -406,13 +433,20 @@ void ImageWriter::CalculateNewObjectOffsets(size_t oat_loaded_size, size_t oat_d self->EndAssertNoThreadSuspension(old); } + // Create the image bitmap. + image_bitmap_.reset(gc::accounting::SpaceBitmap::Create("image bitmap", image_->Begin(), + image_end_)); const byte* oat_file_begin = image_begin_ + RoundUp(image_end_, kPageSize); const byte* oat_file_end = oat_file_begin + oat_loaded_size; oat_data_begin_ = oat_file_begin + oat_data_offset; const byte* oat_data_end = oat_data_begin_ + oat_file_->Size(); - // return to write header at start of image with future location of image_roots + // Return to write header at start of image with future location of image_roots. At this point, + // image_end_ is the size of the image (excluding bitmaps). ImageHeader image_header(reinterpret_cast<uint32_t>(image_begin_), + static_cast<uint32_t>(image_end_), + RoundUp(image_end_, kPageSize), + image_bitmap_->Size(), reinterpret_cast<uint32_t>(GetImageAddress(image_roots.get())), oat_file_->GetOatHeader().GetChecksum(), reinterpret_cast<uint32_t>(oat_file_begin), diff --git a/compiler/image_writer.h b/compiler/image_writer.h index 6a126b84ab..0d85f36a5b 100644 --- a/compiler/image_writer.h +++ b/compiler/image_writer.h @@ -58,7 +58,10 @@ class ImageWriter { private: bool AllocMemory(); - // we use the lock word to store the offset of the object in the image + // Mark the objects defined in this space in the given live bitmap. + void RecordImageAllocations() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + + // We use the lock word to store the offset of the object in the image. void AssignImageOffset(mirror::Object* object) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { DCHECK(object != NULL); @@ -194,6 +197,9 @@ class ImageWriter { // Beginning target oat address for the pointers from the output image to its oat file. const byte* oat_data_begin_; + // Image bitmap which lets us know where the objects inside of the image reside. + UniquePtr<gc::accounting::SpaceBitmap> image_bitmap_; + // Offset from oat_data_begin_ to the stubs. uint32_t interpreter_to_interpreter_bridge_offset_; uint32_t interpreter_to_compiled_code_bridge_offset_; diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index 02273817c9..d683c8effc 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -675,6 +675,9 @@ class ImageDumper { os << "IMAGE BEGIN: " << reinterpret_cast<void*>(image_header_.GetImageBegin()) << "\n\n"; + os << "IMAGE BITMAP OFFSET: " << reinterpret_cast<void*>(image_header_.GetImageBitmapOffset()) + << " SIZE: " << reinterpret_cast<void*>(image_header_.GetImageBitmapSize()) << "\n\n"; + os << "OAT CHECKSUM: " << StringPrintf("0x%08x\n\n", image_header_.GetOatChecksum()); os << "OAT FILE BEGIN:" << reinterpret_cast<void*>(image_header_.GetOatFileBegin()) << "\n\n"; diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc index 702e162262..63b24ffb4e 100644 --- a/runtime/gc/accounting/space_bitmap.cc +++ b/runtime/gc/accounting/space_bitmap.cc @@ -45,11 +45,19 @@ std::string SpaceBitmap::Dump() const { } void SpaceSetMap::Walk(SpaceBitmap::Callback* callback, void* arg) { - for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) { - callback(const_cast<mirror::Object*>(*it), arg); + for (const mirror::Object* obj : contained_) { + callback(const_cast<mirror::Object*>(obj), arg); } } +SpaceBitmap* SpaceBitmap::CreateFromMemMap(const std::string& name, MemMap* mem_map, + byte* heap_begin, size_t heap_capacity) { + CHECK(mem_map != nullptr); + word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin()); + size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize; + return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin); +} + SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) { CHECK(heap_begin != NULL); // Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord. @@ -59,8 +67,7 @@ SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size LOG(ERROR) << "Failed to allocate bitmap " << name; return NULL; } - word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin()); - return new SpaceBitmap(name, mem_map.release(), bitmap_begin, bitmap_size, heap_begin); + return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity); } // Clean up any resources associated with the bitmap. @@ -74,14 +81,11 @@ void SpaceBitmap::SetHeapLimit(uintptr_t new_end) { } // Not sure if doing this trim is necessary, since nothing past the end of the heap capacity // should be marked. - // TODO: Fix this code is, it broken and causes rare heap corruption! - // mem_map_->Trim(reinterpret_cast<byte*>(heap_begin_ + bitmap_size_)); } void SpaceBitmap::Clear() { if (bitmap_begin_ != NULL) { - // This returns the memory to the system. Successive page faults - // will return zeroed memory. + // This returns the memory to the system. Successive page faults will return zeroed memory. int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED); if (result == -1) { PLOG(FATAL) << "madvise failed"; diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h index 26ab1ded96..f975692a3f 100644 --- a/runtime/gc/accounting/space_bitmap.h +++ b/runtime/gc/accounting/space_bitmap.h @@ -48,10 +48,16 @@ class SpaceBitmap { typedef void SweepCallback(size_t ptr_count, mirror::Object** ptrs, void* arg); - // Initialize a HeapBitmap so that it points to a bitmap large enough to cover a heap at + // Initialize a space bitmap so that it points to a bitmap large enough to cover a heap at // heap_begin of heap_capacity bytes, where objects are guaranteed to be kAlignment-aligned. static SpaceBitmap* Create(const std::string& name, byte* heap_begin, size_t heap_capacity); + // Initialize a space bitmap using the provided mem_map as the live bits. Takes ownership of the + // mem map. The address range covered starts at heap_begin and is of size equal to heap_capacity. + // Objects are kAlignement-aligned. + static SpaceBitmap* CreateFromMemMap(const std::string& name, MemMap* mem_map, + byte* heap_begin, size_t heap_capacity); + ~SpaceBitmap(); // <offset> is the difference from .base to a pointer address. diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc index e20c2c5f05..e0048a05f4 100644 --- a/runtime/gc/heap.cc +++ b/runtime/gc/heap.cc @@ -187,14 +187,6 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max heap_capacity += continuous_spaces_.back()->AsDlMallocSpace()->NonGrowthLimitCapacity(); } - // Mark image objects in the live bitmap. - for (const auto& space : continuous_spaces_) { - if (space->IsImageSpace()) { - space::ImageSpace* image_space = space->AsImageSpace(); - image_space->RecordImageAllocations(image_space->GetLiveBitmap()); - } - } - // Allocate the card table. card_table_.reset(accounting::CardTable::Create(heap_begin, heap_capacity)); CHECK(card_table_.get() != NULL) << "Failed to create card table"; diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index f959cff7b4..e2e5bf7d3d 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -35,15 +35,13 @@ namespace art { namespace gc { namespace space { -size_t ImageSpace::bitmap_index_ = 0; - -ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map) -: MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) { - const size_t bitmap_index = bitmap_index_++; - live_bitmap_.reset(accounting::SpaceBitmap::Create( - StringPrintf("imagespace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)), - Begin(), Capacity())); - DCHECK(live_bitmap_.get() != NULL) << "could not create imagespace live bitmap #" << bitmap_index; +AtomicInteger ImageSpace::bitmap_index_(0); + +ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map, + accounting::SpaceBitmap* live_bitmap) + : MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) { + DCHECK(live_bitmap != NULL); + live_bitmap_.reset(live_bitmap); } static bool GenerateImage(const std::string& image_file_name) { @@ -151,6 +149,17 @@ ImageSpace* ImageSpace::Create(const std::string& original_image_file_name) { return space::ImageSpace::Init(image_file_name, true); } +void ImageSpace::VerifyImageAllocations() { + byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment); + while (current < End()) { + DCHECK_ALIGNED(current, kObjectAlignment); + const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current); + CHECK(live_bitmap_->Test(obj)); + CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class"; + current += RoundUp(obj->SizeOf(), kObjectAlignment); + } +} + ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_oat_file) { CHECK(!image_file_name.empty()); @@ -171,8 +180,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o LOG(ERROR) << "Invalid image header " << image_file_name; return NULL; } + + // Note: The image header is part of the image due to mmap page alignment required of offset. UniquePtr<MemMap> map(MemMap::MapFileAtAddress(image_header.GetImageBegin(), - file->GetLength(), + image_header.GetImageSize(), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED, file->Fd(), @@ -185,6 +196,20 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o CHECK_EQ(image_header.GetImageBegin(), map->Begin()); DCHECK_EQ(0, memcmp(&image_header, map->Begin(), sizeof(ImageHeader))); + UniquePtr<MemMap> image_map(MemMap::MapFileAtAddress(nullptr, image_header.GetImageBitmapSize(), + PROT_READ, MAP_PRIVATE, + file->Fd(), image_header.GetBitmapOffset(), + false)); + CHECK(image_map.get() != nullptr) << "failed to map image bitmap"; + size_t bitmap_index = bitmap_index_.fetch_add(1); + std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_file_name.c_str(), + bitmap_index)); + UniquePtr<accounting::SpaceBitmap> bitmap( + accounting::SpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(), + reinterpret_cast<byte*>(map->Begin()), + map->Size())); + CHECK(bitmap.get() != nullptr) << "could not create " << bitmap_name; + Runtime* runtime = Runtime::Current(); mirror::Object* resolution_method = image_header.GetImageRoot(ImageHeader::kResolutionMethod); runtime->SetResolutionMethod(down_cast<mirror::ArtMethod*>(resolution_method)); @@ -196,7 +221,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o callee_save_method = image_header.GetImageRoot(ImageHeader::kRefsAndArgsSaveMethod); runtime->SetCalleeSaveMethod(down_cast<mirror::ArtMethod*>(callee_save_method), Runtime::kRefsAndArgs); - UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release())); + UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release(), bitmap.release())); + if (kIsDebugBuild) { + space->VerifyImageAllocations(); + } space->oat_file_.reset(space->OpenOatFile()); if (space->oat_file_.get() == NULL) { @@ -245,9 +273,7 @@ OatFile* ImageSpace::OpenOatFile() const { bool ImageSpace::ValidateOatFile() const { CHECK(oat_file_.get() != NULL); - std::vector<const OatFile::OatDexFile*> oat_dex_files = oat_file_->GetOatDexFiles(); - for (size_t i = 0; i < oat_dex_files.size(); i++) { - const OatFile::OatDexFile* oat_dex_file = oat_dex_files[i]; + for (const OatFile::OatDexFile* oat_dex_file : oat_file_->GetOatDexFiles()) { const std::string& dex_file_location = oat_dex_file->GetDexFileLocation(); uint32_t dex_file_location_checksum; if (!DexFile::GetChecksum(dex_file_location.c_str(), dex_file_location_checksum)) { @@ -270,28 +296,6 @@ OatFile& ImageSpace::ReleaseOatFile() { return *oat_file_.release(); } -void ImageSpace::RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const { - uint64_t start_time = 0; - if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { - LOG(INFO) << "ImageSpace::RecordImageAllocations entering"; - start_time = NanoTime(); - } - DCHECK(!Runtime::Current()->IsStarted()); - CHECK(live_bitmap != NULL); - byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment); - byte* end = End(); - while (current < end) { - DCHECK_ALIGNED(current, kObjectAlignment); - const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current); - live_bitmap->Set(obj); - current += RoundUp(obj->SizeOf(), kObjectAlignment); - } - if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) { - LOG(INFO) << "ImageSpace::RecordImageAllocations exiting (" - << PrettyDuration(NanoTime() - start_time) << ")"; - } -} - void ImageSpace::Dump(std::ostream& os) const { os << GetType() << "begin=" << reinterpret_cast<void*>(Begin()) diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h index bdda9fa4b1..381a98e386 100644 --- a/runtime/gc/space/image_space.h +++ b/runtime/gc/space/image_space.h @@ -53,6 +53,9 @@ class ImageSpace : public MemMapSpace { OatFile& ReleaseOatFile() SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + void VerifyImageAllocations() + SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); + const ImageHeader& GetImageHeader() const { return *reinterpret_cast<ImageHeader*>(Begin()); } @@ -61,10 +64,6 @@ class ImageSpace : public MemMapSpace { return GetName(); } - // Mark the objects defined in this space in the given live bitmap - void RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const - SHARED_LOCKS_REQUIRED(Locks::mutator_lock_); - accounting::SpaceBitmap* GetLiveBitmap() const { return live_bitmap_.get(); } @@ -96,11 +95,11 @@ class ImageSpace : public MemMapSpace { friend class Space; - static size_t bitmap_index_; + static AtomicInteger bitmap_index_; UniquePtr<accounting::SpaceBitmap> live_bitmap_; - ImageSpace(const std::string& name, MemMap* mem_map); + ImageSpace(const std::string& name, MemMap* mem_map, accounting::SpaceBitmap* live_bitmap); // The OatFile associated with the image during early startup to // reserve space contiguous to the image. It is later released to diff --git a/runtime/image.cc b/runtime/image.cc index 686a117c99..d11594c905 100644 --- a/runtime/image.cc +++ b/runtime/image.cc @@ -24,9 +24,12 @@ namespace art { const byte ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' }; -const byte ImageHeader::kImageVersion[] = { '0', '0', '4', '\0' }; +const byte ImageHeader::kImageVersion[] = { '0', '0', '5', '\0' }; ImageHeader::ImageHeader(uint32_t image_begin, + uint32_t image_size, + uint32_t image_bitmap_offset, + uint32_t image_bitmap_size, uint32_t image_roots, uint32_t oat_checksum, uint32_t oat_file_begin, @@ -34,6 +37,9 @@ ImageHeader::ImageHeader(uint32_t image_begin, uint32_t oat_data_end, uint32_t oat_file_end) : image_begin_(image_begin), + image_size_(image_size), + image_bitmap_offset_(image_bitmap_offset), + image_bitmap_size_(image_bitmap_size), oat_checksum_(oat_checksum), oat_file_begin_(oat_file_begin), oat_data_begin_(oat_data_begin), diff --git a/runtime/image.h b/runtime/image.h index 35e4c5cabd..5119e3a495 100644 --- a/runtime/image.h +++ b/runtime/image.h @@ -21,6 +21,7 @@ #include "globals.h" #include "mirror/object.h" +#include "utils.h" namespace art { @@ -30,6 +31,9 @@ class PACKED(4) ImageHeader { ImageHeader() {} ImageHeader(uint32_t image_begin, + uint32_t image_size_, + uint32_t image_bitmap_offset, + uint32_t image_bitmap_size, uint32_t image_roots, uint32_t oat_checksum, uint32_t oat_file_begin, @@ -56,6 +60,18 @@ class PACKED(4) ImageHeader { return reinterpret_cast<byte*>(image_begin_); } + size_t GetImageSize() const { + return static_cast<uint32_t>(image_size_); + } + + size_t GetImageBitmapOffset() const { + return image_bitmap_offset_; + } + + size_t GetImageBitmapSize() const { + return image_bitmap_size_; + } + uint32_t GetOatChecksum() const { return oat_checksum_; } @@ -80,6 +96,10 @@ class PACKED(4) ImageHeader { return reinterpret_cast<byte*>(oat_file_end_); } + size_t GetBitmapOffset() const { + return RoundUp(image_size_, kPageSize); + } + enum ImageRoot { kResolutionMethod, kCalleeSaveMethod, @@ -106,6 +126,15 @@ class PACKED(4) ImageHeader { // Required base address for mapping the image. uint32_t image_begin_; + // Image size, not page aligned. + uint32_t image_size_; + + // Image bitmap offset in the file. + uint32_t image_bitmap_offset_; + + // Size of the image bitmap. + uint32_t image_bitmap_size_; + // Checksum of the oat file we link to for load time sanity check. uint32_t oat_checksum_; |
