Write out image bitmap inside of image file.

We now create the image bitmap when we generate the image. The image
bitmap is written after the image inside of the image file. This
speeds up dex2oat by making walking the image during heap creation
unnecessary. This should also help memory pressure by enabling the
image bitmap to be swappable.

Bug: 10432288

Change-Id: Idebf459ed15edbb41a7d9b9b353934155bce2f19

7 files changed, 100 insertions, 60 deletions

diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index 702e162262..63b24ffb4e 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -45,11 +45,19 @@ std::string SpaceBitmap::Dump() const {
 }
 
 void SpaceSetMap::Walk(SpaceBitmap::Callback* callback, void* arg) {
-  for (Objects::iterator it = contained_.begin(); it != contained_.end(); ++it) {
-    callback(const_cast<mirror::Object*>(*it), arg);
+  for (const mirror::Object* obj : contained_) {
+    callback(const_cast<mirror::Object*>(obj), arg);
   }
 }
 
+SpaceBitmap* SpaceBitmap::CreateFromMemMap(const std::string& name, MemMap* mem_map,
+                                           byte* heap_begin, size_t heap_capacity) {
+  CHECK(mem_map != nullptr);
+  word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
+  size_t bitmap_size = OffsetToIndex(RoundUp(heap_capacity, kAlignment * kBitsPerWord)) * kWordSize;
+  return new SpaceBitmap(name, mem_map, bitmap_begin, bitmap_size, heap_begin);
+}
+
 SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size_t heap_capacity) {
   CHECK(heap_begin != NULL);
   // Round up since heap_capacity is not necessarily a multiple of kAlignment * kBitsPerWord.
@@ -59,8 +67,7 @@ SpaceBitmap* SpaceBitmap::Create(const std::string& name, byte* heap_begin, size
     LOG(ERROR) << "Failed to allocate bitmap " << name;
     return NULL;
   }
-  word* bitmap_begin = reinterpret_cast<word*>(mem_map->Begin());
-  return new SpaceBitmap(name, mem_map.release(), bitmap_begin, bitmap_size, heap_begin);
+  return CreateFromMemMap(name, mem_map.release(), heap_begin, heap_capacity);
 }
 
 // Clean up any resources associated with the bitmap.
@@ -74,14 +81,11 @@ void SpaceBitmap::SetHeapLimit(uintptr_t new_end) {
   }
   // Not sure if doing this trim is necessary, since nothing past the end of the heap capacity
   // should be marked.
-  // TODO: Fix this code is, it broken and causes rare heap corruption!
-  // mem_map_->Trim(reinterpret_cast<byte*>(heap_begin_ + bitmap_size_));
 }
 
 void SpaceBitmap::Clear() {
   if (bitmap_begin_ != NULL) {
-    // This returns the memory to the system.  Successive page faults
-    // will return zeroed memory.
+    // This returns the memory to the system.  Successive page faults will return zeroed memory.
     int result = madvise(bitmap_begin_, bitmap_size_, MADV_DONTNEED);
     if (result == -1) {
       PLOG(FATAL) << "madvise failed";
diff --git a/runtime/gc/accounting/space_bitmap.h b/runtime/gc/accounting/space_bitmap.h
index 26ab1ded96..f975692a3f 100644
--- a/runtime/gc/accounting/space_bitmap.h
+++ b/runtime/gc/accounting/space_bitmap.h
@@ -48,10 +48,16 @@ class SpaceBitmap {
 
   typedef void SweepCallback(size_t ptr_count, mirror::Object** ptrs, void* arg);
 
-  // Initialize a HeapBitmap so that it points to a bitmap large enough to cover a heap at
+  // Initialize a space bitmap so that it points to a bitmap large enough to cover a heap at
   // heap_begin of heap_capacity bytes, where objects are guaranteed to be kAlignment-aligned.
   static SpaceBitmap* Create(const std::string& name, byte* heap_begin, size_t heap_capacity);
 
+  // Initialize a space bitmap using the provided mem_map as the live bits. Takes ownership of the
+  // mem map. The address range covered starts at heap_begin and is of size equal to heap_capacity.
+  // Objects are kAlignement-aligned.
+  static SpaceBitmap* CreateFromMemMap(const std::string& name, MemMap* mem_map,
+                                       byte* heap_begin, size_t heap_capacity);
+
   ~SpaceBitmap();
 
   // <offset> is the difference from .base to a pointer address.
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index e20c2c5f05..e0048a05f4 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -187,14 +187,6 @@ Heap::Heap(size_t initial_size, size_t growth_limit, size_t min_free, size_t max
     heap_capacity += continuous_spaces_.back()->AsDlMallocSpace()->NonGrowthLimitCapacity();
   }
 
-  // Mark image objects in the live bitmap.
-  for (const auto& space : continuous_spaces_) {
-    if (space->IsImageSpace()) {
-      space::ImageSpace* image_space = space->AsImageSpace();
-      image_space->RecordImageAllocations(image_space->GetLiveBitmap());
-    }
-  }
-
   // Allocate the card table.
   card_table_.reset(accounting::CardTable::Create(heap_begin, heap_capacity));
   CHECK(card_table_.get() != NULL) << "Failed to create card table";
diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index f959cff7b4..e2e5bf7d3d 100644
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -35,15 +35,13 @@ namespace art {
 namespace gc {
 namespace space {
 
-size_t ImageSpace::bitmap_index_ = 0;
-
-ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map)
-: MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
-  const size_t bitmap_index = bitmap_index_++;
-  live_bitmap_.reset(accounting::SpaceBitmap::Create(
-      StringPrintf("imagespace %s live-bitmap %d", name.c_str(), static_cast<int>(bitmap_index)),
-      Begin(), Capacity()));
-  DCHECK(live_bitmap_.get() != NULL) << "could not create imagespace live bitmap #" << bitmap_index;
+AtomicInteger ImageSpace::bitmap_index_(0);
+
+ImageSpace::ImageSpace(const std::string& name, MemMap* mem_map,
+                       accounting::SpaceBitmap* live_bitmap)
+    : MemMapSpace(name, mem_map, mem_map->Size(), kGcRetentionPolicyNeverCollect) {
+  DCHECK(live_bitmap != NULL);
+  live_bitmap_.reset(live_bitmap);
 }
 
 static bool GenerateImage(const std::string& image_file_name) {
@@ -151,6 +149,17 @@ ImageSpace* ImageSpace::Create(const std::string& original_image_file_name) {
   return space::ImageSpace::Init(image_file_name, true);
 }
 
+void ImageSpace::VerifyImageAllocations() {
+  byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
+  while (current < End()) {
+    DCHECK_ALIGNED(current, kObjectAlignment);
+    const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
+    CHECK(live_bitmap_->Test(obj));
+    CHECK(obj->GetClass() != nullptr) << "Image object at address " << obj << " has null class";
+    current += RoundUp(obj->SizeOf(), kObjectAlignment);
+  }
+}
+
 ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_oat_file) {
   CHECK(!image_file_name.empty());
 
@@ -171,8 +180,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
     LOG(ERROR) << "Invalid image header " << image_file_name;
     return NULL;
   }
+
+  // Note: The image header is part of the image due to mmap page alignment required of offset.
   UniquePtr<MemMap> map(MemMap::MapFileAtAddress(image_header.GetImageBegin(),
-                                                 file->GetLength(),
+                                                 image_header.GetImageSize(),
                                                  PROT_READ | PROT_WRITE,
                                                  MAP_PRIVATE | MAP_FIXED,
                                                  file->Fd(),
@@ -185,6 +196,20 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
   CHECK_EQ(image_header.GetImageBegin(), map->Begin());
   DCHECK_EQ(0, memcmp(&image_header, map->Begin(), sizeof(ImageHeader)));
 
+  UniquePtr<MemMap> image_map(MemMap::MapFileAtAddress(nullptr, image_header.GetImageBitmapSize(),
+                                                       PROT_READ, MAP_PRIVATE,
+                                                       file->Fd(), image_header.GetBitmapOffset(),
+                                                       false));
+  CHECK(image_map.get() != nullptr) << "failed to map image bitmap";
+  size_t bitmap_index = bitmap_index_.fetch_add(1);
+  std::string bitmap_name(StringPrintf("imagespace %s live-bitmap %u", image_file_name.c_str(),
+                                       bitmap_index));
+  UniquePtr<accounting::SpaceBitmap> bitmap(
+      accounting::SpaceBitmap::CreateFromMemMap(bitmap_name, image_map.release(),
+                                                reinterpret_cast<byte*>(map->Begin()),
+                                                map->Size()));
+  CHECK(bitmap.get() != nullptr) << "could not create " << bitmap_name;
+
   Runtime* runtime = Runtime::Current();
   mirror::Object* resolution_method = image_header.GetImageRoot(ImageHeader::kResolutionMethod);
   runtime->SetResolutionMethod(down_cast<mirror::ArtMethod*>(resolution_method));
@@ -196,7 +221,10 @@ ImageSpace* ImageSpace::Init(const std::string& image_file_name, bool validate_o
   callee_save_method = image_header.GetImageRoot(ImageHeader::kRefsAndArgsSaveMethod);
   runtime->SetCalleeSaveMethod(down_cast<mirror::ArtMethod*>(callee_save_method), Runtime::kRefsAndArgs);
 
-  UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release()));
+  UniquePtr<ImageSpace> space(new ImageSpace(image_file_name, map.release(), bitmap.release()));
+  if (kIsDebugBuild) {
+    space->VerifyImageAllocations();
+  }
 
   space->oat_file_.reset(space->OpenOatFile());
   if (space->oat_file_.get() == NULL) {
@@ -245,9 +273,7 @@ OatFile* ImageSpace::OpenOatFile() const {
 
 bool ImageSpace::ValidateOatFile() const {
   CHECK(oat_file_.get() != NULL);
-  std::vector<const OatFile::OatDexFile*> oat_dex_files = oat_file_->GetOatDexFiles();
-  for (size_t i = 0; i < oat_dex_files.size(); i++) {
-    const OatFile::OatDexFile* oat_dex_file = oat_dex_files[i];
+  for (const OatFile::OatDexFile* oat_dex_file : oat_file_->GetOatDexFiles()) {
     const std::string& dex_file_location = oat_dex_file->GetDexFileLocation();
     uint32_t dex_file_location_checksum;
     if (!DexFile::GetChecksum(dex_file_location.c_str(), dex_file_location_checksum)) {
@@ -270,28 +296,6 @@ OatFile& ImageSpace::ReleaseOatFile() {
   return *oat_file_.release();
 }
 
-void ImageSpace::RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const {
-  uint64_t start_time = 0;
-  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
-    LOG(INFO) << "ImageSpace::RecordImageAllocations entering";
-    start_time = NanoTime();
-  }
-  DCHECK(!Runtime::Current()->IsStarted());
-  CHECK(live_bitmap != NULL);
-  byte* current = Begin() + RoundUp(sizeof(ImageHeader), kObjectAlignment);
-  byte* end = End();
-  while (current < end) {
-    DCHECK_ALIGNED(current, kObjectAlignment);
-    const mirror::Object* obj = reinterpret_cast<const mirror::Object*>(current);
-    live_bitmap->Set(obj);
-    current += RoundUp(obj->SizeOf(), kObjectAlignment);
-  }
-  if (VLOG_IS_ON(heap) || VLOG_IS_ON(startup)) {
-    LOG(INFO) << "ImageSpace::RecordImageAllocations exiting ("
-        << PrettyDuration(NanoTime() - start_time) << ")";
-  }
-}
-
 void ImageSpace::Dump(std::ostream& os) const {
   os << GetType()
       << "begin=" << reinterpret_cast<void*>(Begin())
diff --git a/runtime/gc/space/image_space.h b/runtime/gc/space/image_space.h
index bdda9fa4b1..381a98e386 100644
--- a/runtime/gc/space/image_space.h
+++ b/runtime/gc/space/image_space.h
@@ -53,6 +53,9 @@ class ImageSpace : public MemMapSpace {
   OatFile& ReleaseOatFile()
       SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
 
+  void VerifyImageAllocations()
+      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
+
   const ImageHeader& GetImageHeader() const {
     return *reinterpret_cast<ImageHeader*>(Begin());
   }
@@ -61,10 +64,6 @@ class ImageSpace : public MemMapSpace {
     return GetName();
   }
 
-  // Mark the objects defined in this space in the given live bitmap
-  void RecordImageAllocations(accounting::SpaceBitmap* live_bitmap) const
-      SHARED_LOCKS_REQUIRED(Locks::mutator_lock_);
-
   accounting::SpaceBitmap* GetLiveBitmap() const {
     return live_bitmap_.get();
   }
@@ -96,11 +95,11 @@ class ImageSpace : public MemMapSpace {
 
   friend class Space;
 
-  static size_t bitmap_index_;
+  static AtomicInteger bitmap_index_;
 
   UniquePtr<accounting::SpaceBitmap> live_bitmap_;
 
-  ImageSpace(const std::string& name, MemMap* mem_map);
+  ImageSpace(const std::string& name, MemMap* mem_map, accounting::SpaceBitmap* live_bitmap);
 
   // The OatFile associated with the image during early startup to
   // reserve space contiguous to the image. It is later released to
diff --git a/runtime/image.cc b/runtime/image.cc
index 686a117c99..d11594c905 100644
--- a/runtime/image.cc
+++ b/runtime/image.cc
@@ -24,9 +24,12 @@
 namespace art {
 
 const byte ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' };
-const byte ImageHeader::kImageVersion[] = { '0', '0', '4', '\0' };
+const byte ImageHeader::kImageVersion[] = { '0', '0', '5', '\0' };
 
 ImageHeader::ImageHeader(uint32_t image_begin,
+                         uint32_t image_size,
+                         uint32_t image_bitmap_offset,
+                         uint32_t image_bitmap_size,
                          uint32_t image_roots,
                          uint32_t oat_checksum,
                          uint32_t oat_file_begin,
@@ -34,6 +37,9 @@ ImageHeader::ImageHeader(uint32_t image_begin,
                          uint32_t oat_data_end,
                          uint32_t oat_file_end)
   : image_begin_(image_begin),
+    image_size_(image_size),
+    image_bitmap_offset_(image_bitmap_offset),
+    image_bitmap_size_(image_bitmap_size),
     oat_checksum_(oat_checksum),
     oat_file_begin_(oat_file_begin),
     oat_data_begin_(oat_data_begin),
diff --git a/runtime/image.h b/runtime/image.h
index 35e4c5cabd..5119e3a495 100644
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -21,6 +21,7 @@
 
 #include "globals.h"
 #include "mirror/object.h"
+#include "utils.h"
 
 namespace art {
 
@@ -30,6 +31,9 @@ class PACKED(4) ImageHeader {
   ImageHeader() {}
 
   ImageHeader(uint32_t image_begin,
+              uint32_t image_size_,
+              uint32_t image_bitmap_offset,
+              uint32_t image_bitmap_size,
               uint32_t image_roots,
               uint32_t oat_checksum,
               uint32_t oat_file_begin,
@@ -56,6 +60,18 @@ class PACKED(4) ImageHeader {
     return reinterpret_cast<byte*>(image_begin_);
   }
 
+  size_t GetImageSize() const {
+    return static_cast<uint32_t>(image_size_);
+  }
+
+  size_t GetImageBitmapOffset() const {
+    return image_bitmap_offset_;
+  }
+
+  size_t GetImageBitmapSize() const {
+    return image_bitmap_size_;
+  }
+
   uint32_t GetOatChecksum() const {
     return oat_checksum_;
   }
@@ -80,6 +96,10 @@ class PACKED(4) ImageHeader {
     return reinterpret_cast<byte*>(oat_file_end_);
   }
 
+  size_t GetBitmapOffset() const {
+    return RoundUp(image_size_, kPageSize);
+  }
+
   enum ImageRoot {
     kResolutionMethod,
     kCalleeSaveMethod,
@@ -106,6 +126,15 @@ class PACKED(4) ImageHeader {
   // Required base address for mapping the image.
   uint32_t image_begin_;
 
+  // Image size, not page aligned.
+  uint32_t image_size_;
+
+  // Image bitmap offset in the file.
+  uint32_t image_bitmap_offset_;
+
+  // Size of the image bitmap.
+  uint32_t image_bitmap_size_;
+
   // Checksum of the oat file we link to for load time sanity check.
   uint32_t oat_checksum_;