diff options
author | Andreas Gampe <agampe@google.com> | 2014-12-08 16:59:43 -0800 |
---|---|---|
committer | Andreas Gampe <agampe@google.com> | 2014-12-22 10:01:27 -0800 |
commit | e21dc3db191df04c100620965bee4617b3b24397 (patch) | |
tree | 2ad762c6afb024bf95e1eced3d584649a4d57d23 | |
parent | 6d1a047b4b3f9707d4ee1cc19e99717ee021ef48 (diff) | |
download | art-e21dc3db191df04c100620965bee4617b3b24397.tar.gz art-e21dc3db191df04c100620965bee4617b3b24397.tar.bz2 art-e21dc3db191df04c100620965bee4617b3b24397.zip |
ART: Swap-space in the compiler
Introduce a swap-space and corresponding allocator to transparently
switch native allocations to memory backed by a file.
Bug: 18596910
(cherry picked from commit 62746d8d9c4400e4764f162b22bfb1a32be287a9)
Change-Id: I131448f3907115054a592af73db86d2b9257ea33
29 files changed, 1072 insertions, 239 deletions
diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index 4c19ba0b4c..7f82726887 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -173,6 +173,7 @@ COMPILER_GTEST_COMMON_SRC_FILES := \ compiler/output_stream_test.cc \ compiler/utils/arena_allocator_test.cc \ compiler/utils/dedupe_set_test.cc \ + compiler/utils/swap_space_test.cc \ compiler/utils/arm/managed_register_arm_test.cc \ compiler/utils/arm64/managed_register_arm64_test.cc \ compiler/utils/x86/managed_register_x86_test.cc \ diff --git a/compiler/Android.mk b/compiler/Android.mk index 8bcc2f99ec..db338f0538 100644 --- a/compiler/Android.mk +++ b/compiler/Android.mk @@ -127,6 +127,7 @@ LIBART_COMPILER_SRC_FILES := \ utils/x86_64/assembler_x86_64.cc \ utils/x86_64/managed_register_x86_64.cc \ utils/scoped_arena_allocator.cc \ + utils/swap_space.cc \ buffered_output_stream.cc \ compiler.cc \ elf_writer.cc \ diff --git a/compiler/common_compiler_test.cc b/compiler/common_compiler_test.cc index 059a9eea50..d5c5e601e7 100644 --- a/compiler/common_compiler_test.cc +++ b/compiler/common_compiler_test.cc @@ -48,16 +48,16 @@ void CommonCompilerTest::MakeExecutable(mirror::ArtMethod* method) { method->GetDexMethodIndex())); } if (compiled_method != nullptr) { - const std::vector<uint8_t>* code = compiled_method->GetQuickCode(); + const SwapVector<uint8_t>* code = compiled_method->GetQuickCode(); uint32_t code_size = code->size(); CHECK_NE(0u, code_size); - const std::vector<uint8_t>& vmap_table = compiled_method->GetVmapTable(); + const SwapVector<uint8_t>& vmap_table = compiled_method->GetVmapTable(); uint32_t vmap_table_offset = vmap_table.empty() ? 0u : sizeof(OatQuickMethodHeader) + vmap_table.size(); - const std::vector<uint8_t>& mapping_table = *compiled_method->GetMappingTable(); + const SwapVector<uint8_t>& mapping_table = *compiled_method->GetMappingTable(); uint32_t mapping_table_offset = mapping_table.empty() ? 0u : sizeof(OatQuickMethodHeader) + vmap_table.size() + mapping_table.size(); - const std::vector<uint8_t>& gc_map = *compiled_method->GetGcMap(); + const SwapVector<uint8_t>& gc_map = *compiled_method->GetGcMap(); uint32_t gc_map_offset = gc_map.empty() ? 0u : sizeof(OatQuickMethodHeader) + vmap_table.size() + mapping_table.size() + gc_map.size(); OatQuickMethodHeader method_header(mapping_table_offset, vmap_table_offset, gc_map_offset, @@ -156,7 +156,7 @@ void CommonCompilerTest::SetUp() { compiler_kind, instruction_set, instruction_set_features_.get(), true, new std::set<std::string>, nullptr, - 2, true, true, timer_.get(), "")); + 2, true, true, timer_.get(), -1, "")); } // We typically don't generate an image in unit tests, disable this optimization by default. compiler_driver_->SetSupportBootImageFixup(false); diff --git a/compiler/compiled_method.cc b/compiler/compiled_method.cc index 060af723a7..234e8b96f6 100644 --- a/compiler/compiled_method.cc +++ b/compiler/compiled_method.cc @@ -20,13 +20,13 @@ namespace art { CompiledCode::CompiledCode(CompilerDriver* compiler_driver, InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code) + const ArrayRef<const uint8_t>& quick_code) : compiler_driver_(compiler_driver), instruction_set_(instruction_set), quick_code_(nullptr) { SetCode(&quick_code); } -void CompiledCode::SetCode(const std::vector<uint8_t>* quick_code) { +void CompiledCode::SetCode(const ArrayRef<const uint8_t>* quick_code) { if (quick_code != nullptr) { CHECK(!quick_code->empty()); quick_code_ = compiler_driver_->DeduplicateCode(*quick_code); @@ -108,61 +108,88 @@ void CompiledCode::AddOatdataOffsetToCompliledCodeOffset(uint32_t offset) { CompiledMethod::CompiledMethod(CompilerDriver* driver, InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code, + const ArrayRef<const uint8_t>& quick_code, const size_t frame_size_in_bytes, const uint32_t core_spill_mask, const uint32_t fp_spill_mask, - SrcMap* src_mapping_table, - const std::vector<uint8_t>& mapping_table, - const std::vector<uint8_t>& vmap_table, - const std::vector<uint8_t>& native_gc_map, - const std::vector<uint8_t>* cfi_info, + DefaultSrcMap* src_mapping_table, + const ArrayRef<const uint8_t>& mapping_table, + const ArrayRef<const uint8_t>& vmap_table, + const ArrayRef<const uint8_t>& native_gc_map, + const ArrayRef<const uint8_t>& cfi_info, const ArrayRef<LinkerPatch>& patches) : CompiledCode(driver, instruction_set, quick_code), frame_size_in_bytes_(frame_size_in_bytes), core_spill_mask_(core_spill_mask), fp_spill_mask_(fp_spill_mask), - src_mapping_table_(driver->DeduplicateSrcMappingTable(src_mapping_table->Arrange())), - mapping_table_(driver->DeduplicateMappingTable(mapping_table)), + src_mapping_table_(src_mapping_table == nullptr ? + driver->DeduplicateSrcMappingTable(ArrayRef<SrcMapElem>()) : + driver->DeduplicateSrcMappingTable(ArrayRef<SrcMapElem>(src_mapping_table->Arrange()))), + mapping_table_(mapping_table.data() == nullptr ? + nullptr : driver->DeduplicateMappingTable(mapping_table)), vmap_table_(driver->DeduplicateVMapTable(vmap_table)), - gc_map_(driver->DeduplicateGCMap(native_gc_map)), - cfi_info_(driver->DeduplicateCFIInfo(cfi_info)), - patches_(patches.begin(), patches.end()) { + gc_map_(native_gc_map.data() == nullptr ? nullptr : driver->DeduplicateGCMap(native_gc_map)), + cfi_info_(cfi_info.data() == nullptr ? nullptr : driver->DeduplicateCFIInfo(cfi_info)), + patches_(patches.begin(), patches.end(), driver->GetSwapSpaceAllocator()) { } -CompiledMethod::CompiledMethod(CompilerDriver* driver, - InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code, - const size_t frame_size_in_bytes, - const uint32_t core_spill_mask, - const uint32_t fp_spill_mask, - const std::vector<uint8_t>& stack_map) - : CompiledCode(driver, instruction_set, quick_code), - frame_size_in_bytes_(frame_size_in_bytes), - core_spill_mask_(core_spill_mask), - fp_spill_mask_(fp_spill_mask), - src_mapping_table_(driver->DeduplicateSrcMappingTable(SrcMap())), - mapping_table_(nullptr), - vmap_table_(driver->DeduplicateVMapTable(stack_map)), - gc_map_(nullptr), - cfi_info_(nullptr), - patches_() { +CompiledMethod* CompiledMethod::SwapAllocCompiledMethod( + CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + DefaultSrcMap* src_mapping_table, + const ArrayRef<const uint8_t>& mapping_table, + const ArrayRef<const uint8_t>& vmap_table, + const ArrayRef<const uint8_t>& native_gc_map, + const ArrayRef<const uint8_t>& cfi_info, + const ArrayRef<LinkerPatch>& patches) { + SwapAllocator<CompiledMethod> alloc(driver->GetSwapSpaceAllocator()); + CompiledMethod* ret = alloc.allocate(1); + alloc.construct(ret, driver, instruction_set, quick_code, frame_size_in_bytes, core_spill_mask, + fp_spill_mask, src_mapping_table, mapping_table, vmap_table, native_gc_map, + cfi_info, patches); + return ret; } -CompiledMethod::CompiledMethod(CompilerDriver* driver, - InstructionSet instruction_set, - const std::vector<uint8_t>& code, - const size_t frame_size_in_bytes, - const uint32_t core_spill_mask, - const uint32_t fp_spill_mask, - const std::vector<uint8_t>* cfi_info) - : CompiledCode(driver, instruction_set, code), - frame_size_in_bytes_(frame_size_in_bytes), - core_spill_mask_(core_spill_mask), fp_spill_mask_(fp_spill_mask), - src_mapping_table_(driver->DeduplicateSrcMappingTable(SrcMap())), - mapping_table_(driver->DeduplicateMappingTable(std::vector<uint8_t>())), - vmap_table_(driver->DeduplicateVMapTable(std::vector<uint8_t>())), - gc_map_(driver->DeduplicateGCMap(std::vector<uint8_t>())), - cfi_info_(driver->DeduplicateCFIInfo(cfi_info)), - patches_() { +CompiledMethod* CompiledMethod::SwapAllocCompiledMethodStackMap( + CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + const ArrayRef<const uint8_t>& stack_map) { + SwapAllocator<CompiledMethod> alloc(driver->GetSwapSpaceAllocator()); + CompiledMethod* ret = alloc.allocate(1); + alloc.construct(ret, driver, instruction_set, quick_code, frame_size_in_bytes, core_spill_mask, + fp_spill_mask, nullptr, ArrayRef<const uint8_t>(), stack_map, + ArrayRef<const uint8_t>(), ArrayRef<const uint8_t>(), ArrayRef<LinkerPatch>()); + return ret; +} + +CompiledMethod* CompiledMethod::SwapAllocCompiledMethodCFI( + CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + const ArrayRef<const uint8_t>& cfi_info) { + SwapAllocator<CompiledMethod> alloc(driver->GetSwapSpaceAllocator()); + CompiledMethod* ret = alloc.allocate(1); + alloc.construct(ret, driver, instruction_set, quick_code, frame_size_in_bytes, core_spill_mask, + fp_spill_mask, nullptr, ArrayRef<const uint8_t>(), + ArrayRef<const uint8_t>(), ArrayRef<const uint8_t>(), + cfi_info, ArrayRef<LinkerPatch>()); + return ret; +} + + +void CompiledMethod::ReleaseSwapAllocatedCompiledMethod(CompilerDriver* driver, CompiledMethod* m) { + SwapAllocator<CompiledMethod> alloc(driver->GetSwapSpaceAllocator()); + alloc.destroy(m); + alloc.deallocate(m, 1); } } // namespace art diff --git a/compiler/compiled_method.h b/compiler/compiled_method.h index d93db03806..6013507ac4 100644 --- a/compiler/compiled_method.h +++ b/compiler/compiled_method.h @@ -25,6 +25,7 @@ #include "method_reference.h" #include "utils.h" #include "utils/array_ref.h" +#include "utils/swap_space.h" namespace llvm { class Function; @@ -38,17 +39,17 @@ class CompiledCode { public: // For Quick to supply an code blob CompiledCode(CompilerDriver* compiler_driver, InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code); + const ArrayRef<const uint8_t>& quick_code); InstructionSet GetInstructionSet() const { return instruction_set_; } - const std::vector<uint8_t>* GetQuickCode() const { + const SwapVector<uint8_t>* GetQuickCode() const { return quick_code_; } - void SetCode(const std::vector<uint8_t>* quick_code); + void SetCode(const ArrayRef<const uint8_t>* quick_code); bool operator==(const CompiledCode& rhs) const; @@ -78,7 +79,7 @@ class CompiledCode { const InstructionSet instruction_set_; // Used to store the PIC code for Quick. - std::vector<uint8_t>* quick_code_; + SwapVector<uint8_t>* quick_code_; // There are offsets from the oatdata symbol to where the offset to // the compiled method will be found. These are computed by the @@ -109,8 +110,23 @@ class SrcMapElem { } }; -class SrcMap FINAL : public std::vector<SrcMapElem> { +template <class Allocator> +class SrcMap FINAL : public std::vector<SrcMapElem, Allocator> { public: + using std::vector<SrcMapElem, Allocator>::begin; + using typename std::vector<SrcMapElem, Allocator>::const_iterator; + using std::vector<SrcMapElem, Allocator>::empty; + using std::vector<SrcMapElem, Allocator>::end; + using std::vector<SrcMapElem, Allocator>::resize; + using std::vector<SrcMapElem, Allocator>::shrink_to_fit; + using std::vector<SrcMapElem, Allocator>::size; + + explicit SrcMap() {} + + template <class InputIt> + SrcMap(InputIt first, InputIt last, const Allocator& alloc) + : std::vector<SrcMapElem, Allocator>(first, last, alloc) {} + void SortByFrom() { std::sort(begin(), end(), [] (const SrcMapElem& lhs, const SrcMapElem& rhs) -> bool { return lhs.from_ < rhs.from_; @@ -158,6 +174,10 @@ class SrcMap FINAL : public std::vector<SrcMapElem> { } }; +using DefaultSrcMap = SrcMap<std::allocator<SrcMapElem>>; +using SwapSrcMap = SrcMap<SwapAllocator<SrcMapElem>>; + + enum LinkerPatchType { kLinkerPatchMethod, kLinkerPatchCall, @@ -255,40 +275,57 @@ inline bool operator<(const LinkerPatch& lhs, const LinkerPatch& rhs) { class CompiledMethod FINAL : public CompiledCode { public: - // Constructs a CompiledMethod for Quick. + // Constructs a CompiledMethod. + // Note: Consider using the static allocation methods below that will allocate the CompiledMethod + // in the swap space. CompiledMethod(CompilerDriver* driver, InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code, + const ArrayRef<const uint8_t>& quick_code, const size_t frame_size_in_bytes, const uint32_t core_spill_mask, const uint32_t fp_spill_mask, - SrcMap* src_mapping_table, - const std::vector<uint8_t>& mapping_table, - const std::vector<uint8_t>& vmap_table, - const std::vector<uint8_t>& native_gc_map, - const std::vector<uint8_t>* cfi_info, + DefaultSrcMap* src_mapping_table, + const ArrayRef<const uint8_t>& mapping_table, + const ArrayRef<const uint8_t>& vmap_table, + const ArrayRef<const uint8_t>& native_gc_map, + const ArrayRef<const uint8_t>& cfi_info, const ArrayRef<LinkerPatch>& patches = ArrayRef<LinkerPatch>()); - // Constructs a CompiledMethod for Optimizing. - CompiledMethod(CompilerDriver* driver, - InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code, - const size_t frame_size_in_bytes, - const uint32_t core_spill_mask, - const uint32_t fp_spill_mask, - const std::vector<uint8_t>& vmap_table); - - // Constructs a CompiledMethod for the QuickJniCompiler. - CompiledMethod(CompilerDriver* driver, - InstructionSet instruction_set, - const std::vector<uint8_t>& quick_code, - const size_t frame_size_in_bytes, - const uint32_t core_spill_mask, - const uint32_t fp_spill_mask, - const std::vector<uint8_t>* cfi_info); - ~CompiledMethod() {} + static CompiledMethod* SwapAllocCompiledMethod( + CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + DefaultSrcMap* src_mapping_table, + const ArrayRef<const uint8_t>& mapping_table, + const ArrayRef<const uint8_t>& vmap_table, + const ArrayRef<const uint8_t>& native_gc_map, + const ArrayRef<const uint8_t>& cfi_info, + const ArrayRef<LinkerPatch>& patches = ArrayRef<LinkerPatch>()); + + static CompiledMethod* SwapAllocCompiledMethodStackMap( + CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + const ArrayRef<const uint8_t>& stack_map); + + static CompiledMethod* SwapAllocCompiledMethodCFI(CompilerDriver* driver, + InstructionSet instruction_set, + const ArrayRef<const uint8_t>& quick_code, + const size_t frame_size_in_bytes, + const uint32_t core_spill_mask, + const uint32_t fp_spill_mask, + const ArrayRef<const uint8_t>& cfi_info); + + static void ReleaseSwapAllocatedCompiledMethod(CompilerDriver* driver, CompiledMethod* m); + size_t GetFrameSizeInBytes() const { return frame_size_in_bytes_; } @@ -301,29 +338,29 @@ class CompiledMethod FINAL : public CompiledCode { return fp_spill_mask_; } - const SrcMap& GetSrcMappingTable() const { + const SwapSrcMap& GetSrcMappingTable() const { DCHECK(src_mapping_table_ != nullptr); return *src_mapping_table_; } - std::vector<uint8_t> const* GetMappingTable() const { + SwapVector<uint8_t> const* GetMappingTable() const { return mapping_table_; } - const std::vector<uint8_t>& GetVmapTable() const { + const SwapVector<uint8_t>& GetVmapTable() const { DCHECK(vmap_table_ != nullptr); return *vmap_table_; } - std::vector<uint8_t> const* GetGcMap() const { + SwapVector<uint8_t> const* GetGcMap() const { return gc_map_; } - const std::vector<uint8_t>* GetCFIInfo() const { + const SwapVector<uint8_t>* GetCFIInfo() const { return cfi_info_; } - const std::vector<LinkerPatch>& GetPatches() const { + const SwapVector<LinkerPatch>& GetPatches() const { return patches_; } @@ -335,19 +372,19 @@ class CompiledMethod FINAL : public CompiledCode { // For quick code, a bit mask describing spilled FPR callee-save registers. const uint32_t fp_spill_mask_; // For quick code, a set of pairs (PC, Line) mapping from native PC offset to Java line - SrcMap* src_mapping_table_; + SwapSrcMap* src_mapping_table_; // For quick code, a uleb128 encoded map from native PC offset to dex PC aswell as dex PC to // native PC offset. Size prefixed. - std::vector<uint8_t>* mapping_table_; + SwapVector<uint8_t>* mapping_table_; // For quick code, a uleb128 encoded map from GPR/FPR register to dex register. Size prefixed. - std::vector<uint8_t>* vmap_table_; + SwapVector<uint8_t>* vmap_table_; // For quick code, a map keyed by native PC indices to bitmaps describing what dalvik registers // are live. - std::vector<uint8_t>* gc_map_; + SwapVector<uint8_t>* gc_map_; // For quick code, a FDE entry for the debug_frame section. - std::vector<uint8_t>* cfi_info_; + SwapVector<uint8_t>* cfi_info_; // For quick code, linker patches needed by the method. - std::vector<LinkerPatch> patches_; + SwapVector<LinkerPatch> patches_; }; } // namespace art diff --git a/compiler/dex/quick/codegen_util.cc b/compiler/dex/quick/codegen_util.cc index cc61e93d82..67ea8972b7 100644 --- a/compiler/dex/quick/codegen_util.cc +++ b/compiler/dex/quick/codegen_util.cc @@ -1086,12 +1086,20 @@ CompiledMethod* Mir2Lir::GetCompiledMethod() { }); std::unique_ptr<std::vector<uint8_t>> cfi_info(ReturnFrameDescriptionEntry()); - CompiledMethod* result = - new CompiledMethod(cu_->compiler_driver, cu_->instruction_set, code_buffer_, frame_size_, - core_spill_mask_, fp_spill_mask_, &src_mapping_table_, encoded_mapping_table_, - vmap_encoder.GetData(), native_gc_map_, cfi_info.get(), - ArrayRef<LinkerPatch>(patches_)); - return result; + ArrayRef<const uint8_t> cfi_ref; + if (cfi_info.get() != nullptr) { + cfi_ref = ArrayRef<const uint8_t>(*cfi_info); + } + return CompiledMethod::SwapAllocCompiledMethod( + cu_->compiler_driver, cu_->instruction_set, + ArrayRef<const uint8_t>(code_buffer_), + frame_size_, core_spill_mask_, fp_spill_mask_, + &src_mapping_table_, + ArrayRef<const uint8_t>(encoded_mapping_table_), + ArrayRef<const uint8_t>(vmap_encoder.GetData()), + ArrayRef<const uint8_t>(native_gc_map_), + cfi_ref, + ArrayRef<LinkerPatch>(patches_)); } size_t Mir2Lir::GetMaxPossibleCompilerTemps() const { diff --git a/compiler/dex/quick/mir_to_lir.h b/compiler/dex/quick/mir_to_lir.h index a2b85ffb6c..19e7bf3e43 100644 --- a/compiler/dex/quick/mir_to_lir.h +++ b/compiler/dex/quick/mir_to_lir.h @@ -1741,7 +1741,7 @@ class Mir2Lir : public Backend { int live_sreg_; CodeBuffer code_buffer_; // The source mapping table data (pc -> dex). More entries than in encoded_mapping_table_ - SrcMap src_mapping_table_; + DefaultSrcMap src_mapping_table_; // The encoding mapping table data (dex -> pc offset and pc offset -> dex) with a size prefix. std::vector<uint8_t> encoded_mapping_table_; ArenaVector<uint32_t> core_vmap_table_; diff --git a/compiler/driver/compiler_driver.cc b/compiler/driver/compiler_driver.cc index cbb23c26b9..9985d66469 100644 --- a/compiler/driver/compiler_driver.cc +++ b/compiler/driver/compiler_driver.cc @@ -62,6 +62,7 @@ #include "thread_pool.h" #include "trampolines/trampoline_compiler.h" #include "transaction.h" +#include "utils/swap_space.h" #include "verifier/method_verifier.h" #include "verifier/method_verifier-inl.h" @@ -339,8 +340,10 @@ CompilerDriver::CompilerDriver(const CompilerOptions* compiler_options, bool image, std::set<std::string>* image_classes, std::set<std::string>* compiled_classes, size_t thread_count, bool dump_stats, bool dump_passes, CumulativeLogger* timer, - const std::string& profile_file) - : profile_present_(false), compiler_options_(compiler_options), + int swap_fd, const std::string& profile_file) + : swap_space_(swap_fd == -1 ? nullptr : new SwapSpace(swap_fd, 10 * MB)), + swap_space_allocator_(new SwapAllocator<void>(swap_space_.get())), + profile_present_(false), compiler_options_(compiler_options), verification_results_(verification_results), method_inliner_map_(method_inliner_map), compiler_(Compiler::Create(this, compiler_kind)), @@ -349,7 +352,7 @@ CompilerDriver::CompilerDriver(const CompilerOptions* compiler_options, freezing_constructor_lock_("freezing constructor lock"), compiled_classes_lock_("compiled classes lock"), compiled_methods_lock_("compiled method lock"), - compiled_methods_(), + compiled_methods_(MethodTable::key_compare()), non_relative_linker_patch_count_(0u), image_(image), image_classes_(image_classes), @@ -361,12 +364,12 @@ CompilerDriver::CompilerDriver(const CompilerOptions* compiler_options, timings_logger_(timer), compiler_context_(nullptr), support_boot_image_fixup_(instruction_set != kMips), - dedupe_code_("dedupe code"), - dedupe_src_mapping_table_("dedupe source mapping table"), - dedupe_mapping_table_("dedupe mapping table"), - dedupe_vmap_table_("dedupe vmap table"), - dedupe_gc_map_("dedupe gc map"), - dedupe_cfi_info_("dedupe cfi info") { + dedupe_code_("dedupe code", *swap_space_allocator_), + dedupe_src_mapping_table_("dedupe source mapping table", *swap_space_allocator_), + dedupe_mapping_table_("dedupe mapping table", *swap_space_allocator_), + dedupe_vmap_table_("dedupe vmap table", *swap_space_allocator_), + dedupe_gc_map_("dedupe gc map", *swap_space_allocator_), + dedupe_cfi_info_("dedupe cfi info", *swap_space_allocator_) { DCHECK(compiler_options_ != nullptr); DCHECK(verification_results_ != nullptr); DCHECK(method_inliner_map_ != nullptr); @@ -393,31 +396,28 @@ CompilerDriver::CompilerDriver(const CompilerOptions* compiler_options, } } -std::vector<uint8_t>* CompilerDriver::DeduplicateCode(const std::vector<uint8_t>& code) { +SwapVector<uint8_t>* CompilerDriver::DeduplicateCode(const ArrayRef<const uint8_t>& code) { return dedupe_code_.Add(Thread::Current(), code); } -SrcMap* CompilerDriver::DeduplicateSrcMappingTable(const SrcMap& src_map) { +SwapSrcMap* CompilerDriver::DeduplicateSrcMappingTable(const ArrayRef<SrcMapElem>& src_map) { return dedupe_src_mapping_table_.Add(Thread::Current(), src_map); } -std::vector<uint8_t>* CompilerDriver::DeduplicateMappingTable(const std::vector<uint8_t>& code) { +SwapVector<uint8_t>* CompilerDriver::DeduplicateMappingTable(const ArrayRef<const uint8_t>& code) { return dedupe_mapping_table_.Add(Thread::Current(), code); } -std::vector<uint8_t>* CompilerDriver::DeduplicateVMapTable(const std::vector<uint8_t>& code) { +SwapVector<uint8_t>* CompilerDriver::DeduplicateVMapTable(const ArrayRef<const uint8_t>& code) { return dedupe_vmap_table_.Add(Thread::Current(), code); } -std::vector<uint8_t>* CompilerDriver::DeduplicateGCMap(const std::vector<uint8_t>& code) { +SwapVector<uint8_t>* CompilerDriver::DeduplicateGCMap(const ArrayRef<const uint8_t>& code) { return dedupe_gc_map_.Add(Thread::Current(), code); } -std::vector<uint8_t>* CompilerDriver::DeduplicateCFIInfo(const std::vector<uint8_t>* cfi_info) { - if (cfi_info == nullptr) { - return nullptr; - } - return dedupe_cfi_info_.Add(Thread::Current(), *cfi_info); +SwapVector<uint8_t>* CompilerDriver::DeduplicateCFIInfo(const ArrayRef<const uint8_t>& cfi_info) { + return dedupe_cfi_info_.Add(Thread::Current(), cfi_info); } CompilerDriver::~CompilerDriver() { @@ -428,7 +428,9 @@ CompilerDriver::~CompilerDriver() { } { MutexLock mu(self, compiled_methods_lock_); - STLDeleteValues(&compiled_methods_); + for (auto& pair : compiled_methods_) { + CompiledMethod::ReleaseSwapAllocatedCompiledMethod(this, pair.second); + } } compiler_->UnInit(); } @@ -2337,6 +2339,14 @@ std::string CompilerDriver::GetMemoryUsageString() const { oss << " native alloc=" << PrettySize(allocated_space) << " free=" << PrettySize(free_space); #endif + if (swap_space_.get() != nullptr) { + oss << " swap=" << PrettySize(swap_space_->GetSize()); + } + oss << "\nCode dedupe: " << dedupe_code_.DumpStats(); + oss << "\nMapping table dedupe: " << dedupe_mapping_table_.DumpStats(); + oss << "\nVmap table dedupe: " << dedupe_vmap_table_.DumpStats(); + oss << "\nGC map dedupe: " << dedupe_gc_map_.DumpStats(); + oss << "\nCFI info dedupe: " << dedupe_cfi_info_.DumpStats(); return oss.str(); } diff --git a/compiler/driver/compiler_driver.h b/compiler/driver/compiler_driver.h index edc6468a85..7ddc32cdd8 100644 --- a/compiler/driver/compiler_driver.h +++ b/compiler/driver/compiler_driver.h @@ -39,6 +39,8 @@ #include "thread_pool.h" #include "utils/arena_allocator.h" #include "utils/dedupe_set.h" +#include "utils/swap_space.h" +#include "utils.h" #include "dex/verified_method.h" namespace art { @@ -77,6 +79,8 @@ enum DexToDexCompilationLevel { }; std::ostream& operator<<(std::ostream& os, const DexToDexCompilationLevel& rhs); +static constexpr bool kUseMurmur3Hash = true; + class CompilerDriver { public: // Create a compiler targeting the requested "instruction_set". @@ -93,7 +97,8 @@ class CompilerDriver { bool image, std::set<std::string>* image_classes, std::set<std::string>* compiled_classes, size_t thread_count, bool dump_stats, bool dump_passes, - CumulativeLogger* timer, const std::string& profile_file); + CumulativeLogger* timer, int swap_fd, + const std::string& profile_file); ~CompilerDriver(); @@ -334,6 +339,9 @@ class CompilerDriver { const ArenaPool* GetArenaPool() const { return &arena_pool_; } + SwapAllocator<void>& GetSwapSpaceAllocator() { + return *swap_space_allocator_.get(); + } bool WriteElf(const std::string& android_root, bool is_host, @@ -376,15 +384,12 @@ class CompilerDriver { void RecordClassStatus(ClassReference ref, mirror::Class::Status status) LOCKS_EXCLUDED(compiled_classes_lock_); - std::vector<uint8_t>* DeduplicateCode(const std::vector<uint8_t>& code); - SrcMap* DeduplicateSrcMappingTable(const SrcMap& src_map); - std::vector<uint8_t>* DeduplicateMappingTable(const std::vector<uint8_t>& code); - std::vector<uint8_t>* DeduplicateVMapTable(const std::vector<uint8_t>& code); - std::vector<uint8_t>* DeduplicateGCMap(const std::vector<uint8_t>& code); - std::vector<uint8_t>* DeduplicateCFIInfo(const std::vector<uint8_t>* cfi_info); - - ProfileFile profile_file_; - bool profile_present_; + SwapVector<uint8_t>* DeduplicateCode(const ArrayRef<const uint8_t>& code); + SwapSrcMap* DeduplicateSrcMappingTable(const ArrayRef<SrcMapElem>& src_map); + SwapVector<uint8_t>* DeduplicateMappingTable(const ArrayRef<const uint8_t>& code); + SwapVector<uint8_t>* DeduplicateVMapTable(const ArrayRef<const uint8_t>& code); + SwapVector<uint8_t>* DeduplicateGCMap(const ArrayRef<const uint8_t>& code); + SwapVector<uint8_t>* DeduplicateCFIInfo(const ArrayRef<const uint8_t>& cfi_info); // Should the compiler run on this method given profile information? bool SkipCompilation(const std::string& method_name); @@ -484,6 +489,14 @@ class CompilerDriver { static void CompileClass(const ParallelCompilationManager* context, size_t class_def_index) LOCKS_EXCLUDED(Locks::mutator_lock_); + // Swap pool and allocator used for native allocations. May be file-backed. Needs to be first + // as other fields rely on this. + std::unique_ptr<SwapSpace> swap_space_; + std::unique_ptr<SwapAllocator<void> > swap_space_allocator_; + + ProfileFile profile_file_; + bool profile_present_; + const CompilerOptions* const compiler_options_; VerificationResults* const verification_results_; DexFileToMethodInlinerMap* const method_inliner_map_; @@ -551,47 +564,92 @@ class CompilerDriver { bool support_boot_image_fixup_; // DeDuplication data structures, these own the corresponding byte arrays. - template <typename ByteArray> + template <typename ContentType> class DedupeHashFunc { public: - size_t operator()(const ByteArray& array) const { - // For small arrays compute a hash using every byte. - static const size_t kSmallArrayThreshold = 16; - size_t hash = 0x811c9dc5; - if (array.size() <= kSmallArrayThreshold) { - for (auto b : array) { - hash = (hash * 16777619) ^ static_cast<uint8_t>(b); + size_t operator()(const ArrayRef<ContentType>& array) const { + const uint8_t* data = reinterpret_cast<const uint8_t*>(array.data()); + static_assert(IsPowerOfTwo(sizeof(ContentType)), + "ContentType is not power of two, don't know whether array layout is as assumed"); + uint32_t len = sizeof(ContentType) * array.size(); + if (kUseMurmur3Hash) { + static constexpr uint32_t c1 = 0xcc9e2d51; + static constexpr uint32_t c2 = 0x1b873593; + static constexpr uint32_t r1 = 15; + static constexpr uint32_t r2 = 13; + static constexpr uint32_t m = 5; + static constexpr uint32_t n = 0xe6546b64; + + uint32_t hash = 0; + + const int nblocks = len / 4; + typedef __attribute__((__aligned__(1))) uint32_t unaligned_uint32_t; + const unaligned_uint32_t *blocks = reinterpret_cast<const uint32_t*>(data); + int i; + for (i = 0; i < nblocks; i++) { + uint32_t k = blocks[i]; + k *= c1; + k = (k << r1) | (k >> (32 - r1)); + k *= c2; + + hash ^= k; + hash = ((hash << r2) | (hash >> (32 - r2))) * m + n; } - } else { - // For larger arrays use the 2 bytes at 6 bytes (the location of a push registers - // instruction field for quick generated code on ARM) and then select a number of other - // values at random. - static const size_t kRandomHashCount = 16; - for (size_t i = 0; i < 2; ++i) { - uint8_t b = static_cast<uint8_t>(array[i + 6]); - hash = (hash * 16777619) ^ b; + + const uint8_t *tail = reinterpret_cast<const uint8_t*>(data + nblocks * 4); + uint32_t k1 = 0; + + switch (len & 3) { + case 3: + k1 ^= tail[2] << 16; + FALLTHROUGH_INTENDED; + case 2: + k1 ^= tail[1] << 8; + FALLTHROUGH_INTENDED; + case 1: + k1 ^= tail[0]; + + k1 *= c1; + k1 = (k1 << r1) | (k1 >> (32 - r1)); + k1 *= c2; + hash ^= k1; } - for (size_t i = 2; i < kRandomHashCount; ++i) { - size_t r = i * 1103515245 + 12345; - uint8_t b = static_cast<uint8_t>(array[r % array.size()]); - hash = (hash * 16777619) ^ b; + + hash ^= len; + hash ^= (hash >> 16); + hash *= 0x85ebca6b; + hash ^= (hash >> 13); + hash *= 0xc2b2ae35; + hash ^= (hash >> 16); + + return hash; + } else { + size_t hash = 0x811c9dc5; + for (uint32_t i = 0; i < len; ++i) { + hash = (hash * 16777619) ^ data[i]; } + hash += hash << 13; + hash ^= hash >> 7; + hash += hash << 3; + hash ^= hash >> 17; + hash += hash << 5; + return hash; } - hash += hash << 13; - hash ^= hash >> 7; - hash += hash << 3; - hash ^= hash >> 17; - hash += hash << 5; - return hash; } }; - DedupeSet<std::vector<uint8_t>, size_t, DedupeHashFunc<std::vector<uint8_t>>, 4> dedupe_code_; - DedupeSet<SrcMap, size_t, DedupeHashFunc<SrcMap>, 4> dedupe_src_mapping_table_; - DedupeSet<std::vector<uint8_t>, size_t, DedupeHashFunc<std::vector<uint8_t>>, 4> dedupe_mapping_table_; - DedupeSet<std::vector<uint8_t>, size_t, DedupeHashFunc<std::vector<uint8_t>>, 4> dedupe_vmap_table_; - DedupeSet<std::vector<uint8_t>, size_t, DedupeHashFunc<std::vector<uint8_t>>, 4> dedupe_gc_map_; - DedupeSet<std::vector<uint8_t>, size_t, DedupeHashFunc<std::vector<uint8_t>>, 4> dedupe_cfi_info_; + DedupeSet<ArrayRef<const uint8_t>, + SwapVector<uint8_t>, size_t, DedupeHashFunc<const uint8_t>, 4> dedupe_code_; + DedupeSet<ArrayRef<SrcMapElem>, + SwapSrcMap, size_t, DedupeHashFunc<SrcMapElem>, 4> dedupe_src_mapping_table_; + DedupeSet<ArrayRef<const uint8_t>, + SwapVector<uint8_t>, size_t, DedupeHashFunc<const uint8_t>, 4> dedupe_mapping_table_; + DedupeSet<ArrayRef<const uint8_t>, + SwapVector<uint8_t>, size_t, DedupeHashFunc<const uint8_t>, 4> dedupe_vmap_table_; + DedupeSet<ArrayRef<const uint8_t>, + SwapVector<uint8_t>, size_t, DedupeHashFunc<const uint8_t>, 4> dedupe_gc_map_; + DedupeSet<ArrayRef<const uint8_t>, + SwapVector<uint8_t>, size_t, DedupeHashFunc<const uint8_t>, 4> dedupe_cfi_info_; DISALLOW_COPY_AND_ASSIGN(CompilerDriver); }; diff --git a/compiler/elf_writer_quick.cc b/compiler/elf_writer_quick.cc index d651c0fb84..9ec4f281cb 100644 --- a/compiler/elf_writer_quick.cc +++ b/compiler/elf_writer_quick.cc @@ -358,8 +358,8 @@ class LineTableGenerator FINAL : public Leb128Encoder { }; // TODO: rewriting it using DexFile::DecodeDebugInfo needs unneeded stuff. -static void GetLineInfoForJava(const uint8_t* dbgstream, const SrcMap& pc2dex, - SrcMap* result, uint32_t start_pc = 0) { +static void GetLineInfoForJava(const uint8_t* dbgstream, const SwapSrcMap& pc2dex, + DefaultSrcMap* result, uint32_t start_pc = 0) { if (dbgstream == nullptr) { return; } @@ -415,7 +415,7 @@ static void GetLineInfoForJava(const uint8_t* dbgstream, const SrcMap& pc2dex, dex_offset += adjopcode / DexFile::DBG_LINE_RANGE; java_line += DexFile::DBG_LINE_BASE + (adjopcode % DexFile::DBG_LINE_RANGE); - for (SrcMap::const_iterator found = pc2dex.FindByTo(dex_offset); + for (SwapSrcMap::const_iterator found = pc2dex.FindByTo(dex_offset); found != pc2dex.end() && found->to_ == static_cast<int32_t>(dex_offset); found++) { result->push_back({found->from_ + start_pc, static_cast<int32_t>(java_line)}); @@ -615,7 +615,7 @@ static void FillInCFIInformation(OatWriter* oat_writer, LineTableGenerator line_table_generator(LINE_BASE, LINE_RANGE, OPCODE_BASE, dbg_line, 0, 1); - SrcMap pc2java_map; + DefaultSrcMap pc2java_map; for (size_t i = 0; i < method_info.size(); ++i) { const OatWriter::DebugInfo &dbg = method_info[i]; const char* file_name = (dbg.src_file_name_ == nullptr) ? "null" : dbg.src_file_name_; @@ -700,7 +700,7 @@ static void WriteDebugSymbols(const CompilerDriver* compiler_driver, DCHECK(it->compiled_method_ != nullptr); // Copy in the FDE, if present - const std::vector<uint8_t>* fde = it->compiled_method_->GetCFIInfo(); + const SwapVector<uint8_t>* fde = it->compiled_method_->GetCFIInfo(); if (fde != nullptr) { // Copy the information into cfi_info and then fix the address in the new copy. int cur_offset = cfi_info->size(); diff --git a/compiler/jni/quick/jni_compiler.cc b/compiler/jni/quick/jni_compiler.cc index c3fe75b3f1..bf996a251f 100644 --- a/compiler/jni/quick/jni_compiler.cc +++ b/compiler/jni/quick/jni_compiler.cc @@ -430,13 +430,18 @@ CompiledMethod* ArtJniCompileMethodInternal(CompilerDriver* driver, MemoryRegion code(&managed_code[0], managed_code.size()); __ FinalizeInstructions(code); jni_asm->FinalizeFrameDescriptionEntry(); - return new CompiledMethod(driver, - instruction_set, - managed_code, - frame_size, - main_jni_conv->CoreSpillMask(), - main_jni_conv->FpSpillMask(), - jni_asm->GetFrameDescriptionEntry()); + std::vector<uint8_t>* fde(jni_asm->GetFrameDescriptionEntry()); + ArrayRef<const uint8_t> cfi_ref; + if (fde != nullptr) { + cfi_ref = ArrayRef<const uint8_t>(*fde); + } + return CompiledMethod::SwapAllocCompiledMethodCFI(driver, + instruction_set, + ArrayRef<const uint8_t>(managed_code), + frame_size, + main_jni_conv->CoreSpillMask(), + main_jni_conv->FpSpillMask(), + cfi_ref); } // Copy a single parameter from the managed to the JNI calling convention diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 3ca0cdf011..d14153872b 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -59,7 +59,7 @@ class OatTest : public CommonCompilerTest { EXPECT_EQ(oat_method.GetFpSpillMask(), compiled_method->GetFpSpillMask()); uintptr_t oat_code_aligned = RoundDown(reinterpret_cast<uintptr_t>(quick_oat_code), 2); quick_oat_code = reinterpret_cast<const void*>(oat_code_aligned); - const std::vector<uint8_t>* quick_code = compiled_method->GetQuickCode(); + const SwapVector<uint8_t>* quick_code = compiled_method->GetQuickCode(); EXPECT_TRUE(quick_code != nullptr); size_t code_size = quick_code->size() * sizeof(quick_code[0]); EXPECT_EQ(0, memcmp(quick_oat_code, &quick_code[0], code_size)) @@ -92,7 +92,7 @@ TEST_F(OatTest, WriteRead) { method_inliner_map_.get(), compiler_kind, insn_set, insn_features.get(), false, nullptr, nullptr, 2, true, - true, timer_.get(), "")); + true, timer_.get(), -1, "")); jobject class_loader = nullptr; if (kCompile) { TimingLogger timings2("OatTest::WriteRead", false, false); diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index 7d14de1306..3c36ffa4e9 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -504,7 +504,7 @@ OatWriter::~OatWriter() { } struct OatWriter::GcMapDataAccess { - static const std::vector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { + static const SwapVector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { return compiled_method->GetGcMap(); } @@ -526,7 +526,7 @@ struct OatWriter::GcMapDataAccess { }; struct OatWriter::MappingTableDataAccess { - static const std::vector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { + static const SwapVector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { return compiled_method->GetMappingTable(); } @@ -548,7 +548,7 @@ struct OatWriter::MappingTableDataAccess { }; struct OatWriter::VmapTableDataAccess { - static const std::vector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { + static const SwapVector<uint8_t>* GetData(const CompiledMethod* compiled_method) ALWAYS_INLINE { return &compiled_method->GetVmapTable(); } @@ -719,7 +719,7 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { // Derived from CompiledMethod. uint32_t quick_code_offset = 0; - const std::vector<uint8_t>* quick_code = compiled_method->GetQuickCode(); + const SwapVector<uint8_t>* quick_code = compiled_method->GetQuickCode(); CHECK(quick_code != nullptr); offset_ = writer_->relative_call_patcher_->ReserveSpace(offset_, compiled_method); offset_ = compiled_method->AlignCode(offset_); @@ -829,7 +829,7 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { } else { status = mirror::Class::kStatusNotReady; } - std::vector<uint8_t> const * gc_map = compiled_method->GetGcMap(); + const SwapVector<uint8_t>* gc_map = compiled_method->GetGcMap(); if (gc_map != nullptr) { size_t gc_map_size = gc_map->size() * sizeof(gc_map[0]); bool is_native = it.MemberIsNative(); @@ -871,7 +871,7 @@ class OatWriter::InitMapMethodVisitor : public OatDexMethodVisitor { DCHECK_LT(method_offsets_index_, oat_class->method_offsets_.size()); DCHECK_EQ(DataAccess::GetOffset(oat_class, method_offsets_index_), 0u); - const std::vector<uint8_t>* map = DataAccess::GetData(compiled_method); + const SwapVector<uint8_t>* map = DataAccess::GetData(compiled_method); uint32_t map_size = map == nullptr ? 0 : map->size() * sizeof((*map)[0]); if (map_size != 0u) { auto lb = dedupe_map_.lower_bound(map); @@ -893,7 +893,7 @@ class OatWriter::InitMapMethodVisitor : public OatDexMethodVisitor { private: // Deduplication is already done on a pointer basis by the compiler driver, // so we can simply compare the pointers to find out if things are duplicated. - SafeMap<const std::vector<uint8_t>*, uint32_t> dedupe_map_; + SafeMap<const SwapVector<uint8_t>*, uint32_t> dedupe_map_; }; class OatWriter::InitImageMethodVisitor : public OatDexMethodVisitor { @@ -990,8 +990,11 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { size_t file_offset = file_offset_; OutputStream* out = out_; - const std::vector<uint8_t>* quick_code = compiled_method->GetQuickCode(); + const SwapVector<uint8_t>* quick_code = compiled_method->GetQuickCode(); if (quick_code != nullptr) { + // Need a wrapper if we create a copy for patching. + ArrayRef<const uint8_t> wrapped(*quick_code); + offset_ = writer_->relative_call_patcher_->WriteThunks(out, offset_); if (offset_ == 0u) { ReportWriteFailure("relative call thunk", it); @@ -1030,8 +1033,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { DCHECK_OFFSET_(); if (!compiled_method->GetPatches().empty()) { - patched_code_ = *quick_code; - quick_code = &patched_code_; + patched_code_ = std::vector<uint8_t>(quick_code->begin(), quick_code->end()); + wrapped = ArrayRef<const uint8_t>(patched_code_); for (const LinkerPatch& patch : compiled_method->GetPatches()) { if (patch.Type() == kLinkerPatchCallRelative) { // NOTE: Relative calls across oat files are not supported. @@ -1052,8 +1055,8 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } } - writer_->oat_header_->UpdateChecksum(&(*quick_code)[0], code_size); - if (!out->WriteFully(&(*quick_code)[0], code_size)) { + writer_->oat_header_->UpdateChecksum(wrapped.data(), code_size); + if (!out->WriteFully(wrapped.data(), code_size)) { ReportWriteFailure("method code", it); return false; } @@ -1170,7 +1173,7 @@ class OatWriter::WriteMapMethodVisitor : public OatDexMethodVisitor { ++method_offsets_index_; // Write deduplicated map. - const std::vector<uint8_t>* map = DataAccess::GetData(compiled_method); + const SwapVector<uint8_t>* map = DataAccess::GetData(compiled_method); size_t map_size = map == nullptr ? 0 : map->size() * sizeof((*map)[0]); DCHECK((map_size == 0u && map_offset == 0u) || (map_size != 0u && map_offset != 0u && map_offset <= offset_)) diff --git a/compiler/optimizing/code_generator.cc b/compiler/optimizing/code_generator.cc index 6f424ce11d..91426f347b 100644 --- a/compiler/optimizing/code_generator.cc +++ b/compiler/optimizing/code_generator.cc @@ -374,7 +374,7 @@ void CodeGenerator::BuildNativeGCMap( } } -void CodeGenerator::BuildMappingTable(std::vector<uint8_t>* data, SrcMap* src_map) const { +void CodeGenerator::BuildMappingTable(std::vector<uint8_t>* data, DefaultSrcMap* src_map) const { uint32_t pc2dex_data_size = 0u; uint32_t pc2dex_entries = pc_infos_.Size(); uint32_t pc2dex_offset = 0u; diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index 1d42c47d56..2e7eca2ead 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -44,7 +44,10 @@ class Assembler; class CodeGenerator; class DexCompilationUnit; class ParallelMoveResolver; +class SrcMapElem; +template <class Alloc> class SrcMap; +using DefaultSrcMap = SrcMap<std::allocator<SrcMapElem>>; class CodeAllocator { public: @@ -146,7 +149,7 @@ class CodeGenerator : public ArenaObject<kArenaAllocMisc> { void GenerateSlowPaths(); - void BuildMappingTable(std::vector<uint8_t>* vector, SrcMap* src_map) const; + void BuildMappingTable(std::vector<uint8_t>* vector, DefaultSrcMap* src_map) const; void BuildVMapTable(std::vector<uint8_t>* vector) const; void BuildNativeGCMap( std::vector<uint8_t>* vector, const DexCompilationUnit& dex_compilation_unit) const; diff --git a/compiler/optimizing/optimizing_compiler.cc b/compiler/optimizing/optimizing_compiler.cc index 94751f876c..87f2b90775 100644 --- a/compiler/optimizing/optimizing_compiler.cc +++ b/compiler/optimizing/optimizing_compiler.cc @@ -225,13 +225,13 @@ static void RunOptimizations(HGraph* graph, // The stack map we generate must be 4-byte aligned on ARM. Since existing // maps are generated alongside these stack maps, we must also align them. -static std::vector<uint8_t>& AlignVectorSize(std::vector<uint8_t>& vector) { +static ArrayRef<const uint8_t> AlignVectorSize(std::vector<uint8_t>& vector) { size_t size = vector.size(); size_t aligned_size = RoundUp(size, 4); for (; size < aligned_size; ++size) { vector.push_back(0); } - return vector; + return ArrayRef<const uint8_t>(vector); } CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, @@ -332,13 +332,14 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, codegen->BuildStackMaps(&stack_map); compilation_stats_.RecordStat(MethodCompilationStat::kCompiledOptimized); - return new CompiledMethod(GetCompilerDriver(), - instruction_set, - allocator.GetMemory(), - codegen->GetFrameSize(), - codegen->GetCoreSpillMask(), - 0, /* FPR spill mask, unused */ - stack_map); + return CompiledMethod::SwapAllocCompiledMethodStackMap( + GetCompilerDriver(), + instruction_set, + ArrayRef<const uint8_t>(allocator.GetMemory()), + codegen->GetFrameSize(), + codegen->GetCoreSpillMask(), + 0, /* FPR spill mask, unused */ + ArrayRef<const uint8_t>(stack_map)); } else if (shouldOptimize && RegisterAllocator::Supports(instruction_set)) { LOG(FATAL) << "Could not allocate registers in optimizing compiler"; UNREACHABLE(); @@ -356,7 +357,7 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, codegen->CompileBaseline(&allocator); std::vector<uint8_t> mapping_table; - SrcMap src_mapping_table; + DefaultSrcMap src_mapping_table; codegen->BuildMappingTable(&mapping_table, GetCompilerDriver()->GetCompilerOptions().GetIncludeDebugSymbols() ? &src_mapping_table : nullptr); @@ -366,17 +367,17 @@ CompiledMethod* OptimizingCompiler::Compile(const DexFile::CodeItem* code_item, codegen->BuildNativeGCMap(&gc_map, dex_compilation_unit); compilation_stats_.RecordStat(MethodCompilationStat::kCompiledBaseline); - return new CompiledMethod(GetCompilerDriver(), - instruction_set, - allocator.GetMemory(), - codegen->GetFrameSize(), - codegen->GetCoreSpillMask(), - 0, /* FPR spill mask, unused */ - &src_mapping_table, - AlignVectorSize(mapping_table), - AlignVectorSize(vmap_table), - AlignVectorSize(gc_map), - nullptr); + return CompiledMethod::SwapAllocCompiledMethod(GetCompilerDriver(), + instruction_set, + ArrayRef<const uint8_t>(allocator.GetMemory()), + codegen->GetFrameSize(), + codegen->GetCoreSpillMask(), + 0, /* FPR spill mask, unused */ + &src_mapping_table, + AlignVectorSize(mapping_table), + AlignVectorSize(vmap_table), + AlignVectorSize(gc_map), + ArrayRef<const uint8_t>()); } } diff --git a/compiler/utils/array_ref.h b/compiler/utils/array_ref.h index 1a7f2e8c02..b1b0ee5e53 100644 --- a/compiler/utils/array_ref.h +++ b/compiler/utils/array_ref.h @@ -84,7 +84,7 @@ class ArrayRef { template <typename U, typename Alloc> ArrayRef(const std::vector<U, Alloc>& v, - typename std::enable_if<std::is_same<T, const U>::value, tag>::tag + typename std::enable_if<std::is_same<T, const U>::value, tag>::type t ATTRIBUTE_UNUSED = tag()) : array_(v.data()), size_(v.size()) { } diff --git a/compiler/utils/assembler.h b/compiler/utils/assembler.h index 67711e312c..134dda4b2c 100644 --- a/compiler/utils/assembler.h +++ b/compiler/utils/assembler.h @@ -502,6 +502,8 @@ class Assembler { virtual void InitializeFrameDescriptionEntry() {} virtual void FinalizeFrameDescriptionEntry() {} + // Give a vector containing FDE data, or null if not used. Note: the assembler must take care + // of handling the lifecycle. virtual std::vector<uint8_t>* GetFrameDescriptionEntry() { return nullptr; } virtual ~Assembler() {} diff --git a/compiler/utils/dedupe_set.h b/compiler/utils/dedupe_set.h index 4c52174936..b062a2aa86 100644 --- a/compiler/utils/dedupe_set.h +++ b/compiler/utils/dedupe_set.h @@ -17,50 +17,89 @@ #ifndef ART_COMPILER_UTILS_DEDUPE_SET_H_ #define ART_COMPILER_UTILS_DEDUPE_SET_H_ +#include <algorithm> +#include <inttypes.h> +#include <memory> #include <set> #include <string> #include "base/mutex.h" #include "base/stl_util.h" #include "base/stringprintf.h" +#include "utils/swap_space.h" namespace art { // A set of Keys that support a HashFunc returning HashType. Used to find duplicates of Key in the // Add method. The data-structure is thread-safe through the use of internal locks, it also // supports the lock being sharded. -template <typename Key, typename HashType, typename HashFunc, HashType kShard = 1> +template <typename InKey, typename StoreKey, typename HashType, typename HashFunc, + HashType kShard = 1> class DedupeSet { - typedef std::pair<HashType, Key*> HashedKey; + typedef std::pair<HashType, const InKey*> HashedInKey; + struct HashedKey { + StoreKey* store_ptr; + union { + HashType store_hash; // Valid if store_ptr != nullptr. + const HashedInKey* in_key; // Valid if store_ptr == nullptr. + }; + }; class Comparator { public: bool operator()(const HashedKey& a, const HashedKey& b) const { - if (a.first != b.first) { - return a.first < b.first; + HashType a_hash = (a.store_ptr != nullptr) ? a.store_hash : a.in_key->first; + HashType b_hash = (b.store_ptr != nullptr) ? b.store_hash : b.in_key->first; + if (a_hash != b_hash) { + return a_hash < b_hash; + } + if (a.store_ptr != nullptr && b.store_ptr != nullptr) { + return std::lexicographical_compare(a.store_ptr->begin(), a.store_ptr->end(), + b.store_ptr->begin(), b.store_ptr->end()); + } else if (a.store_ptr != nullptr && b.store_ptr == nullptr) { + return std::lexicographical_compare(a.store_ptr->begin(), a.store_ptr->end(), + b.in_key->second->begin(), b.in_key->second->end()); + } else if (a.store_ptr == nullptr && b.store_ptr != nullptr) { + return std::lexicographical_compare(a.in_key->second->begin(), a.in_key->second->end(), + b.store_ptr->begin(), b.store_ptr->end()); } else { - return *a.second < *b.second; + return std::lexicographical_compare(a.in_key->second->begin(), a.in_key->second->end(), + b.in_key->second->begin(), b.in_key->second->end()); } } }; public: - Key* Add(Thread* self, const Key& key) { + StoreKey* Add(Thread* self, const InKey& key) { + uint64_t hash_start; + if (kIsDebugBuild) { + hash_start = NanoTime(); + } HashType raw_hash = HashFunc()(key); + if (kIsDebugBuild) { + uint64_t hash_end = NanoTime(); + hash_time_ += hash_end - hash_start; + } HashType shard_hash = raw_hash / kShard; HashType shard_bin = raw_hash % kShard; - HashedKey hashed_key(shard_hash, const_cast<Key*>(&key)); + HashedInKey hashed_in_key(shard_hash, &key); + HashedKey hashed_key; + hashed_key.store_ptr = nullptr; + hashed_key.in_key = &hashed_in_key; MutexLock lock(self, *lock_[shard_bin]); auto it = keys_[shard_bin].find(hashed_key); if (it != keys_[shard_bin].end()) { - return it->second; + DCHECK(it->store_ptr != nullptr); + return it->store_ptr; } - hashed_key.second = new Key(key); + hashed_key.store_ptr = CreateStoreKey(key); + hashed_key.store_hash = shard_hash; keys_[shard_bin].insert(hashed_key); - return hashed_key.second; + return hashed_key.store_ptr; } - explicit DedupeSet(const char* set_name) { + explicit DedupeSet(const char* set_name, SwapAllocator<void>& alloc) + : allocator_(alloc), hash_time_(0) { for (HashType i = 0; i < kShard; ++i) { std::ostringstream oss; oss << set_name << " lock " << i; @@ -70,15 +109,59 @@ class DedupeSet { } ~DedupeSet() { - for (HashType i = 0; i < kShard; ++i) { - STLDeleteValues(&keys_[i]); + // Have to manually free all pointers. + for (auto& shard : keys_) { + for (const auto& hashed_key : shard) { + DCHECK(hashed_key.store_ptr != nullptr); + DeleteStoreKey(hashed_key.store_ptr); + } + } + } + + std::string DumpStats() const { + size_t collision_sum = 0; + size_t collision_max = 0; + for (HashType shard = 0; shard < kShard; ++shard) { + HashType last_hash = 0; + size_t collision_cur_max = 0; + for (const HashedKey& key : keys_[shard]) { + DCHECK(key.store_ptr != nullptr); + if (key.store_hash == last_hash) { + collision_cur_max++; + if (collision_cur_max > 1) { + collision_sum++; + if (collision_cur_max > collision_max) { + collision_max = collision_cur_max; + } + } + } else { + collision_cur_max = 1; + last_hash = key.store_hash; + } + } } + return StringPrintf("%zu collisions, %zu max bucket size, %" PRIu64 " ns hash time", + collision_sum, collision_max, hash_time_); } private: + StoreKey* CreateStoreKey(const InKey& key) { + StoreKey* ret = allocator_.allocate(1); + allocator_.construct(ret, key.begin(), key.end(), allocator_); + return ret; + } + + void DeleteStoreKey(StoreKey* key) { + SwapAllocator<StoreKey> alloc(allocator_); + alloc.destroy(key); + alloc.deallocate(key, 1); + } + std::string lock_name_[kShard]; std::unique_ptr<Mutex> lock_[kShard]; std::set<HashedKey, Comparator> keys_[kShard]; + SwapAllocator<StoreKey> allocator_; + uint64_t hash_time_; DISALLOW_COPY_AND_ASSIGN(DedupeSet); }; diff --git a/compiler/utils/dedupe_set_test.cc b/compiler/utils/dedupe_set_test.cc index 8abe6debc1..637964e484 100644 --- a/compiler/utils/dedupe_set_test.cc +++ b/compiler/utils/dedupe_set_test.cc @@ -15,6 +15,10 @@ */ #include "dedupe_set.h" + +#include <algorithm> +#include <cstdio> + #include "gtest/gtest.h" #include "thread-inl.h" @@ -35,19 +39,22 @@ class DedupeHashFunc { TEST(DedupeSetTest, Test) { Thread* self = Thread::Current(); typedef std::vector<uint8_t> ByteArray; - DedupeSet<ByteArray, size_t, DedupeHashFunc> deduplicator("test"); - ByteArray* array1; + SwapAllocator<void> swap(nullptr); + DedupeSet<ByteArray, SwapVector<uint8_t>, size_t, DedupeHashFunc> deduplicator("test", swap); + SwapVector<uint8_t>* array1; { ByteArray test1; test1.push_back(10); test1.push_back(20); test1.push_back(30); test1.push_back(45); + array1 = deduplicator.Add(self, test1); - ASSERT_EQ(test1, *array1); + ASSERT_NE(array1, nullptr); + ASSERT_TRUE(std::equal(test1.begin(), test1.end(), array1->begin())); } - ByteArray* array2; + SwapVector<uint8_t>* array2; { ByteArray test1; test1.push_back(10); @@ -56,10 +63,10 @@ TEST(DedupeSetTest, Test) { test1.push_back(45); array2 = deduplicator.Add(self, test1); ASSERT_EQ(array2, array1); - ASSERT_EQ(test1, *array2); + ASSERT_TRUE(std::equal(test1.begin(), test1.end(), array2->begin())); } - ByteArray* array3; + SwapVector<uint8_t>* array3; { ByteArray test1; test1.push_back(10); @@ -67,8 +74,8 @@ TEST(DedupeSetTest, Test) { test1.push_back(30); test1.push_back(47); array3 = deduplicator.Add(self, test1); - ASSERT_NE(array3, &test1); - ASSERT_EQ(test1, *array3); + ASSERT_NE(array3, nullptr); + ASSERT_TRUE(std::equal(test1.begin(), test1.end(), array3->begin())); } } diff --git a/compiler/utils/swap_space.cc b/compiler/utils/swap_space.cc new file mode 100644 index 0000000000..19338c06a2 --- /dev/null +++ b/compiler/utils/swap_space.cc @@ -0,0 +1,205 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "swap_space.h" + +#include <algorithm> +#include <numeric> + +#include "base/logging.h" +#include "base/macros.h" +#include "base/mutex.h" +#include "thread-inl.h" + +namespace art { + +// The chunk size by which the swap file is increased and mapped. +static constexpr size_t kMininumMapSize = 16 * MB; + +static constexpr bool kCheckFreeMaps = false; + +template <typename FreeBySizeSet> +static void DumpFreeMap(const FreeBySizeSet& free_by_size) { + size_t last_size = static_cast<size_t>(-1); + for (const auto& entry : free_by_size) { + if (last_size != entry.first) { + last_size = entry.first; + LOG(INFO) << "Size " << last_size; + } + LOG(INFO) << " 0x" << std::hex << entry.second->Start() + << " size=" << std::dec << entry.second->size; + } +} + +template <typename FreeByStartSet, typename FreeBySizeSet> +static void RemoveChunk(FreeByStartSet* free_by_start, + FreeBySizeSet* free_by_size, + typename FreeBySizeSet::const_iterator free_by_size_pos) { + auto free_by_start_pos = free_by_size_pos->second; + free_by_size->erase(free_by_size_pos); + free_by_start->erase(free_by_start_pos); +} + +template <typename FreeByStartSet, typename FreeBySizeSet> +static void InsertChunk(FreeByStartSet* free_by_start, + FreeBySizeSet* free_by_size, + const SpaceChunk& chunk) { + DCHECK_NE(chunk.size, 0u); + auto insert_result = free_by_start->insert(chunk); + DCHECK(insert_result.second); + free_by_size->emplace(chunk.size, insert_result.first); +} + +SwapSpace::SwapSpace(int fd, size_t initial_size) + : fd_(fd), + size_(0), + lock_("SwapSpace lock", static_cast<LockLevel>(LockLevel::kDefaultMutexLevel - 1)) { + // Assume that the file is unlinked. + + InsertChunk(&free_by_start_, &free_by_size_, NewFileChunk(initial_size)); +} + +SwapSpace::~SwapSpace() { + // All arenas are backed by the same file. Just close the descriptor. + close(fd_); +} + +template <typename FreeByStartSet, typename FreeBySizeSet> +static size_t CollectFree(const FreeByStartSet& free_by_start, const FreeBySizeSet& free_by_size) { + if (free_by_start.size() != free_by_size.size()) { + LOG(FATAL) << "Size: " << free_by_start.size() << " vs " << free_by_size.size(); + } + + // Calculate over free_by_size. + size_t sum1 = 0; + for (const auto& entry : free_by_size) { + sum1 += entry.second->size; + } + + // Calculate over free_by_start. + size_t sum2 = 0; + for (const auto& entry : free_by_start) { + sum2 += entry.size; + } + + if (sum1 != sum2) { + LOG(FATAL) << "Sum: " << sum1 << " vs " << sum2; + } + return sum1; +} + +void* SwapSpace::Alloc(size_t size) { + MutexLock lock(Thread::Current(), lock_); + size = RoundUp(size, 8U); + + // Check the free list for something that fits. + // TODO: Smarter implementation. Global biggest chunk, ... + SpaceChunk old_chunk; + auto it = free_by_start_.empty() + ? free_by_size_.end() + : free_by_size_.lower_bound(FreeBySizeEntry { size, free_by_start_.begin() }); + if (it != free_by_size_.end()) { + old_chunk = *it->second; + RemoveChunk(&free_by_start_, &free_by_size_, it); + } else { + // Not a big enough free chunk, need to increase file size. + old_chunk = NewFileChunk(size); + } + + void* ret = old_chunk.ptr; + + if (old_chunk.size != size) { + // Insert the remainder. + SpaceChunk new_chunk = { old_chunk.ptr + size, old_chunk.size - size }; + InsertChunk(&free_by_start_, &free_by_size_, new_chunk); + } + + return ret; +} + +SpaceChunk SwapSpace::NewFileChunk(size_t min_size) { + size_t next_part = std::max(RoundUp(min_size, kPageSize), RoundUp(kMininumMapSize, kPageSize)); + int result = TEMP_FAILURE_RETRY(ftruncate64(fd_, size_ + next_part)); + if (result != 0) { + PLOG(FATAL) << "Unable to increase swap file."; + } + uint8_t* ptr = reinterpret_cast<uint8_t*>( + mmap(nullptr, next_part, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, size_)); + if (ptr == MAP_FAILED) { + LOG(ERROR) << "Unable to mmap new swap file chunk."; + LOG(ERROR) << "Current size: " << size_ << " requested: " << next_part << "/" << min_size; + LOG(ERROR) << "Free list:"; + MutexLock lock(Thread::Current(), lock_); + DumpFreeMap(free_by_size_); + LOG(ERROR) << "In free list: " << CollectFree(free_by_start_, free_by_size_); + LOG(FATAL) << "Aborting..."; + } + size_ += next_part; + SpaceChunk new_chunk = {ptr, next_part}; + maps_.push_back(new_chunk); + return new_chunk; +} + +// TODO: Full coalescing. +void SwapSpace::Free(void* ptrV, size_t size) { + MutexLock lock(Thread::Current(), lock_); + size = RoundUp(size, 8U); + + size_t free_before = 0; + if (kCheckFreeMaps) { + free_before = CollectFree(free_by_start_, free_by_size_); + } + + SpaceChunk chunk = { reinterpret_cast<uint8_t*>(ptrV), size }; + auto it = free_by_start_.lower_bound(chunk); + if (it != free_by_start_.begin()) { + auto prev = it; + --prev; + CHECK_LE(prev->End(), chunk.Start()); + if (prev->End() == chunk.Start()) { + // Merge *prev with this chunk. + chunk.size += prev->size; + chunk.ptr -= prev->size; + auto erase_pos = free_by_size_.find(FreeBySizeEntry { prev->size, prev }); + DCHECK(erase_pos != free_by_size_.end()); + RemoveChunk(&free_by_start_, &free_by_size_, erase_pos); + // "prev" is invalidated but "it" remains valid. + } + } + if (it != free_by_start_.end()) { + CHECK_LE(chunk.End(), it->Start()); + if (chunk.End() == it->Start()) { + // Merge *it with this chunk. + chunk.size += it->size; + auto erase_pos = free_by_size_.find(FreeBySizeEntry { it->size, it }); + DCHECK(erase_pos != free_by_size_.end()); + RemoveChunk(&free_by_start_, &free_by_size_, erase_pos); + // "it" is invalidated but we don't need it anymore. + } + } + InsertChunk(&free_by_start_, &free_by_size_, chunk); + + if (kCheckFreeMaps) { + size_t free_after = CollectFree(free_by_start_, free_by_size_); + + if (free_after != free_before + size) { + DumpFreeMap(free_by_size_); + CHECK_EQ(free_after, free_before + size) << "Should be " << size << " difference from " << free_before; + } + } +} + +} // namespace art diff --git a/compiler/utils/swap_space.h b/compiler/utils/swap_space.h new file mode 100644 index 0000000000..2d0d77af78 --- /dev/null +++ b/compiler/utils/swap_space.h @@ -0,0 +1,212 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_UTILS_SWAP_SPACE_H_ +#define ART_COMPILER_UTILS_SWAP_SPACE_H_ + +#include <cstdlib> +#include <list> +#include <set> +#include <stdint.h> +#include <stddef.h> + +#include "base/logging.h" +#include "base/macros.h" +#include "base/mutex.h" +#include "mem_map.h" +#include "utils.h" +#include "utils/debug_stack.h" + +namespace art { + +// Chunk of space. +struct SpaceChunk { + uint8_t* ptr; + size_t size; + + uintptr_t Start() const { + return reinterpret_cast<uintptr_t>(ptr); + } + uintptr_t End() const { + return reinterpret_cast<uintptr_t>(ptr) + size; + } +}; + +inline bool operator==(const SpaceChunk& lhs, const SpaceChunk& rhs) { + return (lhs.size == rhs.size) && (lhs.ptr == rhs.ptr); +} + +class SortChunkByPtr { + public: + bool operator()(const SpaceChunk& a, const SpaceChunk& b) const { + return reinterpret_cast<uintptr_t>(a.ptr) < reinterpret_cast<uintptr_t>(b.ptr); + } +}; + +// An arena pool that creates arenas backed by an mmaped file. +class SwapSpace { + public: + SwapSpace(int fd, size_t initial_size); + ~SwapSpace(); + void* Alloc(size_t size) LOCKS_EXCLUDED(lock_); + void Free(void* ptr, size_t size) LOCKS_EXCLUDED(lock_); + + size_t GetSize() { + return size_; + } + + private: + SpaceChunk NewFileChunk(size_t min_size); + + int fd_; + size_t size_; + std::list<SpaceChunk> maps_; + + // NOTE: Boost.Bimap would be useful for the two following members. + + // Map start of a free chunk to its size. + typedef std::set<SpaceChunk, SortChunkByPtr> FreeByStartSet; + FreeByStartSet free_by_start_ GUARDED_BY(lock_); + + // Map size to an iterator to free_by_start_'s entry. + typedef std::pair<size_t, FreeByStartSet::const_iterator> FreeBySizeEntry; + struct FreeBySizeComparator { + bool operator()(const FreeBySizeEntry& lhs, const FreeBySizeEntry& rhs) { + if (lhs.first != rhs.first) { + return lhs.first < rhs.first; + } else { + return lhs.second->Start() < rhs.second->Start(); + } + } + }; + typedef std::set<FreeBySizeEntry, FreeBySizeComparator> FreeBySizeSet; + FreeBySizeSet free_by_size_ GUARDED_BY(lock_); + + mutable Mutex lock_ DEFAULT_MUTEX_ACQUIRED_AFTER; + DISALLOW_COPY_AND_ASSIGN(SwapSpace); +}; + +template <typename T> class SwapAllocator; + +template <> +class SwapAllocator<void> { + public: + typedef void value_type; + typedef void* pointer; + typedef const void* const_pointer; + + template <typename U> + struct rebind { + typedef SwapAllocator<U> other; + }; + + explicit SwapAllocator(SwapSpace* swap_space) : swap_space_(swap_space) {} + + template <typename U> + SwapAllocator(const SwapAllocator<U>& other) : swap_space_(other.swap_space_) {} + + SwapAllocator(const SwapAllocator& other) = default; + SwapAllocator& operator=(const SwapAllocator& other) = default; + ~SwapAllocator() = default; + + private: + SwapSpace* swap_space_; + + template <typename U> + friend class SwapAllocator; +}; + +template <typename T> +class SwapAllocator { + public: + typedef T value_type; + typedef T* pointer; + typedef T& reference; + typedef const T* const_pointer; + typedef const T& const_reference; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + + template <typename U> + struct rebind { + typedef SwapAllocator<U> other; + }; + + explicit SwapAllocator(SwapSpace* swap_space) : swap_space_(swap_space) {} + + template <typename U> + SwapAllocator(const SwapAllocator<U>& other) : swap_space_(other.swap_space_) {} + + SwapAllocator(const SwapAllocator& other) = default; + SwapAllocator& operator=(const SwapAllocator& other) = default; + ~SwapAllocator() = default; + + size_type max_size() const { + return static_cast<size_type>(-1) / sizeof(T); + } + + pointer address(reference x) const { return &x; } + const_pointer address(const_reference x) const { return &x; } + + pointer allocate(size_type n, SwapAllocator<void>::pointer hint ATTRIBUTE_UNUSED = nullptr) { + DCHECK_LE(n, max_size()); + if (swap_space_ == nullptr) { + return reinterpret_cast<T*>(malloc(n * sizeof(T))); + } else { + return reinterpret_cast<T*>(swap_space_->Alloc(n * sizeof(T))); + } + } + void deallocate(pointer p, size_type n) { + if (swap_space_ == nullptr) { + free(p); + } else { + swap_space_->Free(p, n * sizeof(T)); + } + } + + void construct(pointer p, const_reference val) { + new (static_cast<void*>(p)) value_type(val); + } + template <class U, class... Args> + void construct(U* p, Args&&... args) { + ::new (static_cast<void*>(p)) U(std::forward<Args>(args)...); + } + void destroy(pointer p) { + p->~value_type(); + } + + inline bool operator==(SwapAllocator const& other) { + return swap_space_ == other.swap_space_; + } + inline bool operator!=(SwapAllocator const& other) { + return !operator==(other); + } + + private: + SwapSpace* swap_space_; + + template <typename U> + friend class SwapAllocator; +}; + +template <typename T> +using SwapVector = std::vector<T, SwapAllocator<T>>; +template <typename T, typename Comparator> +using SwapSet = std::set<T, Comparator, SwapAllocator<T>>; + +} // namespace art + +#endif // ART_COMPILER_UTILS_SWAP_SPACE_H_ diff --git a/compiler/utils/swap_space_test.cc b/compiler/utils/swap_space_test.cc new file mode 100644 index 0000000000..bf50ac3209 --- /dev/null +++ b/compiler/utils/swap_space_test.cc @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/swap_space.h" + +#include <cstdio> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include "gtest/gtest.h" + +#include "base/unix_file/fd_file.h" +#include "common_runtime_test.h" +#include "os.h" + +namespace art { + +class SwapSpaceTest : public CommonRuntimeTest { +}; + +static void SwapTest(bool use_file) { + ScratchFile scratch; + int fd = scratch.GetFd(); + unlink(scratch.GetFilename().c_str()); + + SwapSpace pool(fd, 1 * MB); + SwapAllocator<void> alloc(use_file ? &pool : nullptr); + + SwapVector<int32_t> v(alloc); + v.reserve(1000000); + for (int32_t i = 0; i < 1000000; ++i) { + v.push_back(i); + EXPECT_EQ(i, v[i]); + } + + SwapVector<int32_t> v2(alloc); + v2.reserve(1000000); + for (int32_t i = 0; i < 1000000; ++i) { + v2.push_back(i); + EXPECT_EQ(i, v2[i]); + } + + SwapVector<int32_t> v3(alloc); + v3.reserve(500000); + for (int32_t i = 0; i < 1000000; ++i) { + v3.push_back(i); + EXPECT_EQ(i, v2[i]); + } + + // Verify contents. + for (int32_t i = 0; i < 1000000; ++i) { + EXPECT_EQ(i, v[i]); + EXPECT_EQ(i, v2[i]); + EXPECT_EQ(i, v3[i]); + } + + scratch.Close(); +} + +TEST_F(SwapSpaceTest, Memory) { + SwapTest(false); +} + +TEST_F(SwapSpaceTest, Swap) { + SwapTest(true); +} + +} // namespace art diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 2cbfffaea4..63009bf25e 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -248,6 +248,12 @@ static void UsageError(const char* fmt, ...) { UsageError(" Used to specify a pass specific option. The setting itself must be integer."); UsageError(" Separator used between options is a comma."); UsageError(""); + UsageError(" --swap-file=<file-name>: specifies a file to use for swap."); + UsageError(" Example: --swap-file=/data/tmp/swap.001"); + UsageError(""); + UsageError(" --swap-fd=<file-descriptor>: specifies a file to use for swap (by descriptor)."); + UsageError(" Example: --swap-fd=10"); + UsageError(""); std::cerr << "See log for usage error information\n"; exit(EXIT_FAILURE); } @@ -393,6 +399,25 @@ static void ParseDouble(const std::string& option, char after_char, double min, *parsed_value = value; } +static constexpr size_t kMinDexFilesForSwap = 2; +static constexpr size_t kMinDexFileCumulativeSizeForSwap = 20 * MB; + +static bool UseSwap(bool is_image, std::vector<const DexFile*>& dex_files) { + if (is_image) { + // Don't use swap, we know generation should succeed, and we don't want to slow it down. + return false; + } + if (dex_files.size() < kMinDexFilesForSwap) { + // If there are less dex files than the threshold, assume it's gonna be fine. + return false; + } + size_t dex_files_size = 0; + for (const auto* dex_file : dex_files) { + dex_files_size += dex_file->GetHeader().file_size_; + } + return dex_files_size >= kMinDexFileCumulativeSizeForSwap; +} + class Dex2Oat FINAL { public: explicit Dex2Oat(TimingLogger* timings) : @@ -416,6 +441,7 @@ class Dex2Oat FINAL { dump_passes_(false), dump_timing_(false), dump_slow_timing_(kIsDebugBuild), + swap_fd_(-1), timings_(timings) {} ~Dex2Oat() { @@ -684,6 +710,16 @@ class Dex2Oat FINAL { << "failures."; init_failure_output_.reset(); } + } else if (option.starts_with("--swap-file=")) { + swap_file_name_ = option.substr(strlen("--swap-file=")).data(); + } else if (option.starts_with("--swap-fd=")) { + const char* swap_fd_str = option.substr(strlen("--swap-fd=")).data(); + if (!ParseInt(swap_fd_str, &swap_fd_)) { + Usage("Failed to parse --swap-fd argument '%s' as an integer", swap_fd_str); + } + if (swap_fd_ < 0) { + Usage("--swap-fd passed a negative value %d", swap_fd_); + } } else { Usage("Unknown argument %s", option.data()); } @@ -918,7 +954,8 @@ class Dex2Oat FINAL { } } - // Check whether the oat output file is writable, and open it for later. + // Check whether the oat output file is writable, and open it for later. Also open a swap file, + // if a name is given. bool OpenFile() { bool create_file = !oat_unstripped_.empty(); // as opposed to using open file descriptor if (create_file) { @@ -942,6 +979,27 @@ class Dex2Oat FINAL { oat_file_->Erase(); return false; } + + // Swap file handling. + // + // If the swap fd is not -1, we assume this is the file descriptor of an open but unlinked file + // that we can use for swap. + // + // If the swap fd is -1 and we have a swap-file string, open the given file as a swap file. We + // will immediately unlink to satisfy the swap fd assumption. + if (swap_fd_ == -1 && !swap_file_name_.empty()) { + std::unique_ptr<File> swap_file(OS::CreateEmptyFile(swap_file_name_.c_str())); + if (swap_file.get() == nullptr) { + PLOG(ERROR) << "Failed to create swap file: " << swap_file_name_; + return false; + } + swap_fd_ = swap_file->Fd(); + swap_file->MarkUnchecked(); // We don't we to track this, it will be unlinked immediately. + swap_file->DisableAutoClose(); // We'll handle it ourselves, the File object will be + // released immediately. + unlink(swap_file_name_.c_str()); + } + return true; } @@ -1085,6 +1143,18 @@ class Dex2Oat FINAL { } } + // If we use a swap file, ensure we are above the threshold to make it necessary. + if (swap_fd_ != -1) { + if (!UseSwap(image_, dex_files_)) { + close(swap_fd_); + swap_fd_ = -1; + LOG(INFO) << "Decided to run without swap."; + } else { + LOG(INFO) << "Accepted running with swap."; + } + } + // Note that dex2oat won't close the swap_fd_. The compiler driver's swap space will do that. + /* * If we're not in interpret-only or verify-none mode, go ahead and compile small applications. * Don't bother to check if we're doing the image. @@ -1143,6 +1213,7 @@ class Dex2Oat FINAL { dump_stats_, dump_passes_, compiler_phases_timings_.get(), + swap_fd_, profile_file_)); driver_->CompileAll(class_loader, dex_files_, timings_); @@ -1591,6 +1662,8 @@ class Dex2Oat FINAL { bool dump_passes_; bool dump_timing_; bool dump_slow_timing_; + std::string swap_file_name_; + int swap_fd_; std::string profile_file_; // Profile file to use TimingLogger* timings_; std::unique_ptr<CumulativeLogger> compiler_phases_timings_; diff --git a/runtime/base/unix_file/fd_file.cc b/runtime/base/unix_file/fd_file.cc index 780e37a4d0..f272d88807 100644 --- a/runtime/base/unix_file/fd_file.cc +++ b/runtime/base/unix_file/fd_file.cc @@ -256,4 +256,8 @@ int FdFile::FlushClose() { return (flush_result != 0) ? flush_result : close_result; } +void FdFile::MarkUnchecked() { + guard_state_ = GuardState::kNoCheck; +} + } // namespace unix_file diff --git a/runtime/base/unix_file/fd_file.h b/runtime/base/unix_file/fd_file.h index 1b1fb4e298..d51fbd68a7 100644 --- a/runtime/base/unix_file/fd_file.h +++ b/runtime/base/unix_file/fd_file.h @@ -85,6 +85,9 @@ class FdFile : public RandomAccessFile { kNoCheck // Do not check for the current file instance. }; + // WARNING: Only use this when you know what you're doing! + void MarkUnchecked(); + protected: // If the guard state indicates checking (!=kNoCheck), go to the target state "target". Print the // given warning if the current state is or exceeds warn_threshold. diff --git a/runtime/common_runtime_test.cc b/runtime/common_runtime_test.cc index 3d7bf53a69..98fe0798ae 100644 --- a/runtime/common_runtime_test.cc +++ b/runtime/common_runtime_test.cc @@ -84,15 +84,19 @@ int ScratchFile::GetFd() const { return file_->Fd(); } -void ScratchFile::Unlink() { - if (!OS::FileExists(filename_.c_str())) { - return; - } +void ScratchFile::Close() { if (file_.get() != nullptr) { if (file_->FlushCloseOrErase() != 0) { PLOG(WARNING) << "Error closing scratch file."; } } +} + +void ScratchFile::Unlink() { + if (!OS::FileExists(filename_.c_str())) { + return; + } + Close(); int unlink_result = unlink(filename_.c_str()); CHECK_EQ(0, unlink_result); } diff --git a/runtime/common_runtime_test.h b/runtime/common_runtime_test.h index edc3e1e07a..8851185ce1 100644 --- a/runtime/common_runtime_test.h +++ b/runtime/common_runtime_test.h @@ -55,6 +55,7 @@ class ScratchFile { int GetFd() const; + void Close(); void Unlink(); private: diff --git a/test/run-test b/test/run-test index 2abc1fa262..aba4e05d03 100755 --- a/test/run-test +++ b/test/run-test @@ -258,6 +258,9 @@ while true; do elif [ "x$1" = "x--always-clean" ]; then always_clean="yes" shift + elif [ "x$1" = "x--dex2oat-swap" ]; then + run_args="${run_args} --dex2oat-swap" + shift elif expr "x$1" : "x--" >/dev/null 2>&1; then echo "unknown $0 option: $1" 1>&2 usage="yes" @@ -452,6 +455,7 @@ if [ "$usage" = "yes" ]; then echo " --gcverify Run with gc verification" echo " --always-clean Delete the test files even if the test fails." echo " --android-root [path] The path on target for the android root. (/system by default)." + echo " --dex2oat-swap Use a dex2oat swap file." ) 1>&2 exit 1 fi |