Runtime support for CFI

Control Flow Integrity support in bionic.

General design:
http://clang.llvm.org/docs/ControlFlowIntegrityDesign.html#shared-library-support

This CL implements subsections "CFI Shadow" and "CFI_SlowPath" in the above document.

Bug: 22033465
Test: bionic device tests
Change-Id: I14dfea630de468eb5620e7f55f92b1397ba06217

25 files changed, 791 insertions, 16 deletions

diff --git a/libc/include/dlfcn.h b/libc/include/dlfcn.h
index 9aa4a1f26..b8f3cecee 100644
--- a/libc/include/dlfcn.h
+++ b/libc/include/dlfcn.h
@@ -29,6 +29,7 @@
 #ifndef __DLFCN_H__
 #define __DLFCN_H__
 
+#include <stdint.h>
 #include <sys/cdefs.h>
 
 __BEGIN_DECLS
diff --git a/libc/private/CFIShadow.h b/libc/private/CFIShadow.h
new file mode 100644
index 000000000..26351db40
--- /dev/null
+++ b/libc/private/CFIShadow.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef CFI_SHADOW_H
+#define CFI_SHADOW_H
+
+#include <stdint.h>
+
+#include "private/bionic_page.h"
+#include "private/bionic_macros.h"
+
+constexpr unsigned kLibraryAlignmentBits = 18;
+constexpr size_t kLibraryAlignment = 1UL << kLibraryAlignmentBits;
+
+// This class defines format of the shadow region for Control Flow Integrity support.
+// See documentation in http://clang.llvm.org/docs/ControlFlowIntegrityDesign.html#shared-library-support.
+//
+// CFI shadow is effectively a very fast and specialized implementation of dladdr: given an address that
+// belongs to a shared library or an executable, it can find the address of a specific export in that
+// library (a function called "__cfi_check"). This is only guaranteed to work for
+// addresses of possible CFI targets inside a library: indirectly called functions and virtual
+// tables. A random address inside a library may not work in the future (but it does in the current
+// implementation).
+//
+// Implementation is a sparse array of uint16_t where each element describes the location of
+// __cfi_check for a 2**kShadowGranularity range of memory. Array elements (called "shadow values"
+// below) are interpreted as follows.
+//
+// For an address P and corresponding shadow value V, the address of __cfi_check is calculated as
+//   align_up(P, 2**kShadowGranularity) - (V - 2) * (2 ** kCfiCheckGranularity)
+//
+// Special shadow values:
+//        0 = kInvalidShadow, this memory range has no valid CFI targets.
+//        1 = kUncheckedShadow, any address is this memory range is a valid CFI target
+//
+// Loader requirement: each aligned 2**kShadowGranularity region of address space may contain at
+// most one DSO.
+// Compiler requirement: __cfi_check is aligned at kCfiCheckGranularity.
+// Compiler requirement: __cfi_check for a given DSO is located below any CFI target for that DSO.
+class CFIShadow {
+ public:
+  static constexpr uintptr_t kShadowGranularity = kLibraryAlignmentBits;
+  static constexpr uintptr_t kCfiCheckGranularity = 12;
+
+  // Each uint16_t element of the shadow corresponds to this much application memory.
+  static constexpr uintptr_t kShadowAlign = 1UL << kShadowGranularity;
+
+  // Alignment of __cfi_check.
+  static constexpr uintptr_t kCfiCheckAlign = 1UL << kCfiCheckGranularity;  // 4K
+
+#if defined(__aarch64__)
+  static constexpr uintptr_t kMaxTargetAddr = 0x7fffffffff;
+#elif defined (__LP64__)
+  static constexpr uintptr_t kMaxTargetAddr = 0x7fffffffffff;
+#else
+  static constexpr uintptr_t kMaxTargetAddr = 0xffffffff;
+#endif
+
+  // Shadow is 2 -> 2**kShadowGranularity.
+  static constexpr uintptr_t kShadowSize =
+      align_up((kMaxTargetAddr >> (kShadowGranularity - 1)), PAGE_SIZE);
+
+  // Returns offset inside the shadow region for an address.
+  static constexpr uintptr_t MemToShadowOffset(uintptr_t x) {
+    return (x >> kShadowGranularity) << 1;
+  }
+
+  typedef int (*CFICheckFn)(uint64_t, void *, void *);
+
+ public:
+  enum ShadowValues : uint16_t {
+    kInvalidShadow = 0,    // Not a valid CFI target.
+    kUncheckedShadow = 1,  // Unchecked, valid CFI target.
+    kRegularShadowMin = 2  // This and all higher values encode a negative offset to __cfi_check in
+                           // the units of kCfiCheckGranularity, starting with 0 at
+                           // kRegularShadowMin.
+  };
+};
+
+#endif  // CFI_SHADOW_H
diff --git a/libc/private/bionic_macros.h b/libc/private/bionic_macros.h
index d5c5b9c6f..303218e28 100644
--- a/libc/private/bionic_macros.h
+++ b/libc/private/bionic_macros.h
@@ -48,11 +48,11 @@
     ? (1UL << (64 - __builtin_clzl(static_cast<unsigned long>(value)))) \
     : (1UL << (32 - __builtin_clz(static_cast<unsigned int>(value)))))
 
-static inline uintptr_t align_down(uintptr_t p, size_t align) {
+static constexpr uintptr_t align_down(uintptr_t p, size_t align) {
   return p & ~(align - 1);
 }
 
-static inline uintptr_t align_up(uintptr_t p, size_t align) {
+static constexpr uintptr_t align_up(uintptr_t p, size_t align) {
   return (p + align - 1) & ~(align - 1);
 }
 
diff --git a/libdl/Android.bp b/libdl/Android.bp
index a0aeeffb5..013554a9d 100644
--- a/libdl/Android.bp
+++ b/libdl/Android.bp
@@ -44,7 +44,7 @@ cc_library {
             version_script: "libdl.x86_64.map",
         },
     },
-    srcs: ["libdl.c"],
+    srcs: ["libdl.c", "libdl_cfi.cpp"],
     cflags: [
         "-Wall",
         "-Wextra",
@@ -61,6 +61,9 @@ cc_library {
     allow_undefined_symbols: true,
     system_shared_libs: [],
 
+    // For private/CFIShadow.h.
+    include_dirs: ["bionic/libc"],
+
     // This is placeholder library the actual implementation is (currently)
     // provided by the linker.
     shared_libs: [ "ld-android" ],
diff --git a/libdl/libdl.arm.map b/libdl/libdl.arm.map
index 1fdc1a7f1..f452641ee 100644
--- a/libdl/libdl.arm.map
+++ b/libdl/libdl.arm.map
@@ -36,6 +36,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.arm64.map b/libdl/libdl.arm64.map
index 28d261362..62f5ff45a 100644
--- a/libdl/libdl.arm64.map
+++ b/libdl/libdl.arm64.map
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.map.txt b/libdl/libdl.map.txt
index 0a82a2e7c..cc044fe3f 100644
--- a/libdl/libdl.map.txt
+++ b/libdl/libdl.map.txt
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.mips.map b/libdl/libdl.mips.map
index 28d261362..62f5ff45a 100644
--- a/libdl/libdl.mips.map
+++ b/libdl/libdl.mips.map
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.mips64.map b/libdl/libdl.mips64.map
index 28d261362..62f5ff45a 100644
--- a/libdl/libdl.mips64.map
+++ b/libdl/libdl.mips64.map
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.x86.map b/libdl/libdl.x86.map
index 28d261362..62f5ff45a 100644
--- a/libdl/libdl.x86.map
+++ b/libdl/libdl.x86.map
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl.x86_64.map b/libdl/libdl.x86_64.map
index 28d261362..62f5ff45a 100644
--- a/libdl/libdl.x86_64.map
+++ b/libdl/libdl.x86_64.map
@@ -35,6 +35,9 @@ LIBC_N {
 
 LIBC_PLATFORM {
   global:
+    __cfi_init;
+    __cfi_slowpath;
+    __cfi_slowpath_diag;
     android_dlwarning;
     android_get_application_target_sdk_version;
     android_set_application_target_sdk_version;
diff --git a/libdl/libdl_cfi.cpp b/libdl/libdl_cfi.cpp
new file mode 100644
index 000000000..362b093fa
--- /dev/null
+++ b/libdl/libdl_cfi.cpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sys/mman.h>
+
+#include "private/CFIShadow.h"
+
+__attribute__((__weak__, visibility("default"))) extern "C" void __loader_cfi_fail(
+    uint64_t CallSiteTypeId, void* Ptr, void* DiagData, void* CallerPc);
+
+// Base address of the CFI shadow. Passed down from the linker in __cfi_init()
+// and does not change after that. The contents of the shadow change in
+// dlopen/dlclose.
+static struct {
+  uintptr_t v;
+  char padding[PAGE_SIZE - sizeof(v)];
+} shadow_base_storage alignas(PAGE_SIZE);
+
+extern "C" uintptr_t* __cfi_init(uintptr_t shadow_base) {
+  shadow_base_storage.v = shadow_base;
+  static_assert(sizeof(shadow_base_storage) == PAGE_SIZE, "");
+  mprotect(&shadow_base_storage, PAGE_SIZE, PROT_READ);
+  return &shadow_base_storage.v;
+}
+
+static uint16_t shadow_load(void* p) {
+  uintptr_t addr = reinterpret_cast<uintptr_t>(p);
+  uintptr_t ofs = CFIShadow::MemToShadowOffset(addr);
+  if (ofs > CFIShadow::kShadowSize) return CFIShadow::kInvalidShadow;
+  return *reinterpret_cast<uint16_t*>(shadow_base_storage.v + ofs);
+}
+
+static uintptr_t cfi_check_addr(uint16_t v, void* Ptr) {
+  uintptr_t addr = reinterpret_cast<uintptr_t>(Ptr);
+  uintptr_t aligned_addr = align_up(addr, CFIShadow::kShadowAlign);
+  uintptr_t p = aligned_addr - (static_cast<uintptr_t>(v - CFIShadow::kRegularShadowMin)
+                                << CFIShadow::kCfiCheckGranularity);
+#ifdef __arm__
+  // Assume Thumb encoding. FIXME: force thumb at compile time?
+  p++;
+#endif
+  return p;
+}
+
+static inline void cfi_slowpath_common(uint64_t CallSiteTypeId, void* Ptr, void* DiagData) {
+  uint16_t v = shadow_load(Ptr);
+  switch (v) {
+    case CFIShadow::kInvalidShadow:
+      __loader_cfi_fail(CallSiteTypeId, Ptr, DiagData, __builtin_return_address(0));
+      break;
+    case CFIShadow::kUncheckedShadow:
+      break;
+    default:
+      reinterpret_cast<CFIShadow::CFICheckFn>(cfi_check_addr(v, Ptr))(CallSiteTypeId, Ptr, DiagData);
+  }
+}
+
+extern "C" void __cfi_slowpath(uint64_t CallSiteTypeId, void* Ptr) {
+  cfi_slowpath_common(CallSiteTypeId, Ptr, nullptr);
+}
+
+extern "C" void __cfi_slowpath_diag(uint64_t CallSiteTypeId, void* Ptr, void* DiagData) {
+  cfi_slowpath_common(CallSiteTypeId, Ptr, DiagData);
+}
diff --git a/linker/Android.bp b/linker/Android.bp
index a3b9cb257..29cccf4af 100644
--- a/linker/Android.bp
+++ b/linker/Android.bp
@@ -18,6 +18,7 @@ cc_binary {
         "linker.cpp",
         "linker_block_allocator.cpp",
         "linker_dlwarning.cpp",
+        "linker_cfi.cpp",
         "linker_gdb_support.cpp",
         "linker_globals.cpp",
         "linker_libc_support.c",
diff --git a/linker/dlfcn.cpp b/linker/dlfcn.cpp
index bb7b1ca0e..00921799a 100644
--- a/linker/dlfcn.cpp
+++ b/linker/dlfcn.cpp
@@ -15,6 +15,7 @@
  */
 
 #include "linker.h"
+#include "linker_cfi.h"
 #include "linker_globals.h"
 #include "linker_dlwarning.h"
 
@@ -189,6 +190,10 @@ android_namespace_t* __android_create_namespace(const char* name,
   return result;
 }
 
+void __cfi_fail(uint64_t CallSiteTypeId, void* Ptr, void *DiagData, void *CallerPc) {
+  CFIShadowWriter::CfiFail(CallSiteTypeId, Ptr, DiagData, CallerPc);
+}
+
 // name_offset: starting index of the name in libdl_info.strtab
 #define ELF32_SYM_INITIALIZER(name_offset, value, shndx) \
     { name_offset, \
@@ -225,11 +230,11 @@ static const char ANDROID_LIBDL_STRTAB[] =
   // 012345678901234 567890123456789012345678901234567 8901234567890123456789012345678901 2345678901234567 89
     "et_sdk_version\0__loader_android_init_namespaces\0__loader_android_create_namespace\0__loader_dlvsym\0__"
   // 4*
-  // 0000000000111111111122222 2222233333333334444444 4445555555555666666666677777777778 8888888889999999 999
-  // 0123456789012345678901234 5678901234567890123456 7890123456789012345678901234567890 1234567890123456 789
-    "loader_android_dlwarning\0"
+  // 0000000000111111111122222 222223333333333444 4444444555555555566666666667777 77777788888888889999999999
+  // 0123456789012345678901234 567890123456789012 3456789012345678901234567890123 45678901234567890123456789
+    "loader_android_dlwarning\0__loader_cfi_fail\0"
 #if defined(__arm__)
-  // 425
+  // 443
     "__loader_dl_unwind_find_exidx\0"
 #endif
     ;
@@ -255,8 +260,9 @@ static ElfW(Sym) g_libdl_symtab[] = {
   ELFW(SYM_INITIALIZER)(348, &__android_create_namespace, 1),
   ELFW(SYM_INITIALIZER)(382, &__dlvsym, 1),
   ELFW(SYM_INITIALIZER)(398, &__android_dlwarning, 1),
+  ELFW(SYM_INITIALIZER)(425, &__cfi_fail, 1),
 #if defined(__arm__)
-  ELFW(SYM_INITIALIZER)(425, &__dl_unwind_find_exidx, 1),
+  ELFW(SYM_INITIALIZER)(443, &__dl_unwind_find_exidx, 1),
 #endif
 };
 
@@ -273,9 +279,9 @@ static ElfW(Sym) g_libdl_symtab[] = {
 // Note that adding any new symbols here requires stubbing them out in libdl.
 static unsigned g_libdl_buckets[1] = { 1 };
 #if defined(__arm__)
-static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0 };
+static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0 };
 #else
-static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 };
+static unsigned g_libdl_chains[] = { 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0 };
 #endif
 
 static uint8_t __libdl_info_buf[sizeof(soinfo)] __attribute__((aligned(8)));
diff --git a/linker/linker.cpp b/linker/linker.cpp
index fc8d1efec..83bd9f340 100644
--- a/linker/linker.cpp
+++ b/linker/linker.cpp
@@ -48,6 +48,7 @@
 
 #include "linker.h"
 #include "linker_block_allocator.h"
+#include "linker_cfi.h"
 #include "linker_gdb_support.h"
 #include "linker_globals.h"
 #include "linker_debug.h"
@@ -105,6 +106,12 @@ static const char* const kAsanDefaultLdPaths[] = {
 // Is ASAN enabled?
 static bool g_is_asan = false;
 
+static CFIShadowWriter g_cfi_shadow;
+
+CFIShadowWriter* get_cfi_shadow() {
+  return &g_cfi_shadow;
+}
+
 static bool is_system_library(const std::string& realpath) {
   for (const auto& dir : g_default_namespace.get_default_library_paths()) {
     if (file_is_in_dir(realpath, dir)) {
@@ -1226,7 +1233,7 @@ static bool load_library(android_namespace_t* ns,
 //    target_sdk_version (*candidate != nullptr)
 // 2. The library was not found by soname (*candidate is nullptr)
 static bool find_loaded_library_by_soname(android_namespace_t* ns,
-                                          const char* name, soinfo** candidate) {
+                                         const char* name, soinfo** candidate) {
   *candidate = nullptr;
 
   // Ignore filename with path.
@@ -1504,7 +1511,8 @@ bool find_libraries(android_namespace_t* ns,
 
   bool linked = local_group.visit([&](soinfo* si) {
     if (!si->is_linked()) {
-      if (!si->link_image(global_group, local_group, extinfo)) {
+      if (!si->link_image(global_group, local_group, extinfo) ||
+          !get_cfi_shadow()->AfterLoad(si, solist_get_head())) {
         return false;
       }
     }
@@ -1656,6 +1664,7 @@ static void soinfo_unload(soinfo* soinfos[], size_t count) {
 
   while ((si = local_unload_list.pop_front()) != nullptr) {
     notify_gdb_of_unload(si);
+    get_cfi_shadow()->BeforeUnload(si);
     soinfo_free(si);
   }
 
diff --git a/linker/linker.h b/linker/linker.h
index c65d50358..7746982cb 100644
--- a/linker/linker.h
+++ b/linker/linker.h
@@ -103,6 +103,8 @@ void count_relocation(RelocationKind kind);
 
 soinfo* get_libdl_info(const char* linker_path);
 
+soinfo* find_containing_library(const void* p);
+
 void do_android_get_LD_LIBRARY_PATH(char*, size_t);
 void do_android_update_LD_LIBRARY_PATH(const char* ld_library_path);
 void* do_dlopen(const char* name,
@@ -125,6 +127,10 @@ bool do_dlsym(void* handle, const char* sym_name,
 
 int do_dladdr(const void* addr, Dl_info* info);
 
+// void ___cfi_slowpath(uint64_t CallSiteTypeId, void *Ptr, void *Ret);
+// void ___cfi_slowpath_diag(uint64_t CallSiteTypeId, void *Ptr, void *DiagData, void *Ret);
+void ___cfi_fail(uint64_t CallSiteTypeId, void* Ptr, void *DiagData, void *Ret);
+
 void set_application_target_sdk_version(uint32_t target);
 uint32_t get_application_target_sdk_version();
 
@@ -163,7 +169,4 @@ android_namespace_t* create_namespace(const void* caller_addr,
                                       const char* permitted_when_isolated_path,
                                       android_namespace_t* parent_namespace);
 
-constexpr unsigned kLibraryAlignmentBits = 18;
-constexpr size_t kLibraryAlignment = 1UL << kLibraryAlignmentBits;
-
 #endif
diff --git a/linker/linker_cfi.cpp b/linker/linker_cfi.cpp
new file mode 100644
index 000000000..f788c168c
--- /dev/null
+++ b/linker/linker_cfi.cpp
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "linker_cfi.h"
+
+#include "linker_debug.h"
+#include "linker_globals.h"
+#include "private/bionic_page.h"
+#include "private/bionic_prctl.h"
+
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <cstdint>
+
+// Update shadow without making it writable by preparing the data on the side and mremap-ing it in
+// place.
+class ShadowWrite {
+  char* shadow_start;
+  char* shadow_end;
+  char* aligned_start;
+  char* aligned_end;
+  char* tmp_start;
+
+ public:
+  ShadowWrite(uint16_t* s, uint16_t* e) {
+    shadow_start = reinterpret_cast<char*>(s);
+    shadow_end = reinterpret_cast<char*>(e);
+    aligned_start = reinterpret_cast<char*>(PAGE_START(reinterpret_cast<uintptr_t>(shadow_start)));
+    aligned_end = reinterpret_cast<char*>(PAGE_END(reinterpret_cast<uintptr_t>(shadow_end)));
+    tmp_start =
+        reinterpret_cast<char*>(mmap(nullptr, aligned_end - aligned_start, PROT_READ | PROT_WRITE,
+                                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+    CHECK(tmp_start != MAP_FAILED);
+    memcpy(tmp_start, aligned_start, shadow_start - aligned_start);
+    memcpy(tmp_start + (shadow_end - aligned_start), shadow_end, aligned_end - shadow_end);
+  }
+
+  uint16_t* begin() {
+    return reinterpret_cast<uint16_t*>(tmp_start + (shadow_start - aligned_start));
+  }
+
+  uint16_t* end() {
+    return reinterpret_cast<uint16_t*>(tmp_start + (shadow_end - aligned_start));
+  }
+
+  ~ShadowWrite() {
+    size_t size = aligned_end - aligned_start;
+    mprotect(tmp_start, size, PROT_READ);
+    void* res = mremap(tmp_start, size, size, MREMAP_MAYMOVE | MREMAP_FIXED,
+                       reinterpret_cast<void*>(aligned_start));
+    CHECK(res != MAP_FAILED);
+  }
+};
+
+void CFIShadowWriter::FixupVmaName() {
+  prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, *shadow_start, kShadowSize, "cfi shadow");
+}
+
+void CFIShadowWriter::AddConstant(uintptr_t begin, uintptr_t end, uint16_t v) {
+  uint16_t* shadow_begin = MemToShadow(begin);
+  uint16_t* shadow_end = MemToShadow(end - 1) + 1;
+
+  ShadowWrite sw(shadow_begin, shadow_end);
+  std::fill(sw.begin(), sw.end(), v);
+}
+
+void CFIShadowWriter::AddUnchecked(uintptr_t begin, uintptr_t end) {
+  AddConstant(begin, end, kUncheckedShadow);
+}
+
+void CFIShadowWriter::AddInvalid(uintptr_t begin, uintptr_t end) {
+  AddConstant(begin, end, kInvalidShadow);
+}
+
+void CFIShadowWriter::Add(uintptr_t begin, uintptr_t end, uintptr_t cfi_check) {
+  CHECK((cfi_check & (kCfiCheckAlign - 1)) == 0);
+
+  // Don't fill anything below cfi_check. We can not represent those addresses
+  // in the shadow, and must make sure at codegen to place all valid call
+  // targets above cfi_check.
+  begin = std::max(begin, cfi_check) & ~(kShadowAlign - 1);
+  uint16_t* shadow_begin = MemToShadow(begin);
+  uint16_t* shadow_end = MemToShadow(end - 1) + 1;
+
+  ShadowWrite sw(shadow_begin, shadow_end);
+  uint16_t sv = ((begin + kShadowAlign - cfi_check) >> kCfiCheckGranularity) + kRegularShadowMin;
+
+  // With each step of the loop below, __cfi_check address computation base is increased by
+  // 2**ShadowGranularity.
+  // To compensate for that, each next shadow value must be increased by 2**ShadowGranularity /
+  // 2**CfiCheckGranularity.
+  uint16_t sv_step = 1 << (kShadowGranularity - kCfiCheckGranularity);
+  for (uint16_t& s : sw) {
+    // If there is something there already, fall back to unchecked. This may happen in rare cases
+    // with MAP_FIXED libraries. FIXME: consider using a (slow) resolution function instead.
+    s = (s == kInvalidShadow) ? sv : kUncheckedShadow;
+    sv += sv_step;
+  }
+}
+
+static soinfo* find_libdl(soinfo* solist) {
+  for (soinfo* si = solist; si != nullptr; si = si->next) {
+    const char* soname = si->get_soname();
+    if (soname && strcmp(soname, "libdl.so") == 0) {
+      return si;
+    }
+  }
+  return nullptr;
+}
+
+static uintptr_t soinfo_find_symbol(soinfo* si, const char* s) {
+  SymbolName name(s);
+  const ElfW(Sym) * sym;
+  if (si->find_symbol_by_name(name, nullptr, &sym) && sym) {
+    return si->resolve_symbol_address(sym);
+  }
+  return 0;
+}
+
+uintptr_t soinfo_find_cfi_check(soinfo* si) {
+  return soinfo_find_symbol(si, "__cfi_check");
+}
+
+uintptr_t CFIShadowWriter::MapShadow() {
+  void* p =
+      mmap(nullptr, kShadowSize, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+  CHECK(p != MAP_FAILED);
+  return reinterpret_cast<uintptr_t>(p);
+}
+
+bool CFIShadowWriter::AddLibrary(soinfo* si) {
+  CHECK(shadow_start != nullptr);
+  if (si->base == 0 || si->size == 0) {
+    return true;
+  }
+  uintptr_t cfi_check = soinfo_find_cfi_check(si);
+  if (cfi_check == 0) {
+    INFO("[ CFI add 0x%zx + 0x%zx %s ]", static_cast<uintptr_t>(si->base),
+         static_cast<uintptr_t>(si->size), si->get_soname());
+    AddUnchecked(si->base, si->base + si->size);
+    return true;
+  }
+
+  INFO("[ CFI add 0x%zx + 0x%zx %s: 0x%zx ]", static_cast<uintptr_t>(si->base),
+       static_cast<uintptr_t>(si->size), si->get_soname(), cfi_check);
+#ifdef __arm__
+  // Require Thumb encoding.
+  if ((cfi_check & 1UL) != 1UL) {
+    DL_ERR("__cfi_check in not a Thumb function in the library \"%s\"", si->get_soname());
+    return false;
+  }
+  cfi_check &= ~1UL;
+#endif
+  if ((cfi_check & (kCfiCheckAlign - 1)) != 0) {
+    DL_ERR("unaligned __cfi_check in the library \"%s\"", si->get_soname());
+    return false;
+  }
+  Add(si->base, si->base + si->size, cfi_check);
+  return true;
+}
+
+// Pass the shadow mapping address to libdl.so. In return, we get an pointer to the location
+// libdl.so uses to store the address.
+bool CFIShadowWriter::NotifyLibDl(soinfo* solist, uintptr_t p) {
+  soinfo* libdl = find_libdl(solist);
+  if (libdl == nullptr) {
+    DL_ERR("CFI could not find libdl");
+    return false;
+  }
+
+  uintptr_t cfi_init = soinfo_find_symbol(libdl, "__cfi_init");
+  CHECK(cfi_init != 0);
+  shadow_start = reinterpret_cast<uintptr_t* (*)(uintptr_t)>(cfi_init)(p);
+  CHECK(shadow_start != nullptr);
+  CHECK(*shadow_start == p);
+  return true;
+}
+
+bool CFIShadowWriter::MaybeInit(soinfo* new_si, soinfo* solist) {
+  CHECK(initial_link_done);
+  // Check if CFI shadow must be initialized at this time.
+  bool found = false;
+  if (new_si == nullptr) {
+    // This is the case when we've just completed the initial link. There may have been earlier
+    // calls to MaybeInit that were skipped. Look though the entire solist.
+    for (soinfo* si = solist; si != nullptr; si = si->next) {
+      if (soinfo_find_cfi_check(si)) {
+        found = true;
+        break;
+      }
+    }
+  } else {
+    // See if the new library uses CFI.
+    found = soinfo_find_cfi_check(new_si);
+  }
+
+  // Nothing found.
+  if (!found) {
+    return true;
+  }
+
+  // Init shadow and add all currently loaded libraries (not just the new ones).
+  if (!NotifyLibDl(solist, MapShadow()))
+    return false;
+  for (soinfo* si = solist; si != nullptr; si = si->next) {
+    if (!AddLibrary(si))
+      return false;
+  }
+  FixupVmaName();
+  return true;
+}
+
+bool CFIShadowWriter::AfterLoad(soinfo* si, soinfo* solist) {
+  if (!initial_link_done) {
+    // Too early.
+    return true;
+  }
+
+  if (shadow_start == nullptr) {
+    return MaybeInit(si, solist);
+  }
+
+  // Add the new library to the CFI shadow.
+  if (!AddLibrary(si))
+    return false;
+  FixupVmaName();
+  return true;
+}
+
+void CFIShadowWriter::BeforeUnload(soinfo* si) {
+  if (shadow_start == nullptr) return;
+  if (si->base == 0 || si->size == 0) return;
+  INFO("[ CFI remove 0x%zx + 0x%zx: %s ]", static_cast<uintptr_t>(si->base),
+       static_cast<uintptr_t>(si->size), si->get_soname());
+  AddInvalid(si->base, si->base + si->size);
+  FixupVmaName();
+}
+
+bool CFIShadowWriter::InitialLinkDone(soinfo* solist) {
+  CHECK(!initial_link_done)
+  initial_link_done = true;
+  return MaybeInit(nullptr, solist);
+}
+
+// Find __cfi_check in the caller and let it handle the problem. Since caller_pc is likely not a
+// valid CFI target, we can not use CFI shadow for lookup. This does not need to be fast, do the
+// regular symbol lookup.
+void CFIShadowWriter::CfiFail(uint64_t CallSiteTypeId, void* Ptr, void* DiagData, void* CallerPc) {
+  soinfo* si = find_containing_library(CallerPc);
+  if (!si) {
+    __builtin_trap();
+  }
+
+  uintptr_t cfi_check = soinfo_find_cfi_check(si);
+  if (!cfi_check) {
+    __builtin_trap();
+  }
+
+  reinterpret_cast<CFICheckFn>(cfi_check)(CallSiteTypeId, Ptr, DiagData);
+}
diff --git a/linker/linker_cfi.h b/linker/linker_cfi.h
new file mode 100644
index 000000000..df7642193
--- /dev/null
+++ b/linker/linker_cfi.h
@@ -0,0 +1,96 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _LINKER_CFI_H_
+#define _LINKER_CFI_H_
+
+#include "linker.h"
+#include "linker_debug.h"
+
+#include <algorithm>
+
+#include "private/CFIShadow.h"
+
+// This class keeps the contents of CFI shadow up-to-date with the current set of loaded libraries.
+// See the comment in CFIShadow.h for more context.
+// See documentation in http://clang.llvm.org/docs/ControlFlowIntegrityDesign.html#shared-library-support.
+//
+// Shadow is mapped and initialized lazily as soon as the first CFI-enabled DSO is loaded.
+// It is updated after a set of libraries is loaded (but before any constructors are ran), and
+// before any library is unloaded.
+class CFIShadowWriter : private CFIShadow {
+  // Returns pointer to the shadow element for an address.
+  uint16_t* MemToShadow(uintptr_t x) {
+    return reinterpret_cast<uint16_t*>(*shadow_start + MemToShadowOffset(x));
+  }
+
+  // Update shadow for the address range to the given constant value.
+  void AddConstant(uintptr_t begin, uintptr_t end, uint16_t v);
+
+  // Update shadow for the address range to kUncheckedShadow.
+  void AddUnchecked(uintptr_t begin, uintptr_t end);
+
+  // Update shadow for the address range to kInvalidShadow.
+  void AddInvalid(uintptr_t begin, uintptr_t end);
+
+  // Update shadow for the address range to the given __cfi_check value.
+  void Add(uintptr_t begin, uintptr_t end, uintptr_t cfi_check);
+
+  // Add a DSO to CFI shadow.
+  bool AddLibrary(soinfo* si);
+
+  // Map CFI shadow.
+  uintptr_t MapShadow();
+
+  // Initialize CFI shadow and update its contents for everything in solist if any loaded library is
+  // CFI-enabled. If soinfos != nullptr, do an incremental check by looking only at the libraries in
+  // soinfos[]; otherwise look at the entire solist.
+  //
+  // Returns false if the shadow is already initialized. It is the caller's responsibility to update
+  // the shadow for the new libraries in that case.
+  // Otherwise, returns true and leaves the shadow either up-to-date or uninitialized.
+  bool MaybeInit(soinfo *new_si, soinfo *solist);
+
+  // Set a human readable name for the entire shadow region.
+  void FixupVmaName();
+
+  bool NotifyLibDl(soinfo *solist, uintptr_t p);
+
+  // Pointer to the shadow start address.
+  uintptr_t *shadow_start;
+
+  bool initial_link_done;
+
+ public:
+  // Update shadow after loading a set of DSOs.
+  // This function will initialize the shadow if it sees a CFI-enabled DSO for the first time.
+  // In that case it will retroactively update shadow for all previously loaded DSOs. "solist" is a
+  // pointer to the global list.
+  // This function must be called before any user code has observed the newly loaded DSO.
+  bool AfterLoad(soinfo* si, soinfo *solist);
+
+  // Update shadow before unloading a DSO.
+  void BeforeUnload(soinfo* si);
+
+  bool InitialLinkDone(soinfo *solist);
+
+  // Handle failure to locate __cfi_check for a target address.
+  static void CfiFail(uint64_t CallSiteTypeId, void* Ptr, void* DiagData, void *caller_pc);
+};
+
+CFIShadowWriter* get_cfi_shadow();
+
+#endif // _LINKER_CFI_H_
diff --git a/linker/linker_main.cpp b/linker/linker_main.cpp
index dad74096d..3a2f35b0f 100644
--- a/linker/linker_main.cpp
+++ b/linker/linker_main.cpp
@@ -29,6 +29,7 @@
 #include "linker_main.h"
 
 #include "linker_debug.h"
+#include "linker_cfi.h"
 #include "linker_gdb_support.h"
 #include "linker_globals.h"
 #include "linker_phdr.h"
@@ -368,6 +369,10 @@ static ElfW(Addr) __linker_init_post_relocation(KernelArgumentBlock& args, ElfW(
 
   add_vdso(args);
 
+  if (!get_cfi_shadow()->InitialLinkDone(solist)) {
+    __libc_fatal("CANNOT LINK EXECUTABLE \"%s\": %s", g_argv[0], linker_get_error_buffer());
+  }
+
   {
     ProtectedDataGuard guard;
 
diff --git a/linker/linker_phdr.cpp b/linker/linker_phdr.cpp
index a453bef9a..72549cc73 100644
--- a/linker/linker_phdr.cpp
+++ b/linker/linker_phdr.cpp
@@ -42,6 +42,7 @@
 #include "linker_utils.h"
 
 #include "private/bionic_prctl.h"
+#include "private/CFIShadow.h" // For kLibraryAlignment
 
 static int GetTargetElfMachine() {
 #if defined(__arm__)
diff --git a/tests/Android.bp b/tests/Android.bp
index 84be7bc1f..564ef03b5 100644
--- a/tests/Android.bp
+++ b/tests/Android.bp
@@ -295,6 +295,7 @@ cc_test_library {
     target: {
         android: {
             srcs: [
+                "cfi_test.cpp",
                 "dlext_test.cpp",
                 "libdl_test.cpp",
             ],
diff --git a/tests/cfi_test.cpp b/tests/cfi_test.cpp
new file mode 100644
index 000000000..0f93edb14
--- /dev/null
+++ b/tests/cfi_test.cpp
@@ -0,0 +1,94 @@
+#include <gtest/gtest.h>
+#include <dlfcn.h>
+
+#include "BionicDeathTest.h"
+
+// Private libdl interface.
+extern "C" {
+void __cfi_slowpath(uint64_t CallSiteTypeId, void* Ptr);
+void __cfi_slowpath_diag(uint64_t CallSiteTypeId, void* Ptr, void* DiagData);
+}
+
+static void f() {}
+
+TEST(cfi_test, basic) {
+  void* handle;
+  handle = dlopen("libcfi-test.so", RTLD_NOW | RTLD_LOCAL);
+  ASSERT_TRUE(handle != nullptr) << dlerror();
+
+#define SYM(type, name) auto name = reinterpret_cast<type>(dlsym(handle, #name))
+  SYM(int (*)(), get_count);
+  SYM(uint64_t(*)(), get_last_type_id);
+  SYM(void* (*)(), get_last_address);
+  SYM(void* (*)(), get_last_diag);
+  SYM(void* (*)(), get_global_address);
+  SYM(void (*)(uint64_t, void*, void*), __cfi_check);
+#undef SYM
+
+  int c = get_count();
+
+  // CFI check for code inside the DSO. Can't use just any function address - this is only
+  // guaranteed to work for code addresses above __cfi_check.
+  void* code_ptr = reinterpret_cast<char*>(__cfi_check) + 1234;
+  void* diag_ptr = reinterpret_cast<void*>(5678);
+  __cfi_slowpath_diag(42, code_ptr, diag_ptr);
+  EXPECT_EQ(42U, get_last_type_id());
+  EXPECT_EQ(code_ptr, get_last_address());
+  EXPECT_EQ(diag_ptr, get_last_diag());
+  EXPECT_EQ(++c, get_count());
+
+  // __cfi_slowpath passes nullptr for the Diag argument.
+  __cfi_slowpath(42, code_ptr);
+  EXPECT_EQ(42U, get_last_type_id());
+  EXPECT_EQ(code_ptr, get_last_address());
+  EXPECT_EQ(nullptr, get_last_diag());
+  EXPECT_EQ(++c, get_count());
+
+  // CFI check for a data address inside the DSO.
+  __cfi_slowpath(43, get_global_address());
+  EXPECT_EQ(43U, get_last_type_id());
+  EXPECT_EQ(get_global_address(), get_last_address());
+  EXPECT_EQ(++c, get_count());
+
+  // CFI check for a function inside _this_ DSO. It either goes to this DSO's __cfi_check,
+  // or (if missing) is simply ignored. Any way, it does not affect the test lib's counters.
+  __cfi_slowpath(44, reinterpret_cast<void*>(&f));
+  EXPECT_EQ(43U, get_last_type_id());
+  EXPECT_EQ(get_global_address(), get_last_address());
+  EXPECT_EQ(c, get_count());
+
+  // CFI check for a stack address. This is always invalid and gets the process killed.
+  EXPECT_DEATH(__cfi_slowpath(45, reinterpret_cast<void*>(&c)), "");
+
+  // CFI check for a heap address. This is always invalid and gets the process killed.
+  void* p = malloc(4096);
+  EXPECT_DEATH(__cfi_slowpath(46, p), "");
+  free(p);
+
+  // Load the same library again.
+  void* handle2 = dlopen("libcfi-test.so", RTLD_NOW | RTLD_LOCAL);
+  ASSERT_TRUE(handle2 != nullptr) << dlerror();
+  EXPECT_EQ(handle2, handle);
+
+  // Check that it is still there.
+  __cfi_slowpath(43, get_global_address());
+  EXPECT_EQ(43U, get_last_type_id());
+  EXPECT_EQ(get_global_address(), get_last_address());
+  EXPECT_EQ(++c, get_count());
+
+  dlclose(handle);
+  dlclose(handle2);
+
+  // CFI check for a function inside the unloaded DSO. This is always invalid and gets the process
+  // killed.
+  EXPECT_DEATH(__cfi_slowpath(45, reinterpret_cast<void*>(code_ptr)), "");
+}
+
+TEST(cfi_test, invalid) {
+  void* handle;
+  handle = dlopen("libcfi-test-bad.so", RTLD_NOW | RTLD_LOCAL);
+  ASSERT_FALSE(handle != nullptr) << dlerror();
+
+  handle = dlopen("libcfi-test-bad.so", RTLD_NOW | RTLD_LOCAL);
+  ASSERT_FALSE(handle != nullptr) << dlerror();
+}
diff --git a/tests/libs/Android.bp b/tests/libs/Android.bp
index 4cd991a8d..330f17e6a 100644
--- a/tests/libs/Android.bp
+++ b/tests/libs/Android.bp
@@ -24,7 +24,8 @@ cc_defaults {
     relative_install_path: "bionic-loader-test-libs",
     gtest: false,
     sanitize: {
-        never: true,
+        address: false,
+        coverage: false,
     },
     target: {
         darwin: {
@@ -474,3 +475,21 @@ cc_binary_host {
         "libutils",
     ],
 }
+
+cc_test_library {
+    name: "libcfi-test",
+    defaults: ["bionic_testlib_defaults"],
+    srcs: ["cfi_test_lib.cpp"],
+    sanitize: {
+        cfi: false,
+    },
+}
+
+cc_test_library {
+    name: "libcfi-test-bad",
+    defaults: ["bionic_testlib_defaults"],
+    srcs: ["cfi_test_bad_lib.cpp"],
+    sanitize: {
+        cfi: false,
+    },
+}
diff --git a/tests/libs/cfi_test_bad_lib.cpp b/tests/libs/cfi_test_bad_lib.cpp
new file mode 100644
index 000000000..429c84393
--- /dev/null
+++ b/tests/libs/cfi_test_bad_lib.cpp
@@ -0,0 +1,4 @@
+// Mock an invalid CFI-enabled library.
+__attribute__((aligned(4096))) extern "C" char dummy[16] = {};
+__asm__(".globl __cfi_check");
+__asm__("__cfi_check = dummy + 3"); // Not aligned to anything.
diff --git a/tests/libs/cfi_test_lib.cpp b/tests/libs/cfi_test_lib.cpp
new file mode 100644
index 000000000..b0e2f426d
--- /dev/null
+++ b/tests/libs/cfi_test_lib.cpp
@@ -0,0 +1,68 @@
+#include <assert.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+// This library is built for all targets, including host tests, so __cfi_slowpath may not be
+// present. But it is only used in the bionic loader tests.
+extern "C" __attribute__((weak)) void __cfi_slowpath(uint64_t, void*);
+
+static int g_count;
+static uint64_t g_last_type_id;
+static void* g_last_address;
+static void* g_last_diag;
+
+extern "C" {
+
+// Mock a CFI-enabled library without relying on the compiler.
+__attribute__((aligned(4096))) void __cfi_check(uint64_t CallSiteTypeId, void* TargetAddr,
+                                                void* Diag) {
+  ++g_count;
+  g_last_type_id = CallSiteTypeId;
+  g_last_address = TargetAddr;
+  g_last_diag = Diag;
+}
+
+int get_count() {
+  return g_count;
+}
+
+uint64_t get_last_type_id() {
+  return g_last_type_id;
+}
+
+void* get_last_address() {
+  return g_last_address;
+}
+
+void* get_last_diag() {
+  return g_last_diag;
+}
+
+void* get_global_address() {
+  return &g_count;
+}
+}
+
+// Check that CFI is set up in module constructors and destructors.
+struct A {
+  void check_cfi_self() {
+    g_last_type_id = 0;
+    assert(&__cfi_slowpath);
+    // CFI check for an invalid address. Normally, this would kill the process by routing the call
+    // back to the calling module's __cfi_check, which does the right thing based on
+    // -fsanitize-recover / -fsanitize-trap. But this module has custom __cfi_check that does not do
+    // any of that, so the result looks like a passing check.
+    int zz;
+    __cfi_slowpath(13, static_cast<void*>(&zz));
+    assert(g_last_type_id == 13);
+    // CFI check for a libc function. This never goes into this module's __cfi_check, and must pass.
+    __cfi_slowpath(14, reinterpret_cast<void*>(&exit));
+    assert(g_last_type_id == 13);
+  }
+  A() {
+    check_cfi_self();
+  }
+  ~A() {
+    check_cfi_self();
+  }
+} a;