Merge Android RHEADmaster

Bug: 168057903
Merged-In: I3c77335b908d1ae9b675ef482589858ed27a9b97
Change-Id: I36f1c71de91f477257efed4c541bea519457b918

69 files changed, 13671 insertions, 247 deletions

diff --git a/src/binder/iiorap_def.h b/src/binder/iiorap_def.h
index 0c1eac8..5763380 100644
--- a/src/binder/iiorap_def.h
+++ b/src/binder/iiorap_def.h
@@ -29,6 +29,9 @@ FN_BEGIN(::com::google::android::startup::iorap::,IIorap)
 FN(setTaskListener, (const ::android::sp<::com::google::android::startop::iorap::,ITaskListener,>&,listener)) /*NOLINT*/ \
 FN(onAppLaunchEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request),          \
                     (const ::com::google::android::startop::iorap::,AppLaunchEvent,&,event))       \
+FN(onJobScheduledEvent,                                                                            \
+                    (const ::com::google::android::startop::iorap::,RequestId,&,request),          \
+                    (const ::com::google::android::startop::iorap::,JobScheduledEvent,&,event))    \
 FN(onPackageEvent,  (const ::com::google::android::startop::iorap::,RequestId,&,request),          \
                     (const ::com::google::android::startop::iorap::,PackageEvent,&,event))         \
 FN(onAppIntentEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request),          \
@@ -39,6 +42,8 @@ FN(onSystemServiceEvent,
 FN(onSystemServiceUserEvent,                                                                       \
                     (const ::com::google::android::startop::iorap::,RequestId,&,request),          \
                     (const ::com::google::android::startop::iorap::,SystemServiceUserEvent,&,event))\
+FN(onDexOptEvent,   (const ::com::google::android::startop::iorap::,RequestId,&,request),          \
+                    (const ::com::google::android::startop::iorap::,DexOptEvent,&,event))          \
 FN_END()                                                                                           \
 
 // Convenience macros to unpack the 2nd parameter from IIORAP_IFACE_DEF#FN calls.
diff --git a/src/binder/iiorap_impl.cc b/src/binder/iiorap_impl.cc
index a6f6409..de5a55d 100644
--- a/src/binder/iiorap_impl.cc
+++ b/src/binder/iiorap_impl.cc
@@ -24,6 +24,11 @@
 #include <binder/BinderService.h>
 #include <binder/IPCThreadState.h>
 #include <include/binder/request_id.h>
+#include <utils/Printer.h>
+
+#include <codecvt>
+#include <locale>
+#include <utility>
 
 /*
  * Definitions for the IIorap binder native service implementation.
@@ -76,10 +81,42 @@ static std::atomic<bool> s_service_params_ready_{false};
 static ServiceParams s_service_params_;
 static std::atomic<ServiceParams*> s_service_params_atomic_;
 
+// Convert an android::String16 (UTF-16) to a UTF-8 std::string.
+static std::string String16ToStdString(const ::android::String16& s16) {
+  std::u16string u16{s16.string()};
+  std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> convert;
+
+  std::string res = convert.to_bytes(u16);
+  return res;
+}
+
 }  // namespace anonymous
 
 class IIorapImpl::Impl {
+  // ITaskListener implementation for iorap::manager::EventManager.
+  struct EventManagerTaskCallbacks : public iorap::manager::TaskResultCallbacks {
+    explicit EventManagerTaskCallbacks(iorap::borrowed<IIorapImpl::Impl*> impl) {
+      CHECK(impl != nullptr);
+      impl_ = impl;
+    }
+
+    virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override {
+      impl_->ReplyWithResult(request_id, /*completed*/false, std::move(task_result));
+    }
+    virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override {
+      impl_->ReplyWithResult(request_id, /*completed*/true, std::move(task_result));
+    }
+
+    virtual ~EventManagerTaskCallbacks() {}
+
+    iorap::borrowed<IIorapImpl::Impl*> impl_;
+  };
+
  public:
+  ~Impl() {
+    package_manager_->UnregisterPackageChangeObserver(package_change_observer_);
+  }
+
   void SetTaskListener(const ::android::sp<ITaskListener>& listener) {
     ::android::sp<ITaskListener> old_listener = listener_;
     if (old_listener != nullptr && listener != nullptr) {
@@ -93,7 +130,7 @@ class IIorapImpl::Impl {
     if (listener == nullptr) {
       // No listener. Cannot send anything back to the client.
       // This could be normal, e.g. client had set listener to null before disconnecting.
-      LOG(WARNING) << "Drop result, no listener registered.";
+      LOG(DEBUG) << "Drop result, no listener registered.";
       // TODO: print the result with ostream operator<<
       return;
     }
@@ -111,6 +148,26 @@ class IIorapImpl::Impl {
     }
   }
 
+  void ReplyWithResult(const RequestId& request_id, bool completed, TaskResult result) {
+    ::android::sp<ITaskListener> listener = listener_;
+    if (listener == nullptr) {
+      // No listener. Cannot send anything back to the client.
+      // This could be normal, e.g. client had set listener to null before disconnecting.
+      LOG(DEBUG) << "Drop result, no listener registered.";
+      // TODO: print the result with ostream operator<<
+      return;
+    }
+
+    // TODO: verbose, not info.
+    if (completed) {
+      LOG(VERBOSE) << "ITaskListener::onComplete (request_id=" << request_id.request_id << ")";
+      listener->onComplete(request_id, result);
+    } else {
+      LOG(VERBOSE) << "ITaskListener::onProgress (request_id=" << request_id.request_id << ")";
+      listener->onProgress(request_id, result);
+    }
+  }
+
   bool OnAppLaunchEvent(const RequestId& request_id,
                         const AppLaunchEvent& event) {
     if (MaybeHandleFakeBehavior(request_id)) {
@@ -120,6 +177,89 @@ class IIorapImpl::Impl {
     return service_params_.event_manager_->OnAppLaunchEvent(request_id, event);
   }
 
+  bool OnDexOptEvent(const RequestId& request_id, const DexOptEvent& event) {
+    if (MaybeHandleFakeBehavior(request_id)) {
+      return true;
+    }
+
+    return service_params_.event_manager_->OnDexOptEvent(request_id, event);
+  }
+
+  bool  OnJobScheduledEvent(const RequestId& request_id,
+                            const JobScheduledEvent& event) {
+    if (MaybeHandleFakeBehavior(request_id)) {
+      return true;
+    }
+
+    return service_params_.event_manager_->OnJobScheduledEvent(request_id, event);
+  }
+
+  void Dump(/*borrow*/::android::Printer& printer,
+            const ::android::Vector<::android::String16>& args) {
+
+    if (args.size() == 0) {
+      service_params_.event_manager_->Dump(/*borrow*/printer);
+      return;
+    }
+
+    ::android::String16 arg_prev;
+    for (const ::android::String16& arg : args) {
+      bool unknown = false;
+      if (arg == ::android::String16("--all") || arg == ::android::String16("-a")) {
+        // using 'dumpsys' or 'bugreport' passes a single '-a' flag to this function.
+        service_params_.event_manager_->Dump(/*borrow*/printer);
+      } else if (arg == ::android::String16("--refresh-properties")) {
+        service_params_.event_manager_->RefreshSystemProperties(/*borrow*/printer);
+        printer.printLine("System properties refreshed.");
+      } else if (arg == ::android::String16("--compile-package")) {
+        // Intentionally left blank.
+      } else if (arg_prev == ::android::String16("--compile-package")) {
+        std::string package_name = String16ToStdString(arg);
+
+        if (!service_params_.event_manager_->CompilePackage(/*borrow*/printer, package_name)) {
+          printer.printFormatLine("Failed to compile package %s.", package_name.c_str());
+        } else {
+          printer.printFormatLine("Package %s compiled.", package_name.c_str());
+        }
+      } else if (arg == ::android::String16("--purge-package")) {
+        // Intentionally left blank.
+      } else if (arg_prev == ::android::String16("--purge-package")) {
+        std::string package_name = String16ToStdString(arg);
+
+        if (!service_params_.event_manager_->PurgePackage(/*borrow*/printer, package_name)) {
+          printer.printFormatLine("Failed to purge package %s.", package_name.c_str());
+        } else {
+          printer.printFormatLine("Package %s purged.", package_name.c_str());
+        }
+      } else {
+        unknown = true;
+      }
+
+      if (unknown && arg != ::android::String16("--help")) {
+        printer.printLine("Invalid arguments.");
+        printer.printLine("");
+
+        printer.printLine("Arguments were:");
+        for (const ::android::String16& arg16 : args) {
+          printer.printFormatLine("  '%s'", String16ToStdString(arg16).c_str());
+        }
+        printer.printLine("");
+      }
+
+      if (unknown || arg == ::android::String16("--help")) {
+        printer.printLine("Iorapd dumpsys commands:");
+        printer.printLine("  (none),--all,-a: Print state information for debugging iorapd.");
+        printer.printLine("  --help: Display this help menu");
+        printer.printLine("  --compile-package <name>: Compile single package on device");
+        printer.printLine("  --purge-package <name>: Delete database entries/files for package");
+        printer.printLine("  --refresh-properties: Refresh system properties");
+        return;
+      }
+
+      arg_prev = arg;
+    }
+  }
+
   void HandleFakeBehavior(const RequestId& request_id) {
     DCHECK(service_params_.fake_);
 
@@ -141,11 +281,36 @@ class IIorapImpl::Impl {
 
   ::android::sp<ITaskListener> listener_;
 
-  Impl(ServiceParams p) : service_params_{std::move(p)} {
+  Impl(ServiceParams p) : service_params_{std::move(p)}, event_manager_callbacks_{new EventManagerTaskCallbacks{this}} {
     CHECK(service_params_.event_manager_ != nullptr);
+
+    service_params_.event_manager_->SetTaskResultCallbacks(
+      std::static_pointer_cast<manager::TaskResultCallbacks>(event_manager_callbacks_));
+
+    // Init the package change observer.
+    package_manager_ = PackageManagerRemote::Create();
+
+    if (package_manager_ == nullptr) {
+      LOG(FATAL) << "Failed to get package manager service in IIorapImpl::Impl";
+      return;
+    }
+
+    package_change_observer_ =
+        new PackageChangeObserver(service_params_.event_manager_);
+    package_manager_death_recipient_ =
+        new PackageManagerDeathRecipient(package_manager_, package_change_observer_);
+
+    package_manager_->RegisterPackageChangeObserver(package_change_observer_);
+    package_manager_->
+        RegisterPackageManagerDeathRecipient(package_manager_death_recipient_);
+
   }
 
   ServiceParams service_params_;
+  std::shared_ptr<EventManagerTaskCallbacks> event_manager_callbacks_;
+  android::sp<PackageChangeObserver> package_change_observer_;
+  android::sp<PackageManagerDeathRecipient> package_manager_death_recipient_;
+  std::shared_ptr<PackageManagerRemote> package_manager_;
 };
 
 using Impl = IIorapImpl::Impl;
@@ -203,9 +368,21 @@ bool IIorapImpl::Start(std::shared_ptr<manager::EventManager> event_manager) {
   // Release edge synchronizes-with the top of this function.
   s_service_started_.store(true);
 
+  // TODO: IIRC thread-start(t1) synchronizes-with t1.main, so we should be able
+  // to delete the majority of atomics for any pre-thread-start initialization...
+
   return true;
 }
 
+::android::status_t IIorapImpl::dump(int fd, const ::android::Vector<::android::String16>& args) {
+
+  ::android::FdPrinter printer{fd};
+
+  impl_->Dump(printer, args);
+
+  return ::android::NO_ERROR;
+}
+
 namespace {
 
 #define MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id) \
@@ -260,6 +437,48 @@ Status SendArgs(const char* function_name,
 }
 
 template <typename ... Args>
+Status SendArgs(const char* function_name,
+                Impl* self,
+                const RequestId& request_id,
+                const DexOptEvent& event) {
+  DCHECK_EQ(std::string(function_name), "onDexOptEvent");
+  LOG(VERBOSE) << "IIorap::onDexOptEvent";
+
+  MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id);
+
+  if (self->OnDexOptEvent(request_id, event)) {
+    return Status::ok();
+  } else {
+    return Status::fromStatusT(::android::BAD_VALUE);
+  }
+}
+
+template <typename ... Args>
+Status SendArgs(const char* function_name,
+                Impl* self,
+                const RequestId& request_id,
+                const JobScheduledEvent& event) {
+  DCHECK_EQ(std::string(function_name), "onJobScheduledEvent");
+  LOG(VERBOSE) << "IIorap::onJobScheduledEvent";
+
+  MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id);
+
+  if (self->OnJobScheduledEvent(request_id, event)) {
+    return Status::ok();
+  } else {
+    // TODO: I suppose this should write out an exception back,
+    // like a service-specific error or something.
+    //
+    // It depends on whether or not we even have any synchronous
+    // errors.
+    //
+    // Most of the work here is done async, so it should handle
+    // async callbacks.
+    return Status::fromStatusT(::android::BAD_VALUE);
+  }
+}
+
+template <typename ... Args>
 Status Send(const char* function_name, Args&&... args) {
   LOG(VERBOSE) << "IIorap::Send(" << function_name << ")";
 
diff --git a/src/binder/iiorap_impl.h b/src/binder/iiorap_impl.h
index 811497f..ae1422f 100644
--- a/src/binder/iiorap_impl.h
+++ b/src/binder/iiorap_impl.h
@@ -18,6 +18,8 @@
 #define IORAP_BINDER_IIORAP_IMPL_H
 
 #include "binder/iiorap_def.h"
+#include "binder/package_change_observer.h"
+#include "binder/package_manager_remote.h"
 #include "common/macros.h"
 
 #include "com/google/android/startop/iorap/BnIorap.h"
@@ -45,6 +47,9 @@ public:
   static bool Start(std::shared_ptr<iorap::manager::EventManager> event_manager);
   static constexpr const char* getServiceName() { return "iorapd"; };
 
+  virtual ::android::status_t dump(int fd,
+                                   const ::android::Vector<::android::String16>& args) override;
+
 // Join all parameter declarations by splitting each parameter with a comma.
 // Types are used fully.
 #define IIORAP_IMPL_ARGS(...) \
@@ -65,7 +70,6 @@ private:
 
   std::unique_ptr<Impl> impl_;
 };
-
 }
 }
 
diff --git a/src/binder/package_change_observer.cc b/src/binder/package_change_observer.cc
new file mode 100644
index 0000000..1907af8
--- /dev/null
+++ b/src/binder/package_change_observer.cc
@@ -0,0 +1,37 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "package_change_observer.h"
+#include "package_manager_remote.h"
+#include "manager/event_manager.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+
+namespace iorap::binder {
+
+PackageChangeObserver::PackageChangeObserver(
+    std::shared_ptr<iorap::manager::EventManager> event_manager) :
+  event_manager_(event_manager){}
+
+android::binder::Status PackageChangeObserver::onPackageChanged(
+  const android::content::pm::PackageChangeEvent& event) {
+  LOG(DEBUG) << "Received PackageChangeObserver::onPackageChanged";
+  if (event_manager_->OnPackageChanged(event)) {
+    return android::binder::Status::ok();
+  } else {
+    return android::binder::Status::fromStatusT(android::BAD_VALUE);
+  }
+}
+}  // namespace iorap::binder
diff --git a/src/binder/package_change_observer.h b/src/binder/package_change_observer.h
new file mode 100644
index 0000000..ac488ac
--- /dev/null
+++ b/src/binder/package_change_observer.h
@@ -0,0 +1,42 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_
+#define IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_
+
+#include "binder/iiorap_def.h"
+
+#include <android/content/pm/BnPackageChangeObserver.h>
+#include <android/content/pm/PackageChangeEvent.h>
+#include <android/content/pm/IPackageManagerNative.h>
+
+namespace iorap::manager {
+  class EventManager;
+};
+
+namespace iorap::binder {
+
+class PackageChangeObserver : public android::content::pm::BnPackageChangeObserver  {
+ public:
+  PackageChangeObserver(std::shared_ptr<iorap::manager::EventManager> event_manager);
+
+  // Callback when the package is changed.
+  android::binder::Status onPackageChanged(
+      const ::android::content::pm::PackageChangeEvent& event) override;
+ private:
+  std::shared_ptr<iorap::manager::EventManager> event_manager_;
+};
+}  // namespace iorap::binder
+
+#endif  // IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_
diff --git a/src/binder/package_manager_remote.cc b/src/binder/package_manager_remote.cc
new file mode 100644
index 0000000..daeb6af
--- /dev/null
+++ b/src/binder/package_manager_remote.cc
@@ -0,0 +1,194 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "package_manager_remote.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+
+#include <chrono>
+#include <thread>
+
+namespace iorap::binder {
+
+const constexpr int64_t kTimeoutMs = 60 * 1000; // 1 min
+const constexpr int64_t kIntervalMs = 1000; // 1 sec
+
+std::shared_ptr<PackageManagerRemote> PackageManagerRemote::Create() {
+  std::shared_ptr<PackageManagerRemote> package_manager =
+      std::make_shared<PackageManagerRemote>();
+  if (package_manager->ReconnectWithTimeout(kTimeoutMs)) {
+    return package_manager;
+  }
+  return nullptr;
+}
+
+android::sp<IPackageManager> PackageManagerRemote::GetPackageService() {
+  android::sp<android::IBinder> binder = android::defaultServiceManager()->getService(
+      android::String16{"package_native"});
+  if (binder == nullptr) {
+    LOG(ERROR) << "Cannot get package manager service!";
+    return nullptr;
+  }
+
+  return android::interface_cast<IPackageManager>(binder);
+}
+
+std::optional<int64_t> PackageManagerRemote::GetPackageVersion(
+    const std::string& package_name) {
+  int64_t version_code;
+  android::binder::Status status = InvokeRemote(
+      [this, &version_code, &package_name]() {
+        return package_service_->getVersionCodeForPackage(
+            android::String16(package_name.c_str()), /*out*/&version_code);
+      });
+  if (status.isOk()) {
+    return version_code;
+  } else {
+    LOG(WARNING) << "Failed to get version: "
+                 << status.toString8().c_str()
+                 << " for " << package_name
+                 << ". Retry to connect package manager service.";
+    return std::nullopt;
+  }
+}
+
+std::optional<VersionMap> PackageManagerRemote::GetPackageVersionMap() {
+  VersionMap package_version_map;
+  std::optional<std::vector<std::string>> packages = GetAllPackages();
+  if (!packages) {
+    LOG(DEBUG) << "Failed to get all packages. The package manager may be down.";
+    return std::nullopt;
+  }
+  LOG(DEBUG) << "PackageManagerRemote::GetPackageVersionMap: "
+             << packages->size()
+             << " packages are found.";
+
+  for (const std::string& package : *packages) {
+    std::optional<int64_t> version = GetPackageVersion(package);
+    if (!version) {
+      LOG(DEBUG) << "Cannot get version for " << package
+                 << "Package manager may be down";
+      return std::nullopt;
+    }
+    package_version_map[package] = *version;
+  }
+
+  return package_version_map;
+}
+
+std::optional<std::vector<std::string>> PackageManagerRemote::GetAllPackages() {
+  std::vector<std::string> packages;
+  android::binder::Status status = InvokeRemote(
+      [this, &packages]() {
+        return package_service_->getAllPackages(/*out*/&packages);
+      });
+
+  if (status.isOk()) {
+    return packages;
+  }
+
+  LOG(ERROR) << "Failed to get all packages: " << status.toString8().c_str();
+  return std::nullopt;
+
+}
+
+bool PackageManagerRemote::ReconnectWithTimeout(int64_t timeout_ms) {
+  int64_t count = 0;
+  package_service_ = nullptr;
+  std::chrono::duration interval = std::chrono::milliseconds(1000);
+  std::chrono::duration timeout = std::chrono::milliseconds(timeout_ms);
+
+  while (package_service_ == nullptr) {
+    LOG(WARNING) << "Reconnect to package manager service: " << ++count << " times";
+    package_service_ = GetPackageService();
+    std::this_thread::sleep_for(interval);
+    if (count * interval >= timeout) {
+      LOG(FATAL) << "Fail to create version map in "
+                 << timeout.count()
+                 << " milliseconds."
+                 << " Reason: Failed to connect to package manager service."
+                 << " Is system_server down?";
+      return false;
+    }
+  }
+
+  return true;
+}
+
+template <typename T>
+android::binder::Status PackageManagerRemote::InvokeRemote(T&& lambda) {
+  android::binder::Status status =
+      static_cast<android::binder::Status>(lambda());
+  if (status.isOk()) {
+    return status;
+  }
+
+  if (!ReconnectWithTimeout(kTimeoutMs)) {
+    return status;
+  }
+
+  return lambda();
+}
+
+void PackageManagerRemote::RegisterPackageChangeObserver(
+    android::sp<PackageChangeObserver> observer) {
+  LOG(DEBUG) << "Register package change observer.";
+  android::binder::Status status = InvokeRemote(
+      [this, &observer]() {
+        return package_service_->registerPackageChangeObserver(observer);
+      });
+
+  if (!status.isOk()) {
+    LOG(FATAL) << "Cannot register package change observer.";
+  }
+}
+
+void PackageManagerRemote::UnregisterPackageChangeObserver(
+    android::sp<PackageChangeObserver> observer) {
+  LOG(DEBUG) << "Unregister package change observer.";
+  android::binder::Status status = InvokeRemote(
+      [this, &observer]() {
+        return package_service_->unregisterPackageChangeObserver(observer);
+      });
+
+  if (!status.isOk()) {
+    LOG(WARNING) << "Cannot unregister package change observer.";
+  }
+}
+
+void PackageManagerRemote::RegisterPackageManagerDeathRecipient(
+    android::sp<PackageManagerDeathRecipient> death_recipient) {
+  LOG(DEBUG) << "Register package manager death recipient.";
+  android::status_t status =
+      android::IInterface::asBinder(package_service_.get())->linkToDeath(death_recipient);
+
+  if (status == android::OK) {
+    return;
+  }
+
+  if (!ReconnectWithTimeout(kTimeoutMs) ||
+      android::OK != android::IInterface::asBinder(
+          package_service_.get())->linkToDeath(death_recipient)) {
+    LOG(FATAL) << "Failed to register package manager death recipient.";
+  }
+}
+
+void PackageManagerDeathRecipient::binderDied(const android::wp<android::IBinder>& /* who */) {
+  LOG(DEBUG) << "PackageManagerDeathRecipient::binderDied try to re-register";
+  package_manager_->RegisterPackageChangeObserver(observer_);
+  package_manager_->
+      RegisterPackageManagerDeathRecipient(this);
+}
+}  // namespace iorap::package_manager
diff --git a/src/binder/package_manager_remote.h b/src/binder/package_manager_remote.h
new file mode 100644
index 0000000..9b0dcd9
--- /dev/null
+++ b/src/binder/package_manager_remote.h
@@ -0,0 +1,83 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_
+#define IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_
+
+#include "binder/package_change_observer.h"
+
+#include <android/content/pm/IPackageManagerNative.h>
+#include <binder/IServiceManager.h>
+
+#include <optional>
+#include <unordered_map>
+
+namespace iorap::binder {
+
+using IPackageManager = android::content::pm::IPackageManagerNative;
+// A map between package name and its version.
+using VersionMap = std::unordered_map<std::string, int64_t>;
+
+class PackageManagerRemote;
+
+class PackageManagerDeathRecipient : public android::IBinder::DeathRecipient {
+public:
+  PackageManagerDeathRecipient(std::shared_ptr<PackageManagerRemote> package_manager,
+                               android::sp<PackageChangeObserver> observer) :
+    package_manager_(package_manager), observer_(observer) {}
+  // android::IBinder::DeathRecipient override:
+  void binderDied(const android::wp<android::IBinder>& /* who */) override;
+
+private:
+  std::shared_ptr<PackageManagerRemote> package_manager_;
+
+  android::sp<PackageChangeObserver> observer_;
+};
+
+class PackageManagerRemote {
+ public:
+  static std::shared_ptr<PackageManagerRemote> Create();
+
+  // Gets the package version based on the package name.
+  std::optional<int64_t> GetPackageVersion(const std::string& package_name);
+
+  // Gets a map of package name and its version.
+  std::optional<VersionMap> GetPackageVersionMap();
+
+  void RegisterPackageChangeObserver(android::sp<PackageChangeObserver> observer);
+
+  void UnregisterPackageChangeObserver(android::sp<PackageChangeObserver> observer);
+
+  void RegisterPackageManagerDeathRecipient(
+      android::sp<PackageManagerDeathRecipient> death_recipient);
+
+ private:
+  template <typename T>
+  android::binder::Status InvokeRemote(T&& lambda);
+
+  // Reconnects to the package manager service.
+  // Retry until timeout.
+  bool ReconnectWithTimeout(int64_t timeout_ms);
+
+  // Gets the package manager service.
+  static android::sp<IPackageManager> GetPackageService();
+
+  // Gets all package names.
+  std::optional<std::vector<std::string>> GetAllPackages();
+
+  android::sp<IPackageManager> package_service_;
+};
+}  // namespace iorap::package_manager
+
+#endif  // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_
diff --git a/src/binder/package_version_map.cc b/src/binder/package_version_map.cc
new file mode 100644
index 0000000..784bb1b
--- /dev/null
+++ b/src/binder/package_version_map.cc
@@ -0,0 +1,117 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "package_version_map.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+
+namespace iorap::binder {
+
+std::shared_ptr<PackageVersionMap> PackageVersionMap::Create() {
+  std::shared_ptr<PackageManagerRemote> package_manager =
+      PackageManagerRemote::Create();
+  if (!package_manager) {
+    return std::make_shared<PackageVersionMap>();
+  }
+
+  std::optional<VersionMap> map = package_manager->GetPackageVersionMap();
+  return std::make_shared<PackageVersionMap>(package_manager, map);
+}
+
+void PackageVersionMap::UpdateAll() {std::lock_guard<std::mutex> lock(mutex_);
+  size_t old_size = version_map_->size();
+  std::optional<VersionMap> new_version_map =
+      package_manager_->GetPackageVersionMap();
+  if (!new_version_map) {
+    LOG(DEBUG) << "Failed to get the latest version map";
+    return;
+  }
+  version_map_ = std::move(new_version_map);
+  LOG(DEBUG) << "Update for version is done. The size is from " << old_size
+             << " to " << version_map_->size();
+}
+
+bool PackageVersionMap::Update(std::string package_name, int64_t version) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  if (!version_map_) {
+    LOG(DEBUG) << "The version map doesn't exist. "
+               << "The package manager may be down.";
+    return false;
+  }
+
+  VersionMap::iterator it = version_map_->find(package_name);
+  if (it == version_map_->end()) {
+    LOG(DEBUG) << "New installed package "
+               << package_name
+               << " with version "
+               << version;
+    (*version_map_)[package_name] = version;
+    return true;
+  }
+
+  if (it->second != version) {
+    LOG(DEBUG) << "New version package "
+               << package_name
+               << " with version "
+               << version;
+    (*version_map_)[package_name] = version;
+    return true;
+  }
+
+  LOG(DEBUG) << "Same version package "
+             << package_name
+             << " with version "
+             << version;
+  return false;
+}
+
+size_t PackageVersionMap::Size() {
+  if (!version_map_) {
+    LOG(DEBUG) << "The version map doesn't exist. "
+               << "The package manager may be down.";
+    return -1;
+  }
+  return version_map_->size();
+}
+
+std::optional<int64_t> PackageVersionMap::GetOrQueryPackageVersion(
+    const std::string& package_name) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  if (!version_map_) {
+    LOG(DEBUG) << "The version map doesn't exist. "
+               << "The package manager may be down.";
+    return std::nullopt;
+  }
+
+  VersionMap::iterator it = version_map_->find(package_name);
+  if (it == version_map_->end()) {
+    LOG(DEBUG) << "Cannot find version for: " << package_name
+                 << " in the hash table";
+    std::optional<int64_t> version =
+        package_manager_->GetPackageVersion(package_name);
+    if (version) {
+      LOG(VERBOSE) << "Find version for: " << package_name << " on the fly.";
+      (*version_map_)[package_name] = *version;
+      return *version;
+    } else {
+      LOG(ERROR) << "Cannot find version for: " << package_name
+                 << " on the fly.";
+      return -1;
+    }
+  }
+
+  return it->second;
+}
+}  // namespace iorap::binder
diff --git a/src/binder/package_version_map.h b/src/binder/package_version_map.h
new file mode 100644
index 0000000..f2048fe
--- /dev/null
+++ b/src/binder/package_version_map.h
@@ -0,0 +1,83 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_PACKAGE_VERSION_MAP_H_
+#define IORAP_SRC_PACKAGE_VERSION_MAP_H_
+
+#include <android/content/pm/IPackageManagerNative.h>
+#include <binder/IServiceManager.h>
+
+#include <optional>
+#include <unordered_map>
+
+#include "package_manager_remote.h"
+
+namespace iorap::binder {
+
+class PackageVersionMap {
+ public:
+  static std::shared_ptr<PackageVersionMap> Create();
+
+  PackageVersionMap(std::shared_ptr<PackageManagerRemote> package_manager,
+                    std::optional<VersionMap> version_map)
+      : package_manager_(package_manager),
+        version_map_(version_map) {}
+
+  PackageVersionMap()
+      : package_manager_(nullptr), version_map_(std::nullopt) {}
+
+  // Updates the version specified by 'package_name' to 'version'.
+  //
+  // Post-condition: Find(package_name) == version.
+  // * if the package is newly installed, insert and return true.
+  // * if the package version is changed, update the version to the
+  //   given one and return true.
+  // * otherwise, return false.
+  bool Update(std::string package_name, int64_t version);
+
+  void UpdateAll();
+
+  // Finds the version of the package in the hash table.
+  // -1 means the app is installed by unversioned.
+  // Empty means the app is not inside the RAM version map, maybe due to
+  // the app is newly installed.
+  std::optional<int64_t> Find(const std::string& package_name) {
+    VersionMap::iterator it = version_map_->find(package_name);
+    if (it == version_map_->end()) {
+      return std::nullopt;
+    }
+    return it->second;
+  }
+
+  size_t Size();
+
+  // Gets or queries the version for the package.
+  //
+  // The method firstly access the hash map in the RAM, which is built when
+  // iorapd starts. If the version is not in the map, it tries the query
+  // the package manager via IPC, with a cost of ~0.6ms.
+  //
+  // If no version can be found for some reason, return -1,
+  // because when an app has no version the package manager returns -1.
+  std::optional<int64_t> GetOrQueryPackageVersion(
+      const std::string& package_name);
+
+ private:
+  std::shared_ptr<PackageManagerRemote> package_manager_;
+  std::optional<VersionMap> version_map_;
+  std::mutex mutex_;
+};
+}  // namespace iorap::binder
+
+#endif  // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_
diff --git a/src/common/async_pool.h b/src/common/async_pool.h
new file mode 100644
index 0000000..51cd38c
--- /dev/null
+++ b/src/common/async_pool.h
@@ -0,0 +1,91 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_COMMON_ASYNC_POOL_H_
+#define IORAP_SRC_COMMON_ASYNC_POOL_H_
+
+#include <atomic>
+#include <vector>
+#include <deque>
+#include <future>
+
+#include <condition_variable>
+#include <future>
+
+namespace iorap::common {
+
+class AsyncPool {
+  std::atomic<bool> shutting_down_{false};
+  std::deque<std::future<void>> futures_;
+
+  std::mutex mutex_;
+  std::condition_variable cond_var_;
+
+ public:
+
+  // Any threads calling 'Join' should eventually unblock
+  // once all functors have run to completition.
+  void Shutdown() {
+    shutting_down_ = true;
+
+    cond_var_.notify_all();
+  }
+
+  // Block forever until Shutdown is called *and* all
+  // functors passed to 'LaunchAsync' have run to completition.
+  void Join() {
+    std::unique_lock<std::mutex> lock(mutex_);
+    while (true) {
+      // Pop all items eagerly
+      while (true) {
+        auto it = futures_.begin();
+        if (it == futures_.end()) {
+          break;
+        }
+
+        std::future<void> future = std::move(*it);
+        futures_.pop_front();
+
+        lock.unlock();  // do not stall callers of LaunchAsync
+        future.get();
+        lock.lock();
+      }
+
+      if (shutting_down_) {
+        break;
+      }
+
+      // Wait until we either get more items or ask to be shutdown.
+      cond_var_.wait(lock);
+    }
+  }
+
+  // Execute the functor 'u' in a new thread asynchronously.
+  // Using this spawns a new thread each time to immediately begin
+  // async execution.
+  template <typename T>
+  void LaunchAsync(T&& u) {
+    auto future = std::async(std::launch::async, std::forward<T>(u));
+
+    {
+      std::unique_lock<std::mutex> lock(mutex_);
+      futures_.push_back(std::move(future));
+    }
+    cond_var_.notify_one();
+  }
+};
+
+}  // namespace iorap::common
+
+#endif  // IORAP_SRC_COMMON_ASYNC_POOL_H_
diff --git a/src/common/cmd_utils.h b/src/common/cmd_utils.h
new file mode 100644
index 0000000..4494119
--- /dev/null
+++ b/src/common/cmd_utils.h
@@ -0,0 +1,170 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_COMMON_CMD_UTILS_H_
+#define IORAP_SRC_COMMON_CMD_UTILS_H_
+
+#include <android-base/parsebool.h>
+#include <android-base/properties.h>
+
+#include <iostream>
+#include <sstream>
+#include <optional>
+#include <vector>
+
+namespace iorap::common {
+// Create execve-compatible argv.
+// The lifetime is tied to that of vector.
+inline std::unique_ptr<const char*[]> VecToArgv(const char* program_name,
+                                              const std::vector<std::string>& vector) {
+  // include program name in argv[0]
+  // include a NULL sentinel in the end.
+  std::unique_ptr<const char*[]> ptr{new const char*[vector.size() + 2]};
+
+  // program name
+  ptr[0] = program_name;
+
+  // all the argv
+  for (size_t i = 0; i < vector.size(); ++i) {
+    ptr[i+1] = vector[i].c_str();
+  }
+
+  // null sentinel
+  ptr[vector.size() + 1] = nullptr;
+
+  return ptr;
+}
+
+// Appends an args to the argv.
+template <class T>
+void AppendArgs(std::vector<std::string>& argv,
+                const T& value) {
+  std::stringstream ss;
+  ss << value;
+  argv.push_back(ss.str());
+}
+
+// Appends an args to the argv.
+template <class T, class T2>
+void AppendArgs(std::vector<std::string>& argv,
+                const T& value,
+                const T2& value2) {
+  AppendArgs(argv, value);
+  AppendArgs(argv, value2);
+}
+
+// Appends a named argument to the argv.
+//
+// For example if <name> is "--property" and <value> is int(200):
+// the string "--property=200" is appended to the argv.
+template <class T, class T2>
+void AppendNamedArg(std::vector<std::string>& argv,
+                    const T& name,
+                    const T2& value) {
+  std::stringstream ss;
+  ss << name;
+  ss << "=";
+  ss << value;
+
+  argv.push_back(ss.str());
+}
+
+// Appends args from a vector to the argv repeatedly to argv.
+//
+// For example, if <args> is "--timestamp" and <values> is [100, 200].
+// The "--timestamp 100" and "--timestamp 200" are appended.
+template <class T>
+void AppendArgsRepeatedly(std::vector<std::string>& argv,
+                          std::string args,
+                          const std::vector<T>& values) {
+  for (const T& v : values) {
+     AppendArgs(argv, args, v);
+  }
+}
+
+// Appends args from a vector to the argv repeatedly to argv.
+//
+// For example, if values is [input1.pb, input2.pb],
+// then the "input1.pb" and "input2.pb" are appended.
+template <class T>
+void AppendArgsRepeatedly(std::vector<std::string>& argv,
+                          const std::vector<T>& values) {
+  for (const T& v : values) {
+     AppendArgs(argv, v);
+  }
+}
+
+// Appends a named argument to the argv repeatedly with different values.
+//
+// For example if <name> is "--property" and <value> is [int(200), int(400)]:
+// the strings "--property=200" and "--property=400" are both appended to the argv.
+template <class T, class T2>
+void AppendNamedArgRepeatedly(std::vector<std::string>& argv,
+                              const T& name,
+                              const std::vector<T2>& values) {
+  for (const T2& v :values) {
+    AppendNamedArg(argv, name, v);
+  }
+}
+
+// Get the value of the property.
+// Firstly, try to find the environment variable. If it does not exist,
+// try to get the property. If neither, use the default value..
+//
+// For example, for prop foo.bar.baz, it will first check for
+// FOO_BAR_BAZ environment variable.
+inline std::string GetEnvOrProperty(const std::string& prop, const std::string& default_val) {
+  std::string env_str = prop;
+  // a.b.c -> a_b_c
+  std::replace(env_str.begin(), env_str.end(), '.', '_');
+  // a_b_c -> A_B_C
+  std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper);
+  char *env = getenv(env_str.c_str());
+  if (env) {
+    return std::string(env);
+  }
+  return ::android::base::GetProperty(prop, default_val);
+}
+
+// Get the boolean value of the property.
+// Firstly, try to find the environment variable. If it does not exist,
+// try to get the property. If neither, use the default value..
+//
+// For example, for prop foo.bar.baz, it will first check for
+// FOO_BAR_BAZ environment variable.
+inline bool GetBoolEnvOrProperty(const std::string& prop, bool default_val) {
+  std::string env_str = prop;
+  // a.b.c -> a_b_c
+  std::replace(env_str.begin(), env_str.end(), '.', '_');
+  // a_b_c -> A_B_C
+  std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper);
+  char *env = getenv(env_str.c_str());
+  if (env) {
+    using ::android::base::ParseBoolResult;
+
+    switch (::android::base::ParseBool(env)) {
+      case ParseBoolResult::kError:
+        break;
+      case ParseBoolResult::kFalse:
+        return false;
+      case ParseBoolResult::kTrue:
+        return true;
+    }
+  }
+  return ::android::base::GetBoolProperty(prop, default_val);
+}
+
+}   // namespace iorap::common
+
+#endif  // IORAP_SRC_COMMON_CMD_UTILS_H_
diff --git a/src/common/expected.h b/src/common/expected.h
index 3d8eef7..c58e63f 100644
--- a/src/common/expected.h
+++ b/src/common/expected.h
@@ -292,7 +292,13 @@ struct expected {
     return data_.value();
   }
 
-  // TODO: arrow operator?
+  constexpr const T* _Nonnull operator->() const {
+    return &data_.value();
+  }
+
+  constexpr T* _Nonnull operator->() {
+    return &data_.value();
+  }
 
   constexpr T& value() & {
     CHECK(has_value());
diff --git a/src/common/loggers.h b/src/common/loggers.h
new file mode 100644
index 0000000..1dbb2dc
--- /dev/null
+++ b/src/common/loggers.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef IORAP_SRC_COMMON_LOGGERS
+#define IORAP_SRC_COMMON_LOGGERS
+
+#include <android-base/logging.h>
+
+namespace iorap {
+namespace common {
+
+// Log to both Stderr and Logd for convenience when running this from the command line.
+class StderrAndLogdLogger {
+ public:
+  explicit StderrAndLogdLogger(android::base::LogId default_log_id = android::base::MAIN)
+#ifdef __ANDROID__
+      : logd_(default_log_id)
+#endif
+  {
+  }
+
+  void operator()(::android::base::LogId id,
+                  ::android::base::LogSeverity sev,
+                  const char* tag,
+                  const char* file,
+                  unsigned int line,
+                  const char* message) {
+#ifdef __ANDROID__
+    logd_(id, sev, tag, file, line, message);
+#endif
+    StderrLogger(id, sev, tag, file, line, message);
+  }
+
+ private:
+#ifdef __ANDROID__
+  ::android::base::LogdLogger logd_;
+#endif
+};
+
+}  // namespace iorap
+}  // namespace common
+
+#endif
diff --git a/src/common/printer.h b/src/common/printer.h
new file mode 100644
index 0000000..272e459
--- /dev/null
+++ b/src/common/printer.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef IORAP_UTILS_PRINTER_H_
+#define IORAP_UTILS_PRINTER_H_
+
+#include <utils/Printer.h>
+
+#include <string.h>
+
+namespace iorap::common {
+
+class StderrLogPrinter : public ::android::Printer {
+public:
+    // Create a printer using the specified logcat and log priority
+    // - Unless ignoreBlankLines is false, print blank lines to logcat
+    // (Note that the default ALOG behavior is to ignore blank lines)
+    StderrLogPrinter(const char* logtag,
+                     android_LogPriority priority = ANDROID_LOG_DEBUG,
+                     const char* prefix = nullptr,
+                     bool ignore_blank_lines = false)
+      : log_printer_{logtag, priority, prefix, ignore_blank_lines} {
+      logtag_ = logtag;
+      prefix_ = prefix;
+      ignore_blank_lines_ = ignore_blank_lines;
+    }
+
+    // Print the specified line to logcat. No \n at the end is necessary.
+    virtual void printLine(const char* string) override {
+      if (ignore_blank_lines_ && strlen(string) == 0) {
+        return;
+      }
+      std::cerr << logtag_ << ": ";
+      if (prefix_ != nullptr) {
+        std::cerr << prefix_;
+      }
+      std::cerr << string << std::endl;
+      log_printer_.printLine(string);
+    }
+ private:
+    ::android::LogPrinter log_printer_;
+    const char* logtag_;
+    const char* prefix_;
+    bool ignore_blank_lines_;
+};
+
+}  // namespace iorap::common
+
+#endif  // IORAP_UTILS_PRINTER_H_
diff --git a/src/common/rx_async.h b/src/common/rx_async.h
new file mode 100644
index 0000000..d16f5ae
--- /dev/null
+++ b/src/common/rx_async.h
@@ -0,0 +1,77 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_COMMON_RX_ASYNC_H_
+#define IORAP_SRC_COMMON_RX_ASYNC_H_
+
+#include "common/async_pool.h"
+
+#include <rxcpp/rx.hpp>
+
+namespace iorap::common {
+
+// Helper functions to operate with rx chains asynchronously.
+class RxAsync {
+ public:
+  // Subscribe to the observable on a new thread asynchronously.
+  // If no observe_on/subscribe_on is used, the chain will execute
+  // on that new thread.
+  //
+  // Returns the composite_subscription which can be used to
+  // unsubscribe from if we want to abort the chain early.
+  template <typename T, typename U>
+  static rxcpp::composite_subscription SubscribeAsync(
+      AsyncPool& async_pool,
+      T&& observable,
+      U&& subscriber) {
+    rxcpp::composite_subscription subscription;
+
+    async_pool.LaunchAsync([subscription,  // safe copy: ref-counted
+                            observable=std::forward<T>(observable),
+                            subscriber=std::forward<U>(subscriber)]() mutable {
+      observable
+        .as_blocking()
+        .subscribe(subscription,
+                   std::forward<decltype(subscriber)>(subscriber));
+    });
+
+    return subscription;
+  }
+
+  template <typename T, typename U, typename E>
+  static rxcpp::composite_subscription SubscribeAsync(
+      AsyncPool& async_pool,
+      T&& observable,
+      U&& on_next,
+      E&& on_error) {
+    rxcpp::composite_subscription subscription;
+
+    async_pool.LaunchAsync([subscription,  // safe copy: ref-counted
+                            observable=std::forward<T>(observable),
+                            on_next=std::forward<U>(on_next),
+                            on_error=std::forward<E>(on_error)]() mutable {
+      observable
+        .as_blocking()
+        .subscribe(subscription,
+                   std::forward<decltype(on_next)>(on_next),
+                   std::forward<decltype(on_error)>(on_error));
+    });
+
+    return subscription;
+  }
+};
+
+}   // namespace iorap::common
+
+#endif  // IORAP_SRC_COMMON_RX_ASYNC_H_
diff --git a/src/common/trace.h b/src/common/trace.h
new file mode 100644
index 0000000..a07d7bc
--- /dev/null
+++ b/src/common/trace.h
@@ -0,0 +1,48 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_COMMON_TRACE_H_
+#define IORAP_COMMON_TRACE_H_
+
+#include <cutils/trace.h>
+
+#include <cstdio>
+#include <sstream>
+
+namespace iorap {
+
+// TODO: refactor into utils/Trace.h
+
+class ScopedFormatTrace {
+ public:
+  template <typename ... Args>
+  ScopedFormatTrace(uint64_t tag, const char* fmt, Args&&... args) : tag_{tag} {
+    char buffer[1024];
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-security"
+    snprintf(buffer, sizeof(buffer), fmt, args...);
+#pragma GCC diagnostic pop
+    atrace_begin(tag, buffer);
+  }
+
+  ~ScopedFormatTrace() {
+    atrace_end(tag_);
+  }
+ private:
+  uint64_t tag_;
+};
+
+}  // namespace iorap
+
+#endif  // IORAP_COMMON_TRACE_H_
diff --git a/src/compiler/compiler.cc b/src/compiler/compiler.cc
new file mode 100644
index 0000000..7d0e624
--- /dev/null
+++ b/src/compiler/compiler.cc
@@ -0,0 +1,949 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "compiler/compiler.h"
+
+#include "common/debug.h"
+#include "common/expected.h"
+
+#include "perfetto/rx_producer.h"  // TODO: refactor BinaryWireProtobuf to separate header.
+#include "inode2filename/inode.h"
+#include "inode2filename/search_directories.h"
+#include "serialize/protobuf_io.h"
+
+#include <android-base/unique_fd.h>
+#include <android-base/parseint.h>
+#include <android-base/file.h>
+
+#include <perfetto/trace/trace.pb.h>  // ::perfetto::protos::Trace
+#include <perfetto/trace/trace_packet.pb.h>  // ::perfetto::protos::TracePacket
+
+#include "rxcpp/rx.hpp"
+#include <iostream>
+#include <fstream>
+#include <optional>
+#include <utility>
+#include <regex>
+
+#include <sched.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <syscall.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+namespace iorap::compiler {
+
+using Inode = iorap::inode2filename::Inode;
+using InodeResult = iorap::inode2filename::InodeResult;
+using SearchDirectories = iorap::inode2filename::SearchDirectories;
+
+template <typename T>
+using ProtobufPtr = iorap::perfetto::ProtobufPtr<T>;
+
+struct PerfettoTraceProtoInfo {
+  /* The perfetto trace proto. */
+  ::iorap::perfetto::PerfettoTraceProto proto;
+  /*
+   * The timestamp limit of the trace.
+   * It's used to truncate the trace file.
+   */
+  uint64_t timestamp_limit_ns;
+};
+
+struct PerfettoTracePtrInfo {
+  /* Deserialized protobuf data containing the perfetto trace. */
+  ProtobufPtr<::perfetto::protos::Trace> trace_ptr;
+  /*
+   * The timestamp limit of the trace.
+   * It's used to truncate the trace file.
+   */
+  uint64_t timestamp_limit_ns;
+};
+
+// Attempt to read protobufs from the filenames.
+// Emits one (or none) protobuf for each filename, in the same order as the filenames.
+// On any errors, the items are dropped (errors are written to the error LOG).
+//
+// All work is done on the same Coordinator as the Subscriber.
+template <typename ProtoT /*extends MessageLite*/>
+auto/*observable<PerfettoTracePtrInfo>*/ ReadProtosFromFileNames(
+    rxcpp::observable<CompilationInput> file_infos) {
+  using BinaryWireProtoT = ::iorap::perfetto::PerfettoTraceProto;
+
+  return file_infos
+    .map([](const CompilationInput& file_info) ->
+         std::optional<PerfettoTraceProtoInfo> {
+      LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << file_info.filename
+                   << " TimeStampLimit "<< file_info.timestamp_limit_ns << " [begin]";
+      std::optional<BinaryWireProtoT> maybe_proto =
+          BinaryWireProtoT::ReadFullyFromFile(file_info.filename);
+      if (!maybe_proto) {
+        LOG(ERROR) << "Failed to read file: " << file_info.filename;
+        return std::nullopt;
+      }
+      return {{std::move(maybe_proto.value()), file_info.timestamp_limit_ns}};
+    })
+    .filter([](const std::optional<PerfettoTraceProtoInfo>& proto_info) {
+      return proto_info.has_value();
+    })
+    .map([](std::optional<PerfettoTraceProtoInfo>& proto_info) ->
+         PerfettoTraceProtoInfo {
+      return proto_info.value();
+    })  // TODO: refactor to something that flattens the optional, and logs in one operator.
+    .map([](PerfettoTraceProtoInfo& proto_info) ->
+         std::optional<PerfettoTracePtrInfo> {
+      std::optional<ProtobufPtr<ProtoT>> t = proto_info.proto.template MaybeUnserialize<ProtoT>();
+      if (!t) {
+        LOG(ERROR) << "Failed to parse protobuf: ";  // TODO: filename.
+        return std::nullopt;
+      }
+      return {{std::move(t.value()), proto_info.timestamp_limit_ns}};
+    })
+    .filter([](const std::optional<PerfettoTracePtrInfo>& trace_info) {
+      return trace_info.has_value();
+    })
+    .map([](std::optional<PerfettoTracePtrInfo>& trace_info) ->
+         PerfettoTracePtrInfo {
+      LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]";
+      return trace_info.value();
+      // TODO: protobufs have no move constructor. this might be inefficient?
+    });
+
+/*
+  return filenames
+    .map([](const std::string& filename) {
+      LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << filename << " [begin]";
+      std::optional<BinaryWireProtoT> maybe_proto =
+          BinaryWireProtoT::ReadFullyFromFile(filename);
+      if (!maybe_proto) {
+        LOG(ERROR) << "Failed to read file: " << filename;
+      }
+
+      std::unique_ptr<BinaryWireProtoT> ptr;
+      if (maybe_proto) {
+        ptr.reset(new BinaryWireProtoT{std::move(*maybe_proto)});
+      }
+      return ptr;
+    })
+    .filter([](const std::unique_ptr<BinaryWireProtoT>& proto) {
+      return proto != nullptr;
+    })
+    .map([](std::unique_ptr<BinaryWireProtoT>& proto) {
+      std::optional<ProtoT> t = proto->template MaybeUnserialize<ProtoT>();
+      if (!t) {
+        LOG(ERROR) << "Failed to parse protobuf: ";  // TODO: filename.
+      }
+      return t;
+    })
+    .filter([](const std::optional<ProtoT>& proto) {
+      return proto.has_value();
+    })
+    .map([](std::optional<ProtoT> proto) -> ProtoT {
+      LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]";
+      return std::move(proto.value());
+      // TODO: protobufs have no move constructor. this might be inefficient?
+    });
+    */
+}
+
+auto/*observable<PerfettoTracePtrInfo>*/ ReadPerfettoTraceProtos(
+    std::vector<CompilationInput> file_infos) {
+  auto filename_obs = rxcpp::observable<>::iterate(std::move(file_infos));
+  rxcpp::observable<PerfettoTracePtrInfo> obs =
+      ReadProtosFromFileNames<::perfetto::protos::Trace>(std::move(filename_obs));
+  return obs;
+}
+
+// A flattened data representation of an MmFileMap*FtraceEvent.
+// This representation is used for streaming processing.
+//
+// Note: Perfetto applies a 'union' over all possible fields on all possible devices
+// (and uses the max sizeof per-field).
+//
+// Since all protobuf fields are optional, fields not present on a particular device are always
+// null
+struct PageCacheFtraceEvent {
+  /*
+   * Ftrace buffer-specific
+   */
+  uint32_t cpu;  // e.g. 0-7 for the cpu core number.
+
+  /*
+   * Ftrace-event general data
+   */
+
+  // Nanoseconds since an epoch.
+  // Epoch is configurable by writing into trace_clock.
+  // By default this timestamp is CPU local.
+  uint64_t timestamp;
+  // Kernel pid (do not confuse with userspace pid aka tgid)
+  uint32_t pid;
+
+  // Tagged by our code while parsing the ftraces:
+  uint64_t timestamp_relative;  // timestamp relative to first ftrace within a Trace protobuf.
+  bool add_to_page_cache;  // AddToPageCache=true, DeleteFromPageCache=false.
+
+  /*
+   * mm_filemap-specific data
+   *
+   * Fields are common:
+   * - MmFilemapAddToPageCacheFtraceEvent
+   * - MmFilemapDeleteFromPageCacheFtraceEvent
+   */
+  uint64_t pfn;    // page frame number (physical) - null on some devices, e.g. marlin
+  uint64_t i_ino;  // inode number (use in conjunction with s_dev)
+  uint64_t index;  // offset into file: this is a multiple of the page size (usually 4096).
+  uint64_t s_dev;  // (dev_t) device number
+  uint64_t page;   // struct page*. - null on some devices, e.g. blueline.
+
+  Inode inode() const {
+    return Inode::FromDeviceAndInode(static_cast<dev_t>(s_dev),
+                                     static_cast<ino_t>(i_ino));
+  }
+};
+
+std::ostream& operator<<(std::ostream& os, const PageCacheFtraceEvent& e) {
+  os << "{";
+  os << "cpu:" << e.cpu << ",";
+  os << "timestamp:" << e.timestamp << ",";
+  os << "pid:" << e.pid << ",";
+  os << "timestamp_relative:" << e.timestamp_relative << ",";
+  os << "add_to_page_cache:" << e.add_to_page_cache << ",";
+  os << "pfn:" << e.pfn << ",";
+  os << "i_ino:" << e.i_ino << ",";
+  os << "index:" << e.index << ",";
+  os << "s_dev:" << e.s_dev << ",";
+  os << "page:" << e.page;
+  os << "}";
+
+  return os;
+}
+
+/*
+ * Gets the start timestamp.
+ *
+ * It is the minimium timestamp.
+ */
+std::optional<uint64_t> GetStartTimestamp(const ::perfetto::protos::Trace& trace) {
+  std::optional<uint64_t> timestamp_relative_start;
+  // Traverse each timestamp to get the minimium one.
+  for (const ::perfetto::protos::TracePacket& packet : trace.packet()) {
+    if (packet.has_timestamp()) {
+      timestamp_relative_start = timestamp_relative_start?
+          std::min(*timestamp_relative_start, packet.timestamp()) : packet.timestamp();
+    }
+    if (!packet.has_ftrace_events()) {
+      continue;
+    }
+    const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle =
+        packet.ftrace_events();
+    for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) {
+      if (event.has_timestamp()) {
+        timestamp_relative_start = timestamp_relative_start?
+            std::min(*timestamp_relative_start, event.timestamp()) : event.timestamp();
+      }
+    }
+  }
+  return timestamp_relative_start;
+}
+
+/*
+ * sample blueline output:
+ *
+ * $ adb shell cat /d/tracing/events/filemap/mm_filemap_add_to_page_cache/format
+ *
+ * name: mm_filemap_add_to_page_cache
+ * ID: 178
+ * format:
+ * 	field:unsigned short common_type;	offset:0;	size:2;	signed:0;
+ * 	field:unsigned char common_flags;	offset:2;	size:1;	signed:0;
+ * 	field:unsigned char common_preempt_count;	offset:3;	size:1;	signed:0;
+ * 	field:int common_pid;	offset:4;	size:4;	signed:1;
+ *
+ * 	field:unsigned long pfn;	offset:8;	size:8;	signed:0;
+ * 	field:unsigned long i_ino;	offset:16;	size:8;	signed:0;
+ * 	field:unsigned long index;	offset:24;	size:8;	signed:0;
+ * 	field:dev_t s_dev;	offset:32;	size:4;	signed:0;
+ *
+ * print fmt: "dev %d:%d ino %lx page=%p pfn=%lu ofs=%lu", ((unsigned int) ((REC->s_dev) >> 20)),
+ *            ((unsigned int) ((REC->s_dev) & ((1U << 20) - 1))), REC->i_ino,
+ *             (((struct page *)(((0xffffffffffffffffUL) - ((1UL) << ((39) - 1)) + 1) -
+ *                 ((1UL) << ((39) - 12 - 1 + 6))) - (memstart_addr >> 12)) + (REC->pfn)),
+ *            REC->pfn, REC->index << 12
+ */
+
+auto /*observable<PageCacheFtraceEvent>*/ SelectPageCacheFtraceEvents(
+    PerfettoTracePtrInfo trace_info) {
+  const ::perfetto::protos::Trace& trace = *(trace_info.trace_ptr);
+
+  constexpr bool kDebugFunction = true;
+
+  return rxcpp::observable<>::create<PageCacheFtraceEvent>(
+      [trace=std::move(trace), timestamp_limit_ns=trace_info.timestamp_limit_ns]
+      (rxcpp::subscriber<PageCacheFtraceEvent> sub) {
+    uint64_t timestamp = 0;
+    uint64_t timestamp_relative = 0;
+
+    std::optional<uint64_t> timestamp_relative_start = GetStartTimestamp(trace);
+    uint32_t cpu = 0;
+    uint32_t pid = 0;
+    bool add_to_page_cache = true;
+
+    auto on_next_page_cache_event = [&](const auto& mm_event) {
+      PageCacheFtraceEvent out;
+      out.timestamp = timestamp;
+      out.cpu = cpu;
+      out.pid = pid;
+
+      out.timestamp_relative = timestamp_relative;
+      out.add_to_page_cache = add_to_page_cache;
+
+      out.pfn = mm_event.pfn();
+      out.i_ino = mm_event.i_ino();
+      out.index = mm_event.index();
+      out.s_dev = mm_event.s_dev();
+      out.page = mm_event.page();
+
+      sub.on_next(std::move(out));
+    };
+
+    for (const ::perfetto::protos::TracePacket& packet : trace.packet()) {
+      // Break out of all loops if we are unsubscribed.
+      if (!sub.is_subscribed()) {
+        if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents unsubscribe";
+        return;
+      }
+
+      if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents TracePacket";
+
+      if (packet.has_timestamp()) {
+        timestamp_relative_start = timestamp_relative_start.value_or(packet.timestamp());
+        timestamp = packet.timestamp();  // XX: should we call 'has_timestamp()' ?
+      } else {
+        timestamp = 0;
+      }
+
+      if (packet.has_ftrace_events()) {
+        const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle =
+            packet.ftrace_events();
+
+        cpu = ftrace_event_bundle.cpu();  // XX: has_cpu ?
+
+        for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) {
+          // Break out of all loops if we are unsubscribed.
+          if (!sub.is_subscribed()) {
+            return;
+          }
+
+          if (event.has_timestamp()) {
+            timestamp = event.timestamp();
+            if(timestamp > timestamp_limit_ns) {
+              LOG(VERBOSE) << "The timestamp is " << timestamp <<
+                           ", which exceeds the limit "<< timestamp_limit_ns;
+              continue;
+            }
+          } else {
+            DCHECK(packet.has_timestamp() == false)
+                << "Timestamp in outer packet but not inner packet";
+            // XX: use timestamp from the perfetto TracePacket ???
+            // REVIEWERS: not sure if this is ok, does it use the same clock source and
+            // is the packet data going to be the same clock sample as the Ftrace event?
+          }
+
+          if (timestamp_relative_start){
+            timestamp_relative = timestamp - *timestamp_relative_start;
+          } else {
+            timestamp_relative = 0;
+          }
+
+          pid = event.pid();  // XX: has_pid ?
+
+          if (event.has_mm_filemap_add_to_page_cache()) {
+            add_to_page_cache = true;
+
+            const ::perfetto::protos::MmFilemapAddToPageCacheFtraceEvent& mm_event =
+                event.mm_filemap_add_to_page_cache();
+
+            on_next_page_cache_event(mm_event);
+          } else if (event.has_mm_filemap_delete_from_page_cache()) {
+            add_to_page_cache = false;
+
+            const ::perfetto::protos::MmFilemapDeleteFromPageCacheFtraceEvent& mm_event =
+                event.mm_filemap_delete_from_page_cache();
+
+            on_next_page_cache_event(mm_event);
+          }
+        }
+      } else {
+        if (kDebugFunction) {
+          LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents no ftrace event bundle";
+        }
+      }
+    }
+
+    if (kDebugFunction) {
+      LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents#on_completed";
+    }
+
+    // Let subscriber know there are no more items.
+    sub.on_completed();
+  });
+}
+
+auto /*observable<Inode*/ SelectDistinctInodesFromTraces(
+    rxcpp::observable<PerfettoTracePtrInfo> traces) {
+  // Emit only unique (s_dev, i_ino) pairs from all Trace protos.
+  auto obs = traces
+    .flat_map([](PerfettoTracePtrInfo trace) {
+      rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace));
+      // FIXME: dont check this in
+      // return obs;
+      //return obs.take(100);   // for faster development
+      return obs;
+    })  // TODO: Upstream bug? using []()::perfetto::protos::Trace&) causes a compilation error.
+    .map([](const PageCacheFtraceEvent& event) -> Inode {
+      return Inode::FromDeviceAndInode(static_cast<dev_t>(event.s_dev),
+                                       static_cast<ino_t>(event.i_ino));
+    })
+    .tap([](const Inode& inode) {
+      LOG(VERBOSE) << "SelectDistinctInodesFromTraces (pre-distinct): " << inode;
+    })
+    .distinct()  // observable<Inode>*/
+    ;
+
+  return obs;
+}
+// TODO: static assert checks for convertible return values.
+
+auto/*observable<InodeResult>*/ ResolveInodesToFileNames(
+    rxcpp::observable<Inode> inodes,
+    inode2filename::InodeResolverDependencies dependencies) {
+  std::shared_ptr<inode2filename::InodeResolver> inode_resolver =
+      inode2filename::InodeResolver::Create(std::move(dependencies));
+  return inode_resolver->FindFilenamesFromInodes(std::move(inodes));
+}
+
+using InodeMap = std::unordered_map<Inode, std::string /*filename*/>;
+auto /*just observable<InodeMap>*/ ReduceResolvedInodesToMap(
+      rxcpp::observable<InodeResult> inode_results) {
+  return inode_results.reduce(
+    InodeMap{},
+    [](InodeMap m, InodeResult result) {
+      if (result) {
+        LOG(VERBOSE) << "compiler::ReduceResolvedInodesToMap insert " << result;
+        m.insert({std::move(result.inode), std::move(result.data.value())});
+      } else {
+        // TODO: side stats for how many of these are failed to resolve?
+        LOG(WARNING) << "compiler: Failed to resolve inode, " << result;
+      }
+      return m;
+    },
+    [](InodeMap m) {
+      return m;  // TODO: use an identity function
+    }); // emits exactly 1 InodeMap value.
+}
+
+struct ResolvedPageCacheFtraceEvent {
+  std::string filename;
+  PageCacheFtraceEvent event;
+};
+
+std::ostream& operator<<(std::ostream& os, const ResolvedPageCacheFtraceEvent& e) {
+  os << "{";
+  os << "filename:\"" << e.filename << "\",";
+  os << e.event;
+  os << "}";
+
+  return os;
+}
+
+struct CombinedState {
+  CombinedState() = default;
+  explicit CombinedState(InodeMap inode_map) : inode_map{std::move(inode_map)} {}
+  explicit CombinedState(PageCacheFtraceEvent event) : ftrace_event{std::move(event)} {}
+
+  CombinedState(InodeMap inode_map, PageCacheFtraceEvent event)
+    : inode_map(std::move(inode_map)),
+      ftrace_event{std::move(event)} {}
+
+  std::optional<InodeMap> inode_map;
+  std::optional<PageCacheFtraceEvent> ftrace_event;
+
+  bool HasAll() const {
+    return inode_map.has_value() && ftrace_event.has_value();
+  }
+
+  const InodeMap& GetInodeMap() const {
+    DCHECK(HasAll());
+    return inode_map.value();
+  }
+
+  InodeMap& GetInodeMap() {
+    DCHECK(HasAll());
+    return inode_map.value();
+  }
+
+  const PageCacheFtraceEvent& GetEvent() const {
+    DCHECK(HasAll());
+    return ftrace_event.value();
+  }
+
+  PageCacheFtraceEvent& GetEvent() {
+    DCHECK(HasAll());
+    return ftrace_event.value();
+  }
+
+  void Merge(CombinedState&& other) {
+    if (other.inode_map) {
+      inode_map = std::move(other.inode_map);
+    }
+    if (other.ftrace_event) {
+      ftrace_event = std::move(other.ftrace_event);
+    }
+  }
+};
+
+std::ostream& operator<<(std::ostream& os, const CombinedState& s) {
+  os << "CombinedState{inode_map:";
+  if (s.inode_map) {
+    os << "|sz=" << (s.inode_map.value().size()) << "|";
+  } else {
+    os << "(null)";
+  }
+  os << ",event:";
+  if (s.ftrace_event) {
+    //os << s.ftrace_event.value().timestamp << "ns";
+    os << s.ftrace_event.value();
+  } else {
+    os << "(null)";
+  }
+  os << "}";
+  return os;
+}
+
+auto/*observable<ResolvedPageCacheFtraceEvent>*/ ResolvePageCacheEntriesFromProtos(
+    rxcpp::observable<PerfettoTracePtrInfo> traces,
+    inode2filename::InodeResolverDependencies dependencies) {
+
+  // 1st chain = emits exactly 1 InodeMap.
+
+  // [proto, proto, proto...] -> [inode, inode, inode, ...]
+  auto/*observable<Inode>*/ distinct_inodes = SelectDistinctInodesFromTraces(traces);
+  rxcpp::observable<Inode> distinct_inodes_obs = distinct_inodes.as_dynamic();
+  // [inode, inode, inode, ...] -> [(inode, {filename|error}), ...]
+  auto/*observable<InodeResult>*/ inode_names = ResolveInodesToFileNames(distinct_inodes_obs,
+                                                                         std::move(dependencies));
+  // rxcpp has no 'join' operators, so do a manual join with concat.
+  auto/*observable<InodeMap>*/ inode_name_map = ReduceResolvedInodesToMap(inode_names);
+
+  // 2nd chain = emits all PageCacheFtraceEvent
+  auto/*observable<PageCacheFtraceEvent>*/ page_cache_ftrace_events = traces
+    .flat_map([](PerfettoTracePtrInfo trace) {
+      rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace));
+      return obs;
+    });
+
+  auto inode_name_map_precombine = inode_name_map
+    .map([](InodeMap inode_map) {
+      LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#inode_name_map_precombine ";
+      return CombinedState{std::move(inode_map)};
+    });
+
+  auto page_cache_ftrace_events_precombine = page_cache_ftrace_events
+    .map([](PageCacheFtraceEvent event) {
+      LOG(VERBOSE)
+          << "compiler::ResolvePageCacheEntriesFromProtos#page_cache_ftrace_events_precombine "
+          << event;
+      return CombinedState{std::move(event)};
+    });
+
+  // Combine 1st+2nd chain.
+  //
+  // concat subscribes to each observable, waiting until its completed, before subscribing
+  // to the next observable and waiting again.
+  //
+  // During all this, every #on_next is immediately forwarded to the downstream observables.
+  // In our case, we want to block until InodeNameMap is ready, and re-iterate all ftrace events.
+  auto/*observable<ResolvedPageCacheFtraceEvent>*/ resolved_events = inode_name_map_precombine
+    .concat(page_cache_ftrace_events_precombine)
+    .scan(CombinedState{},
+          [](CombinedState current_state, CombinedState delta_state) {
+            LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#scan "
+                          << "current=" << current_state << ","
+                          << "delta=" << delta_state;
+            // IT0    = (,)               + (InodeMap,)
+            // IT1    = (InodeMap,)       + (,Event)
+            // IT2..N = (InodeMap,Event1) + (,Event2)
+            current_state.Merge(std::move(delta_state));
+            return current_state;
+          })
+    .filter([](const CombinedState& state) {
+      return state.HasAll();
+    })
+    .map([](CombinedState& state) -> std::optional<ResolvedPageCacheFtraceEvent> {
+      PageCacheFtraceEvent& event = state.GetEvent();
+      const InodeMap& inode_map = state.GetInodeMap();
+
+      auto it = inode_map.find(event.inode());
+      if (it != inode_map.end()) {
+        std::string filename = it->second;
+        LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos combine_latest " << event;
+        return ResolvedPageCacheFtraceEvent{std::move(filename), std::move(event)};
+      } else {
+        LOG(ERROR) << "compiler: FtraceEvent's inode did not have resolved filename: " << event;
+        return std::nullopt;
+      }
+    })
+    .filter(
+      [](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) {
+        return maybe_event.has_value();
+      })
+    .map([](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) {
+      return std::move(maybe_event.value());
+    });
+    // -> observable<ResolvedPageCacheFtraceEvent>
+
+  return resolved_events;
+}
+
+namespace detail {
+bool multiless_one(const std::string& a, const std::string& b) {
+  return std::lexicographical_compare(a.begin(), a.end(),
+                                      b.begin(), b.end());
+}
+
+template <typename T>
+constexpr bool multiless_one(T&& a, T&& b) {   // a < b
+  using std::less;  // ADL
+  return less<std::decay_t<T>>{}(std::forward<T>(a), std::forward<T>(b));
+}
+
+constexpr bool multiless() {
+  return false;  // [] < [] is always false.
+}
+
+template <typename T, typename ... Args>
+constexpr bool multiless(T&& a, T&& b, Args&&... args) {
+  if (a != b) {
+    return multiless_one(std::forward<T>(a), std::forward<T>(b));
+  } else {
+    return multiless(std::forward<Args>(args)...);
+  }
+}
+
+}  // namespace detail
+
+// Return [A0...An] < [B0...Bn] ; vector-like scalar comparison of each field.
+// Arguments are passed in the order A0,B0,A1,B1,...,An,Bn.
+template <typename ... Args>
+constexpr bool multiless(Args&&... args) {
+  return detail::multiless(std::forward<Args>(args)...);
+}
+
+struct CompilerPageCacheEvent {
+  std::string filename;
+  uint64_t timestamp_relative;  // use relative timestamp because absolute values aren't comparable
+                                // across different trace protos.
+                                // relative timestamps can be said to be 'approximately' comparable.
+                                // assuming we compare the same application startup's trace times.
+  bool add_to_page_cache;  // AddToPageCache=true, DeleteFromPageCache=false.
+  uint64_t index;          // offset into file: this is a multiple of the page size (usually 4096).
+
+  // All other data from the ftrace is dropped because we don't currently use it in the
+  // compiler algorithms.
+
+  CompilerPageCacheEvent() = default;
+  CompilerPageCacheEvent(const ResolvedPageCacheFtraceEvent& resolved)
+    : CompilerPageCacheEvent(resolved.filename, resolved.event) {
+  }
+
+  CompilerPageCacheEvent(ResolvedPageCacheFtraceEvent&& resolved)
+    : CompilerPageCacheEvent(std::move(resolved.filename), std::move(resolved.event)) {
+  }
+
+  // Compare all fields (except the timestamp field).
+  static bool LessIgnoringTimestamp(const CompilerPageCacheEvent& a,
+                                    const CompilerPageCacheEvent& b) {
+    return multiless(a.filename, b.filename,
+                     a.add_to_page_cache, b.add_to_page_cache,
+                     a.index, b.index);
+  }
+
+  // Compare all fields. Timestamps get highest precedence.
+  bool operator<(const CompilerPageCacheEvent& rhs) const {
+    return multiless(timestamp_relative, rhs.timestamp_relative,
+                     filename, rhs.filename,
+                     add_to_page_cache, rhs.add_to_page_cache,
+                     index, rhs.index);
+  }
+
+ private:
+  CompilerPageCacheEvent(std::string filename, const PageCacheFtraceEvent& event)
+    : filename(std::move(filename)),
+      timestamp_relative(event.timestamp_relative),
+      add_to_page_cache(event.add_to_page_cache),
+      index(event.index) {
+   }
+};
+
+std::ostream& operator<<(std::ostream& os, const CompilerPageCacheEvent& e) {
+  os << "{";
+  os << "filename:\"" << e.filename << "\",";
+  os << "timestamp:" << e.timestamp_relative << ",";
+  os << "add_to_page_cache:" << e.add_to_page_cache << ",";
+  os << "index:" << e.index;
+  os << "}";
+  return os;
+}
+
+// Filter an observable chain of 'ResolvedPageCacheFtraceEvent'
+// into an observable chain of 'ResolvedPageCacheFtraceEvent'.
+//
+// Any items emitted by the input chain that match the regular expression
+// specified by blacklist_filter are not emitted into the output chain.
+auto/*observable<ResolvedPageCacheFtraceEvent>*/ ApplyBlacklistToPageCacheEvents(
+    rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events,
+    std::optional<std::string> blacklist_filter) {
+  bool has_re = blacklist_filter.has_value();
+  // default regex engine is ecmascript.
+  std::regex reg_exp{blacklist_filter ? *blacklist_filter : std::string("")};
+
+  return resolved_events.filter(
+    [reg_exp, has_re](const ResolvedPageCacheFtraceEvent& event) {
+      if (!has_re) {
+        return true;
+      }
+      // Remove any entries that match the regex in --blacklist-filter/-bf.
+      bool res = std::regex_search(event.filename, reg_exp);
+      if (res) {
+        LOG(VERBOSE) << "Blacklist filter removed '" << event.filename << "' from chain.";
+      }
+      return !res;
+    });
+}
+
+// Compile an observable chain of 'ResolvedPageCacheFtraceEvent' into
+// an observable chain of distinct, timestamp-ordered, CompilerPageCacheEvent.
+//
+// This is a reducing operation: No items are emitted until resolved_events is completed.
+auto/*observable<CompilerPageCacheEvent>*/ CompilePageCacheEvents(
+    rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events) {
+
+  struct CompilerPageCacheEventIgnoringTimestampLess {
+    bool operator()(const CompilerPageCacheEvent& lhs,
+                    const CompilerPageCacheEvent& rhs) const {
+      return CompilerPageCacheEvent::LessIgnoringTimestamp(lhs, rhs);
+    }
+  };
+
+  // Greedy O(N) compilation algorithm.
+  //
+  // This produces an inoptimal result (e.g. a small timestamp
+  // that might occur only 1% of the time nevertheless wins out), but the
+  // algorithm itself is quite simple, and doesn't require any heuristic tuning.
+
+  // First pass: *Merge* into set that ignores the timestamp value for order, but retains
+  //             the smallest timestamp value if the same key is re-inserted.
+  using IgnoreTimestampForOrderingSet =
+      std::set<CompilerPageCacheEvent, CompilerPageCacheEventIgnoringTimestampLess>;
+  // Second pass: *Sort* data by smallest timestamp first.
+  using CompilerPageCacheEventSet =
+      std::set<CompilerPageCacheEvent>;
+
+  return resolved_events
+    .map(
+      [](ResolvedPageCacheFtraceEvent event) {
+        // Drop all the extra metadata like pid, cpu, etc.
+        // When we merge we could keep a list of the original data, but there is no advantage
+        // to doing so.
+        return CompilerPageCacheEvent{std::move(event)};
+      }
+    )
+   .reduce(
+    IgnoreTimestampForOrderingSet{},
+    [](IgnoreTimestampForOrderingSet set, CompilerPageCacheEvent event) {
+      // Add each event to the set, keying by everything but the timestamp.
+      // If the key is already inserted, re-insert with the smaller timestamp value.
+      auto it = set.find(event);
+
+      if (it == set.end()) {
+        // Need to insert new element.
+        set.insert(std::move(event));
+      } else if (it->timestamp_relative > event.timestamp_relative) {
+        // Replace existing element: the new element has a smaller timestamp.
+        it = set.erase(it);
+        // Amortized O(1) time if insertion happens in the position before the hint.
+        set.insert(it, std::move(event));
+      } // else: Skip insertion. Element already present with the minimum timestamp.
+
+      return set;
+    },
+    [](IgnoreTimestampForOrderingSet set) {
+      // Extract all elements from 'set', re-insert into 'ts_set'.
+      // The values are now ordered by timestamp (and then the rest of the fields).
+      CompilerPageCacheEventSet ts_set;
+      ts_set.merge(std::move(set));
+
+
+      std::shared_ptr<CompilerPageCacheEventSet> final_set{
+          new CompilerPageCacheEventSet{std::move(ts_set)}};
+      return final_set;
+      // return ts_set;
+    })  // observable<CompilerPageCacheEventSet> (just)
+  .flat_map(
+    [](std::shared_ptr<CompilerPageCacheEventSet> final_set) {
+      // TODO: flat_map seems to make a copy of the parameter _every single iteration_
+      // without the shared_ptr it would just make a copy of the set every time it went
+      // through the iterate function.
+      // Causing absurdly slow compile times x1000 slower than we wanted.
+      // TODO: file a bug upstream and/or fix upstream.
+      CompilerPageCacheEventSet& ts_set = *final_set;
+    // [](CompilerPageCacheEventSet& ts_set) {
+      LOG(DEBUG) << "compiler: Merge-pass completed (" << ts_set.size() << " entries).";
+      //return rxcpp::sources::iterate(std::move(ts_set));
+      return rxcpp::sources::iterate(ts_set).map([](CompilerPageCacheEvent e) { return e; });
+    }
+  );   // observable<CompilerPageCacheEvent>
+}
+
+/** Makes a vector of info that includes filename and timestamp limit. */
+std::vector<CompilationInput> MakeCompilationInputs(
+    std::vector<std::string> input_file_names,
+    std::vector<uint64_t> timestamp_limit_ns){
+  // If the timestamp limit is empty, set the limit to max value
+  // for each trace file.
+  if (timestamp_limit_ns.empty()) {
+    for (size_t i = 0; i < input_file_names.size(); i++) {
+      timestamp_limit_ns.push_back(std::numeric_limits<uint64_t>::max());
+    }
+  }
+  DCHECK_EQ(input_file_names.size(), timestamp_limit_ns.size());
+  std::vector<CompilationInput> file_infos;
+  for (size_t i = 0; i < input_file_names.size(); i++) {
+    file_infos.push_back({input_file_names[i], timestamp_limit_ns[i]});
+  }
+  return file_infos;
+}
+
+bool PerformCompilation(std::vector<CompilationInput> perfetto_traces,
+                        std::string output_file_name,
+                        bool output_proto,
+                        std::optional<std::string> blacklist_filter,
+                        inode2filename::InodeResolverDependencies dependencies) {
+  auto trace_protos = ReadPerfettoTraceProtos(std::move(perfetto_traces));
+  auto resolved_events = ResolvePageCacheEntriesFromProtos(std::move(trace_protos),
+                                                           std::move(dependencies));
+  auto filtered_events =
+      ApplyBlacklistToPageCacheEvents(std::move(resolved_events), blacklist_filter);
+  auto compiled_events = CompilePageCacheEvents(std::move(filtered_events));
+
+  std::ofstream ofs;
+  if (!output_file_name.empty()) {
+
+    if (!output_proto) {
+      ofs.open(output_file_name);
+
+      if (!ofs) {
+        LOG(ERROR) << "compiler: Failed to open output file for writing: " << output_file_name;
+        return false;
+      }
+    }
+  }
+
+  auto trace_file_proto = serialize::ArenaPtr<serialize::proto::TraceFile>::Make();
+
+  // Fast lookup of filename -> FileIndex id.
+  std::unordered_map<std::string, int64_t /*file handle id*/> file_path_map;
+  int64_t file_handle_id = 0;
+
+  int counter = 0;
+  compiled_events
+    // .as_blocking()
+    .tap([&](CompilerPageCacheEvent& event) {
+      if (!output_proto) {
+        return;
+      }
+
+      if (!event.add_to_page_cache) {
+        // Skip DeleteFromPageCache events, they are only used for intermediate.
+        return;
+      }
+
+      DCHECK(trace_file_proto->mutable_index() != nullptr);
+      serialize::proto::TraceFileIndex& index = *trace_file_proto->mutable_index();
+      int64_t file_handle;
+
+      // Add TraceFileIndexEntry if it doesn't exist.
+
+      auto it = file_path_map.find(event.filename);
+      if (it == file_path_map.end()) {
+        file_handle = file_handle_id++;
+        file_path_map[event.filename] = file_handle;
+
+        serialize::proto::TraceFileIndexEntry* entry = index.add_entries();
+        DCHECK(entry != nullptr);
+        entry->set_id(file_handle);
+        entry->set_file_name(event.filename);
+
+        if (kIsDebugBuild) {
+          int i = static_cast<int>(file_handle);
+          const serialize::proto::TraceFileIndexEntry& entry_ex = index.entries(i);
+          DCHECK_EQ(entry->id(), entry_ex.id());
+          DCHECK_EQ(entry->file_name(), entry_ex.file_name());
+        }
+      } else {
+        file_handle = it->second;
+      }
+      int kPageSize = 4096;  // TODO: don't hardcode the page size.
+
+      // Add TraceFileEntry.
+      DCHECK(trace_file_proto->mutable_list() != nullptr);
+      serialize::proto::TraceFileEntry* entry = trace_file_proto->mutable_list()->add_entries();
+      DCHECK(entry != nullptr);
+
+      entry->set_index_id(file_handle);
+      // Page index -> file offset in bytes.
+      entry->set_file_offset(static_cast<int64_t>(event.index) * kPageSize);
+      entry->set_file_length(kPageSize);
+    })
+    .subscribe([&](CompilerPageCacheEvent event) {
+      if (!output_proto) {
+        if (output_file_name.empty()) {
+          LOG(INFO) << "CompilerPageCacheEvent" << event << std::endl;
+        } else {
+          ofs << event << "\n";  // TODO: write in proto format instead.
+        }
+      }
+      ++counter;
+    });
+
+  if (output_proto) {
+    LOG(DEBUG) << "compiler: WriteFully to begin into " << output_file_name;
+    ::google::protobuf::MessageLite& message = *trace_file_proto.get();
+    if (auto res = serialize::ProtobufIO::WriteFully(message, output_file_name); !res) {
+      errno = res.error();
+      PLOG(ERROR) << "compiler: Failed to write protobuf to file: " << output_file_name;
+      return false;
+    } else {
+      LOG(INFO) << "compiler: Wrote protobuf " << output_file_name;
+    }
+  }
+
+  LOG(DEBUG) << "compiler: Compilation completed (" << counter << " events).";
+
+  return true;
+}
+
+}  // namespace iorap::compiler
diff --git a/src/compiler/compiler.h b/src/compiler/compiler.h
new file mode 100644
index 0000000..1fa9ff1
--- /dev/null
+++ b/src/compiler/compiler.h
@@ -0,0 +1,65 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_COMPILER_COMPILER_H_
+#define IORAP_SRC_COMPILER_COMPILER_H_
+
+#include "inode2filename/inode_resolver.h"
+
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace iorap::compiler {
+struct  CompilationInput {
+  /* The name of the perfetto trace. */
+  std::string filename;
+  /*
+   * The timestamp limit of the trace.
+   * It's used to truncate the trace file.
+   */
+  uint64_t timestamp_limit_ns;
+};
+
+// Compile one or more perfetto TracePacket protobufs that are stored on the filesystem
+// by the filenames given with `input_file_names` and timestamp limit given with
+// timestamp_limit_ns.
+//
+// For each input file name and timestamp limit, ignore any events from the input that
+// exceed the associated timestamp limit.
+//
+// If blacklist_filter is non-null, ignore any file entries whose file path matches the
+// regular expression in blacklist_filter.
+//
+// The result is stored into the file path specified by `output_file_name`.
+//
+// This is a blocking function which returns only when compilation finishes. The return value
+// determines success/failure.
+//
+// Operation is transactional -- that is if there is a failure, `output_file_name` is untouched.
+bool PerformCompilation(std::vector<iorap::compiler::CompilationInput> perfetto_traces,
+                        std::string output_file_name,
+                        bool output_proto,
+                        std::optional<std::string> blacklist_filter,
+                        inode2filename::InodeResolverDependencies dependencies);
+
+// The size and order of timestamp_limit_ns should match that of
+// input_file_names, if not empty.
+// If timestamp_limit_ns is empty, will use the max uint64_t.
+std::vector<CompilationInput> MakeCompilationInputs(
+    std::vector<std::string> input_file_names,
+    std::vector<uint64_t> timestamp_limit_ns);
+}
+
+#endif  // IORAP_SRC_COMPILER_COMPILER_H_
diff --git a/src/compiler/main.cc b/src/compiler/main.cc
new file mode 100644
index 0000000..1ecfaed
--- /dev/null
+++ b/src/compiler/main.cc
@@ -0,0 +1,207 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/cmd_utils.h"
+#include "common/debug.h"
+#include "compiler/compiler.h"
+#include "inode2filename/inode_resolver.h"
+
+#include <android-base/parseint.h>
+#include <android-base/logging.h>
+
+#include <iostream>
+#include <limits>
+#include <optional>
+#include <string>
+
+#if defined(IORAP_COMPILER_MAIN)
+
+namespace iorap::compiler {
+
+void Usage(char** argv) {
+  std::cerr << "Usage: " << argv[0] << " [--output-proto=output.pb] input1.pb [input2.pb ...]" << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Request a compilation of multiple inputs (format: PerfettoTraceProto)." << std::endl;
+  std::cerr << "  The result is a TraceFile, representing a merged compiled trace with inodes resolved." << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Optional flags:" << std::endl;
+  std::cerr << "    --help,-h                  Print this Usage." << std::endl;
+  std::cerr << "    --blacklist-filter,-bf     Specify regex acting as a blacklist filter." << std::endl;
+  std::cerr << "                               Filepaths matching this regex are removed from the output file." << std::endl;
+  std::cerr << "    --output-text,-ot          Output ascii text instead of protobuf (default off)." << std::endl;
+  std::cerr << "    --output-proto $,-op $     TraceFile tracebuffer output file (default stdout)." << std::endl;
+  std::cerr << "    --inode-textcache $,-it $  Resolve inode->filename from textcache (disables diskscan)." << std::endl;
+  std::cerr << "    --verbose,-v               Set verbosity (default off)." << std::endl;
+  std::cerr << "    --wait,-w                  Wait for key stroke before continuing (default off)." << std::endl;
+  std::cerr << "    --timestamp_limit_ns,-tl   Set the limit timestamp in nanoseconds for the compiled trace. "
+                                              "The order and size of the timestamp should match that of "
+                                              "the input trace files. If not specified at all, All of"
+                                              "the timestamps are set to max."<< std::endl;
+  exit(1);
+}
+
+int Main(int argc, char** argv) {
+  android::base::InitLogging(argv);
+  android::base::SetLogger(android::base::StderrLogger);
+
+  bool wait_for_keystroke = false;
+  bool enable_verbose = false;
+
+  std::optional<std::string> arg_blacklist_filter;
+  std::string arg_output_proto;
+  bool arg_output_text = false;
+  std::optional<std::string> arg_inode_textcache;
+
+  std::vector<uint64_t> timestamp_limit_ns;
+
+  if (argc == 1) {
+    // Need at least 1 input file to do anything.
+    Usage(argv);
+  }
+
+  std::vector<std::string> arg_input_filenames;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+    bool has_arg_next = (arg+1)<argc;
+    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+
+    if (argstr == "--help" || argstr == "-h") {
+      Usage(argv);
+    } else if (argstr == "--output-proto" || argstr == "-op") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --output-proto <value>" << std::endl;
+        return 1;
+      }
+      arg_output_proto = arg_next;
+      ++arg;
+    } else if (argstr == "--output-text" || argstr == "-ot") {
+      arg_output_text = true;
+    } else if (argstr == "--inode-textcache" || argstr == "-it") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --inode-textcache <value>" << std::endl;
+        return 1;
+      }
+      arg_inode_textcache = arg_next;
+      ++arg;
+    } else if (argstr == "--blacklist-filter" || argstr == "-bf") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --blacklist-filter <value>" << std::endl;
+        return 1;
+      }
+      arg_blacklist_filter = arg_next;
+      ++arg;
+    }
+    else if (argstr == "--verbose" || argstr == "-v") {
+      enable_verbose = true;
+    } else if (argstr == "--wait" || argstr == "-w") {
+      wait_for_keystroke = true;
+    } else if (argstr == "--timestamp_limit_ns" || argstr == "-tl") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --timestamp_limit_ns <value>" << std::endl;
+        return 1;
+      }
+      uint64_t timestamp;
+      if (!::android::base::ParseUint<uint64_t>(arg_next, &timestamp)) {
+        std::cerr << "Invalid --timestamp-limit-ns "<< arg_next << std::endl;
+        return 1;
+      }
+      timestamp_limit_ns.push_back(timestamp);
+      ++arg;
+    } else {
+      arg_input_filenames.push_back(argstr);
+    }
+  }
+
+  if (!timestamp_limit_ns.empty() &&
+      timestamp_limit_ns.size() != arg_input_filenames.size()) {
+    std::cerr << "The size of timestamp limits doesn't match the size of input files."
+              << std::endl;
+    return 1;
+  }
+
+  if (enable_verbose) {
+    android::base::SetMinimumLogSeverity(android::base::VERBOSE);
+
+    LOG(VERBOSE) << "Verbose check";
+    LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild;
+  } else {
+    android::base::SetMinimumLogSeverity(android::base::DEBUG);
+  }
+
+  // Useful to attach a debugger...
+  // 1) $> iorap.cmd.compiler -w <args>
+  // 2) $> gdbclient <pid>
+  if (wait_for_keystroke) {
+    LOG(INFO) << "Self pid: " << getpid();
+    LOG(INFO) << "Press any key to continue...";
+    std::cin >> wait_for_keystroke;
+  }
+
+  auto system_call = std::make_unique<SystemCallImpl>();
+
+  using namespace inode2filename;
+
+  InodeResolverDependencies ir_dependencies;
+  // Passed from command-line.
+  if (arg_inode_textcache) {
+    ir_dependencies.data_source = DataSourceKind::kTextCache;
+    ir_dependencies.text_cache_filename = arg_inode_textcache;
+    ir_dependencies.verify = VerifyKind::kNone;  // required for determinism.
+  } else {
+    ir_dependencies.data_source = DataSourceKind::kDiskScan;
+    LOG(WARNING) << "--inode-textcache unspecified. "
+                 << "Inodes will be resolved by scanning the disk, which makes compilation "
+                 << "non-deterministic.";
+  }
+  // TODO: add command line.
+  ir_dependencies.root_directories.push_back("/system");
+  ir_dependencies.root_directories.push_back("/apex");
+  ir_dependencies.root_directories.push_back("/data");
+  ir_dependencies.root_directories.push_back("/vendor");
+  ir_dependencies.root_directories.push_back("/product");
+  ir_dependencies.root_directories.push_back("/metadata");
+  // Hardcoded.
+  if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.out_of_process", true)) {
+    ir_dependencies.process_mode = ProcessMode::kOutOfProcessIpc;
+  } else {
+    ir_dependencies.process_mode = ProcessMode::kInProcessDirect;
+  }
+  ir_dependencies.system_call = /*borrowed*/system_call.get();
+
+  int return_code = 0;
+  std::vector<CompilationInput> perfetto_traces =
+      MakeCompilationInputs(arg_input_filenames, timestamp_limit_ns);
+  return_code =
+      !PerformCompilation(std::move(perfetto_traces),
+                          std::move(arg_output_proto),
+                          !arg_output_text,
+                          arg_blacklist_filter,
+                          std::move(ir_dependencies));
+
+  // Uncomment this if we want to leave the process around to inspect it from adb shell.
+  // sleep(100000);
+
+  // 0 -> successfully wrote the proto out to file.
+  // 1 -> failed along the way (#on_error and also see the error logs).
+  return return_code;
+}
+
+}  // namespace iorap::compiler
+
+int main(int argc, char** argv) {
+  return ::iorap::compiler::Main(argc, argv);
+}
+
+#endif  // IORAP_COMPILER_MAIN
diff --git a/src/db/app_component_name.h b/src/db/app_component_name.h
new file mode 100644
index 0000000..e364163
--- /dev/null
+++ b/src/db/app_component_name.h
@@ -0,0 +1,130 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_DB_APP_COMPONENT_NAME_H_
+#define IORAP_SRC_DB_APP_COMPONENT_NAME_H_
+
+namespace iorap::db {
+
+struct AppComponentName {
+  std::string package;
+  std::string activity_name;
+
+  // Turns the activity name to the fully qualified name.
+  // For example, if the activity name is ".MainActivity" and the package is
+  // foo.bar. Then the fully qualified name is foo.bar.MainActivity.
+  AppComponentName Canonicalize() const {
+    if (!activity_name.empty() && activity_name[0] == '.') {
+      return {package, package + activity_name};
+    }
+    return {package, activity_name};
+  }
+
+  static bool HasAppComponentName(const std::string& s) {
+    return s.find('/') != std::string::npos;
+  }
+
+  // "com.foo.bar/.A" -> {"com.foo.bar", ".A"}
+  static AppComponentName FromString(const std::string& s) {
+    constexpr const char delimiter = '/';
+
+    if (!HasAppComponentName(s)) {
+      return {std::move(s), ""};
+    }
+
+    std::string package = s.substr(0, s.find(delimiter));
+
+    std::string activity_name = s;
+    activity_name.erase(0, s.find(delimiter) + sizeof(delimiter));
+
+    return {std::move(package), std::move(activity_name)};
+  }
+
+  // {"com.foo.bar", ".A"} -> "com.foo.bar/.A"
+  std::string ToString() const {
+    return package + "/" + activity_name;
+  }
+
+  /*
+   * '/' is encoded into %2F
+   * '%' is encoded into %25
+   *
+   * This allows the component name to be be used as a file name
+   * ('/' is illegal due to being a path separator) with minimal
+   * munging.
+   */
+
+  // "com.foo.bar%2F.A%25" -> {"com.foo.bar", ".A%"}
+  static AppComponentName FromUrlEncodedString(const std::string& s) {
+    std::string cpy = s;
+    Replace(cpy, "%2F", "/");
+    Replace(cpy, "%25", "%");
+
+    return FromString(cpy);
+  }
+
+  // {"com.foo.bar", ".A%"} -> "com.foo.bar%2F.A%25"
+  std::string ToUrlEncodedString() const {
+    std::string s = ToString();
+    Replace(s, "%", "%25");
+    Replace(s, "/", "%2F");
+    return s;
+  }
+
+  /*
+   * '/' is encoded into @@
+   * '%' is encoded into ^^
+   *
+   * Two purpose:
+   * 1. This allows the package name to be used as a file name
+   * ('/' is illegal due to being a path separator) with minimal
+   * munging.
+   * 2. This allows the package name to be used in .mk file because
+   * '%' is a special char and cannot be easily escaped in Makefile.
+   *
+   * This is a workround for test purpose.
+   * Only package name is used because activity name varies on
+   * different testing framework.
+   * Hopefully, the double "@@" and "^^" are not used in other cases.
+   */
+  // {"com.foo.bar", ".A%"} -> "com.foo.bar"
+  std::string ToMakeFileSafeEncodedPkgString() const {
+    std::string s = package;
+    Replace(s, "/", "@@");
+    Replace(s, "%", "^^");
+    return s;
+  }
+
+ private:
+  static bool Replace(std::string& str, const std::string& from, const std::string& to) {
+    // TODO: call in a loop to replace all occurrences, not just the first one.
+    const size_t start_pos = str.find(from);
+    if (start_pos == std::string::npos) {
+      return false;
+    }
+
+    str.replace(start_pos, from.length(), to);
+
+    return true;
+}
+};
+
+inline std::ostream& operator<<(std::ostream& os, const AppComponentName& name) {
+  os << name.ToString();
+  return os;
+}
+
+}  // namespace iorap::db
+
+#endif  // IORAP_SRC_DB_APP_COMPONENT_NAME_H_
diff --git a/src/db/clean_up.cc b/src/db/clean_up.cc
new file mode 100644
index 0000000..de1412d
--- /dev/null
+++ b/src/db/clean_up.cc
@@ -0,0 +1,115 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "db/clean_up.h"
+
+#include <android-base/file.h>
+
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "db/file_models.h"
+#include "db/models.h"
+
+namespace iorap::db {
+
+void CleanUpFilesForActivity(const db::DbHandle& db,
+                             const db::VersionedComponentName& vcn) {
+  LOG(DEBUG) << "Clean up files for activity " << vcn.GetActivity();
+  // Remove perfetto traces.
+  std::vector<db::RawTraceModel> raw_traces =
+      db::RawTraceModel::SelectByVersionedComponentName(db, vcn);
+  for (db::RawTraceModel raw_trace : raw_traces) {
+    std::string file_path = raw_trace.file_path;
+    LOG(DEBUG) << "Remove file: " << file_path;
+    std::filesystem::remove(file_path.c_str());
+    raw_trace.Delete();
+  }
+
+  // Remove compiled traces.
+  std::optional<db::PrefetchFileModel> prefetch_file =
+      db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn);
+
+  if (prefetch_file) {
+    std::string file_path = prefetch_file->file_path;
+    LOG(DEBUG) << "Remove file: " << file_path;
+    std::filesystem::remove(file_path.c_str());
+    prefetch_file->Delete();
+  }
+}
+
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            int package_id,
+                            const std::string& package_name,
+                            int64_t version) {
+  LOG(DEBUG) << "Clean up files for package " << package_name << " with version "
+             << version;
+  std::vector<db::ActivityModel> activities =
+      db::ActivityModel::SelectByPackageId(db, package_id);
+
+  for (db::ActivityModel activity : activities) {
+    db::VersionedComponentName vcn{package_name, activity.name, version};
+    CleanUpFilesForActivity(db, vcn);
+  }
+}
+
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            const std::string& package_name,
+                            int64_t version) {
+  std::optional<PackageModel> package =
+        PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version);
+
+  if (!package) {
+    LOG(DEBUG) << "No package to clean up " << package_name << " with version " << version;
+    return;
+  }
+
+  CleanUpFilesForPackage(db, package->id, package_name, version);
+}
+
+void CleanUpFilesForDb(const db::DbHandle& db) {
+  std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db);
+  for (db::PackageModel package : packages) {
+      CleanUpFilesForPackage(db, package.id, package.name, package.version);
+  }
+}
+
+void CleanUpFilesForPackage(const std::string& db_path,
+                            const std::string& package_name) {
+  iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path);
+  db::DbHandle db{db_schema.db()};
+  CleanUpFilesForPackage(db, package_name);
+}
+
+
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            const std::string& package_name) {
+  std::vector<PackageModel> packages = PackageModel::SelectByName(db, package_name.c_str());;
+
+  for (PackageModel& package : packages) {
+    CleanUpFilesForPackage(db, package.id, package.name, package.version);
+  }
+
+  if (packages.empty()) {
+    LOG(DEBUG) << "No package rows to clean up " << package_name;
+  }
+}
+
+}  // namespace iorap::db
diff --git a/src/db/clean_up.h b/src/db/clean_up.h
new file mode 100644
index 0000000..08b7bde
--- /dev/null
+++ b/src/db/clean_up.h
@@ -0,0 +1,54 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_DB_CLEANER_H_
+#define IORAP_SRC_DB_CLEANER_H_
+
+#include <android/content/pm/IPackageManagerNative.h>
+
+#include <string>
+#include <vector>
+
+#include "binder/package_version_map.h"
+#include "db/file_models.h"
+
+namespace iorap::db {
+
+// Clean up perfetto traces and compiled traces in disk and rows
+// in raw_traces and prefetch_files in the db.
+void CleanUpFilesForDb(const db::DbHandle& db);
+
+// Clean up perfetto traces and compiled traces in disk and rows
+// in raw_traces and prefetch_files in the db for a package id.
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            int package_id,
+                            const std::string& package_name,
+                            int64_t version);
+
+// Clean up all package rows (and files) associated with a package by name.
+void CleanUpFilesForPackage(const std::string& db_path,
+                            const std::string& package_name);
+
+// Clean up all package rows (and files) associated with a package by name.
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            const std::string& package_name);
+// Clean up perfetto traces and compiled traces in disk and rows
+// in raw_traces and prefetch_files in the db for a package name
+// and version.
+void CleanUpFilesForPackage(const db::DbHandle& db,
+                            const std::string& package_name,
+                            int64_t version);
+}  // namespace iorap::db
+
+#endif  // IORAP_SRC_DB_CLEANER_H_
diff --git a/src/db/file_models.cc b/src/db/file_models.cc
new file mode 100644
index 0000000..0fe6d8f
--- /dev/null
+++ b/src/db/file_models.cc
@@ -0,0 +1,181 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/cmd_utils.h"
+#include "db/file_models.h"
+#include "db/models.h"
+
+#include <android-base/file.h>
+#include <android-base/properties.h>
+
+#include <algorithm>
+#include <sstream>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <unistd.h>
+
+namespace iorap::db {
+
+static constexpr const char* kRootPathProp = "iorapd.root.dir";
+static const unsigned int kPerfettoMaxTraces =
+    ::android::base::GetUintProperty("iorapd.perfetto.max_traces", /*default*/10u);
+
+static uint64_t GetTimeNanoseconds() {
+  struct timespec now;
+  clock_gettime(CLOCK_REALTIME, &now);
+
+  uint64_t now_ns = (now.tv_sec * 1000000000LL + now.tv_nsec);
+  return now_ns;
+}
+
+static bool IsDir(const std::string& dirpath) {
+  struct stat st;
+  if (stat(dirpath.c_str(), &st) == 0) {
+    if (S_ISDIR(st.st_mode)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+// Given some path /a/b/c create all of /a, /a/b, /a/b/c/ recursively.
+static bool MkdirWithParents(const std::string& path) {
+  size_t prev_end = 0;
+  while (prev_end < path.size()) {
+    size_t next_end = path.find('/', prev_end + 1);
+
+    std::string dir_path = path.substr(0, next_end);
+    if (!IsDir(dir_path)) {
+#if defined(_WIN32)
+      int ret = mkdir(dir_path.c_str());
+#else
+      mode_t old_mask = umask(0);
+      // The user permission is 5 to allow system server to
+      // read the files. No other users could do that because
+      // the upper directory only allows system server and iorapd
+      // to access. Also selinux rules prevent other users to
+      // read files here.
+      int ret = mkdir(dir_path.c_str(), 0755);
+      umask(old_mask);
+#endif
+      if (ret != 0) {
+        PLOG(ERROR) << "failed to create dir " << dir_path;
+        return false;
+      }
+    }
+    prev_end = next_end;
+
+    if (next_end == std::string::npos) {
+      break;
+    }
+  }
+  return true;
+}
+
+FileModelBase::FileModelBase(VersionedComponentName vcn)
+  : vcn_{std::move(vcn)} {
+    root_path_ = common::GetEnvOrProperty(kRootPathProp,
+                                          /*default*/"/data/misc/iorapd");
+}
+
+std::string FileModelBase::BaseDir() const {
+  std::stringstream ss;
+
+  ss << root_path_ << "/" << vcn_.GetPackage() << "/";
+  ss << vcn_.GetVersion();
+  ss << "/";
+  ss << vcn_.GetActivity() << "/";
+  ss << SubDir();
+
+  return ss.str();
+}
+
+std::string FileModelBase::FilePath() const {
+  std::stringstream ss;
+  ss << BaseDir();
+  ss << "/";
+  ss << BaseFile();
+
+  return ss.str();
+}
+
+bool FileModelBase::MkdirWithParents() {
+  LOG(VERBOSE) << "MkdirWithParents: " << BaseDir();
+  return ::iorap::db::MkdirWithParents(BaseDir());
+}
+
+PerfettoTraceFileModel::PerfettoTraceFileModel(VersionedComponentName vcn,
+                                               uint64_t timestamp)
+  : FileModelBase{std::move(vcn)}, timestamp_{timestamp} {
+}
+
+PerfettoTraceFileModel PerfettoTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) {
+  uint64_t timestamp = GetTimeNanoseconds();
+  return PerfettoTraceFileModel{vcn, timestamp};
+}
+
+std::string PerfettoTraceFileModel::BaseFile() const {
+  std::stringstream ss;
+  ss << timestamp_ << ".perfetto_trace.pb";
+  return ss.str();
+}
+
+void PerfettoTraceFileModel::DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn) {
+  std::vector<RawTraceModel> raw_traces =
+      RawTraceModel::SelectByVersionedComponentName(db, vcn);
+
+  if (WOULD_LOG(VERBOSE)) {
+    size_t raw_traces_size = raw_traces.size();
+    for (size_t i = 0; i < raw_traces_size; ++i) {
+      LOG(VERBOSE) << "DeleteOlderFiles - selected " << raw_traces[i];
+    }
+    LOG(VERBOSE) << "DeleteOlderFiles - queried total " << raw_traces_size << " records";
+  }
+
+  size_t items_to_delete = 0;
+  if (raw_traces.size() > kPerfettoMaxTraces) {
+    items_to_delete = raw_traces.size() - kPerfettoMaxTraces;
+  } else {
+    LOG(VERBOSE) << "DeleteOlderFiles - don't delete older raw traces, too few files: "
+                 << " wanted at least " << kPerfettoMaxTraces << ", but got " << raw_traces.size();
+  }
+
+  for (size_t i = 0; i < items_to_delete; ++i) {
+    RawTraceModel& raw_trace = raw_traces[i];  // sorted ascending -> items to delete are first.
+    std::string err_msg;
+
+    if (!::android::base::RemoveFileIfExists(raw_trace.file_path, /*out*/&err_msg)) {
+      LOG(ERROR) << "Failed to remove raw trace file: " << raw_trace.file_path
+                 << ", reason: " << err_msg;
+    } else {
+      raw_trace.Delete();
+      LOG(DEBUG) << "Deleted raw trace for " << vcn << " at " << raw_trace.file_path;
+    }
+  }
+}
+
+CompiledTraceFileModel::CompiledTraceFileModel(VersionedComponentName vcn)
+  : FileModelBase{std::move(vcn)} {
+}
+
+CompiledTraceFileModel CompiledTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) {
+  return CompiledTraceFileModel{vcn};
+}
+
+std::string CompiledTraceFileModel::BaseFile() const {
+  return "compiled_trace.pb";
+}
+
+}  // namespace iorap::db
diff --git a/src/db/file_models.h b/src/db/file_models.h
new file mode 100644
index 0000000..135c98a
--- /dev/null
+++ b/src/db/file_models.h
@@ -0,0 +1,124 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_DB_FILE_MODELS_H_
+#define IORAP_SRC_DB_FILE_MODELS_H_
+
+#include <optional>
+#include <ostream>
+#include <string>
+
+namespace iorap::db {
+
+struct VersionedComponentName {
+  VersionedComponentName(std::string package,
+                         std::string activity,
+                         int64_t version)
+    : package_{std::move(package)},
+      activity_{std::move(activity)},
+      version_{version} {
+  }
+
+  const std::string& GetPackage() const {
+    return package_;
+  }
+
+  const std::string& GetActivity() const {
+    return activity_;
+  }
+
+  int GetVersion() const {
+    return version_;
+  }
+
+ private:
+  std::string package_;
+  std::string activity_;
+  int64_t version_;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const VersionedComponentName& vcn) {
+  os << vcn.GetPackage() << "/" << vcn.GetActivity() << "@" << vcn.GetVersion();
+  return os;
+}
+
+class FileModelBase {
+ protected:
+  FileModelBase(VersionedComponentName vcn);
+
+  virtual std::string SubDir() const = 0;
+  // Include the last file component only (/a/b/c.txt -> c.txt)
+  virtual std::string BaseFile() const = 0;
+
+  virtual ~FileModelBase() {}
+
+ public:
+  virtual std::string ModelName() const = 0;
+  // Return the absolute file path to this FileModel.
+  std::string FilePath() const;
+
+  // Include the full path minus the basefile (/a/b/c.txt -> /a/b)
+  std::string BaseDir() const;
+
+  // Create the base dir recursively (e.g. mkdir -p $basedir).
+  bool MkdirWithParents();
+ private:
+  VersionedComponentName vcn_;
+  std::string subdir_;
+  std::string filename_;
+  std::string root_path_;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const FileModelBase& fmb) {
+  os << fmb.ModelName() << "{'" << fmb.FilePath() << "'}";
+  return os;
+}
+
+class DbHandle;
+
+class PerfettoTraceFileModel : public FileModelBase {
+ protected:
+  virtual std::string ModelName() const override { return "PerfettoTrace"; }
+  virtual std::string SubDir() const override { return "raw_traces"; }
+  virtual std::string BaseFile() const override;
+
+  PerfettoTraceFileModel(VersionedComponentName vcn,
+                         uint64_t timestamp);
+
+ public:
+  static PerfettoTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn);
+  static void DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn);
+
+  virtual ~PerfettoTraceFileModel() {}
+ private:
+  uint64_t timestamp_;
+};
+
+class CompiledTraceFileModel : public FileModelBase {
+ protected:
+  virtual std::string ModelName() const override { return "CompiledTrace"; }
+  virtual std::string SubDir() const override { return "compiled_traces"; }
+  virtual std::string BaseFile() const override;
+
+  CompiledTraceFileModel(VersionedComponentName vcn);
+
+ public:
+  static CompiledTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn);
+
+  virtual ~CompiledTraceFileModel() {}
+};
+
+}   // namespace iorap::db
+
+#endif  // IORAP_SRC_DB_FILE_MODELS_H_
diff --git a/src/db/main.cc b/src/db/main.cc
new file mode 100644
index 0000000..2edd961
--- /dev/null
+++ b/src/db/main.cc
@@ -0,0 +1,230 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/debug.h"
+#include "common/loggers.h"
+#include "db/app_component_name.h"
+#include "db/models.h"
+
+#include <android-base/parseint.h>
+#include <android-base/logging.h>
+
+#include <iostream>
+#include <optional>
+#include <string_view>
+#include <string>
+#include <vector>
+
+#include <sqlite3.h>
+
+#include <signal.h>
+
+namespace iorap::db {
+
+void Usage(char** argv) {
+  std::cerr << "Usage: " << argv[0] << " <path-to-sqlite.db>" << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Interface with the iorap sqlite database and issue commands." << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Optional flags:" << std::endl;
+  std::cerr << "    --help,-h                  Print this Usage." << std::endl;
+  std::cerr << "    --register-raw-trace,-rrt  Register raw trace file path." << std::endl;
+  std::cerr << "    --register-compiled-trace,-rct  Register compiled trace file path." << std::endl;
+  std::cerr << "    --insert-component,-ic     Add component if it doesn't exist." << std::endl;
+  std::cerr << "    --initialize,-i            Initialize new database." << std::endl;
+  std::cerr << "    --rescan,-rs               Update all from canonical directories." << std::endl;
+  std::cerr << "    --prune,-pr                Remove any stale file paths." << std::endl;
+  std::cerr << "    --verbose,-v               Set verbosity (default off)." << std::endl;
+  std::cerr << "    --wait,-w                  Wait for key stroke before continuing (default off)." << std::endl;
+  exit(1);
+}
+
+void error_log_sqlite_callback(void *pArg, int iErrCode, const char *zMsg) {
+  LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg;
+}
+
+const constexpr int64_t kNoVersion = -1;
+
+int Main(int argc, char** argv) {
+  // Go to system logcat + stderr when running from command line.
+  android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM});
+
+  bool wait_for_keystroke = false;
+  bool enable_verbose = false;
+
+  bool command_format_text = false; // false = binary.
+
+  int arg_input_fd = -1;
+  int arg_output_fd = -1;
+
+  std::vector<std::string> arg_input_filenames;
+  bool arg_use_sockets = false;
+
+  std::vector<std::pair<std::string,std::string>> arg_register_raw_trace;
+  std::vector<std::pair<std::string,std::string>> arg_register_compiled_trace;
+
+  std::vector<std::string> arg_insert_component;
+
+  bool arg_initialize = false;
+  bool arg_rescan = false;
+  bool arg_prune = false;
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+    bool has_arg_next = (arg+1)<argc;
+    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+
+    bool has_arg_next_next = (arg+2)<argc;
+    std::string arg_next_next = has_arg_next_next ? argv[arg+2] : "";
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+
+    if (argstr == "--help" || argstr == "-h") {
+      Usage(argv);
+    } else if (argstr == "--register-raw-trace" || argstr == "-rrt") {
+      if (!has_arg_next_next) {
+        LOG(ERROR) << "--register-raw-trace <component/name> <filepath>";
+        Usage(argv);
+      }
+      arg_register_raw_trace.push_back({arg_next, arg_next_next});
+    } else if (argstr == "--register-compiled-trace" || argstr == "-rct") {
+      if (!has_arg_next_next) {
+        LOG(ERROR) << "--register-compiled-trace <component/name> <filepath>";
+        Usage(argv);
+      }
+      arg_register_compiled_trace.push_back({arg_next, arg_next_next});
+    } else if (argstr == "--insert-component" || argstr == "-ic") {
+      if (!has_arg_next) {
+        LOG(ERROR) << "--insert-component <component/name>";
+        Usage(argv);
+      }
+      arg_insert_component.push_back(arg_next);
+    } else if (argstr == "--initialize" || argstr == "-i") {
+      arg_initialize = true;
+    } else if (argstr == "--rescan" || argstr == "-rs") {
+      arg_rescan = true;
+    } else if (argstr == "--prune" || argstr == "-pr") {
+      arg_prune = true;
+    } else if (argstr == "--verbose" || argstr == "-v") {
+      enable_verbose = true;
+    } else if (argstr == "--wait" || argstr == "-w") {
+      wait_for_keystroke = true;
+    } else {
+      arg_input_filenames.push_back(argstr);
+    }
+  }
+
+  if (arg_input_filenames.empty()) {
+    LOG(ERROR) << "Missing positional filename to a sqlite database.";
+    Usage(argv);
+  }
+
+  if (enable_verbose) {
+    android::base::SetMinimumLogSeverity(android::base::VERBOSE);
+
+    LOG(VERBOSE) << "Verbose check";
+    LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild;
+  } else {
+    android::base::SetMinimumLogSeverity(android::base::DEBUG);
+  }
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+  }
+
+  // Useful to attach a debugger...
+  // 1) $> iorap.cmd.readahead -w <args>
+  // 2) $> gdbclient <pid>
+  if (wait_for_keystroke) {
+    LOG(INFO) << "Self pid: " << getpid();
+
+    raise(SIGSTOP);
+    // LOG(INFO) << "Press any key to continue...";
+    // std::cin >> wait_for_keystroke;
+  }
+
+  // auto system_call = std::make_unique<SystemCallImpl>();
+  // TODO: mock readahead calls?
+  //
+  // Uncomment this if we want to leave the process around to inspect it from adb shell.
+  // sleep(100000);
+
+  int return_code = 0;
+
+  LOG(VERBOSE) << "Hello world";
+
+
+  do {
+    SchemaModel schema_model = SchemaModel::GetOrCreate(arg_input_filenames[0]);
+    DbHandle db = schema_model.db();
+    if (arg_initialize) {
+      // Drop tables and restart from scratch. All rows are effectively dropped.
+      schema_model.Reinitialize();
+    }
+
+    for (const auto& component_and_file_name : arg_register_raw_trace) {
+      AppComponentName component_name = AppComponentName::FromString(component_and_file_name.first);
+      const std::string& file_path = component_and_file_name.second;
+
+      LOG(VERBOSE) << "--register-raw-trace " << component_name << ", file_path: " << file_path;
+
+      std::optional<ActivityModel> activity =
+          ActivityModel::SelectOrInsert(db,
+                                        component_name.package,
+                                        kNoVersion,
+                                        component_name.activity_name);
+      DCHECK(activity.has_value());
+      LOG(DEBUG) << "Component selected/inserted: " << *activity;
+    }
+
+    for (const std::string& component : arg_insert_component) {
+      AppComponentName component_name = AppComponentName::FromString(component);
+
+      LOG(VERBOSE) << "raw component: " << component;
+      LOG(VERBOSE) << "package: " << component_name.package;
+      LOG(VERBOSE) << "activity name: " << component_name.activity_name;
+
+      LOG(VERBOSE) << "--insert-component " << component_name;
+
+      std::optional<ActivityModel> activity =
+          ActivityModel::SelectOrInsert(db,
+                                        component_name.package,
+                                        kNoVersion,
+                                        component_name.activity_name);
+
+      DCHECK(activity.has_value());
+      LOG(DEBUG) << "Component selected/inserted: " << *activity;
+    }
+  } while (false);
+
+  LOG(VERBOSE) << "main: Terminating";
+
+  // 0 -> successfully executed all commands.
+  // 1 -> failed along the way (#on_error and also see the error logs).
+  return return_code;
+}
+
+}  // namespace iorap::db
+
+#if defined(IORAP_DB_MAIN)
+int main(int argc, char** argv) {
+  return ::iorap::db::Main(argc, argv);
+}
+#endif  // IORAP_DB_MAIN
diff --git a/src/db/models.cc b/src/db/models.cc
new file mode 100644
index 0000000..851a061
--- /dev/null
+++ b/src/db/models.cc
@@ -0,0 +1,21 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "db/models.h"
+
+namespace iorap::db {
+
+std::optional<DbHandle> SchemaModel::s_singleton_;
+
+}  // namespace iorap::db
diff --git a/src/db/models.h b/src/db/models.h
new file mode 100644
index 0000000..df6beb6
--- /dev/null
+++ b/src/db/models.h
@@ -0,0 +1,1130 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_DB_MODELS_H_
+#define IORAP_SRC_DB_MODELS_H_
+
+#include "clean_up.h"
+#include "file_models.h"
+
+#include <android-base/logging.h>
+#include <utils/String8.h>
+
+#include <filesystem>
+#include <iostream>
+#include <optional>
+#include <ostream>
+#include <string>
+#include <sstream>
+#include <type_traits>
+#include <vector>
+
+#include <sqlite3.h>
+
+namespace iorap::db {
+
+const constexpr int kDbVersion = 2;
+
+struct SqliteDbDeleter {
+  void operator()(sqlite3* db) {
+    if (db != nullptr) {
+      LOG(VERBOSE) << "sqlite3_close for: " << db;
+      sqlite3_close(db);
+    }
+  }
+};
+
+class DbHandle {
+ public:
+  // Take over ownership of sqlite3 db.
+  explicit DbHandle(sqlite3* db)
+    : db_{std::shared_ptr<sqlite3>{db, SqliteDbDeleter{}}},
+      mutex_{std::make_shared<std::mutex>()} {
+  }
+
+  sqlite3* get() {
+    return db_.get();
+  }
+
+  operator sqlite3*() {
+    return db_.get();
+  }
+
+  std::mutex& mutex() {
+    return *mutex_.get();
+  }
+
+ private:
+  std::shared_ptr<sqlite3> db_;
+  std::shared_ptr<std::mutex> mutex_;
+};
+
+class ScopedLockDb {
+ public:
+  ScopedLockDb(std::mutex& mutex) : mutex(mutex) {
+    mutex.lock();
+  }
+
+  ScopedLockDb(DbHandle& handle) : ScopedLockDb(handle.mutex()) {
+  }
+
+  ~ScopedLockDb() {
+    mutex.unlock();
+  }
+ private:
+  std::mutex& mutex;
+};
+
+class DbStatement {
+ public:
+  template <typename ... Args>
+  static DbStatement Prepare(DbHandle db, const std::string& sql, Args&&... args) {
+    return Prepare(db, sql.c_str(), std::forward<Args>(args)...);
+  }
+
+  template <typename ... Args>
+  static DbStatement Prepare(DbHandle db, const char* sql, Args&&... args) {
+    DCHECK(db.get() != nullptr);
+    DCHECK(sql != nullptr);
+
+    // LOG(VERBOSE) << "Prepare DB=" << db.get();
+
+    sqlite3_stmt* stmt = nullptr;
+    int rc = sqlite3_prepare_v2(db.get(), sql, -1, /*out*/&stmt, nullptr);
+
+    DbStatement db_stmt{db, stmt};
+    DCHECK(db_stmt.CheckOk(rc)) << sql;
+    db_stmt.BindAll(std::forward<Args>(args)...);
+
+    return db_stmt;
+  }
+
+  DbStatement(DbHandle db, sqlite3_stmt* stmt) : db_(db), stmt_(stmt) {
+  }
+
+  sqlite3_stmt* get() {
+    return stmt_;
+  }
+
+  DbHandle db() {
+    return db_;
+  }
+
+  // Successive BindAll calls *do not* start back at the 0th bind position.
+  template <typename T, typename ... Args>
+  void BindAll(T&& arg, Args&&... args) {
+    Bind(std::forward<T>(arg));
+    BindAll(std::forward<Args>(args)...);
+  }
+
+  void BindAll() {}
+
+  template <typename T>
+  void Bind(const std::optional<T>& value) {
+    if (value) {
+      Bind(*value);
+    } else {
+      BindNull();
+    }
+  }
+
+  void Bind(bool value) {
+    CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value));
+  }
+
+  void Bind(int value) {
+    CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value));
+  }
+
+  void Bind(uint64_t value) {
+    CheckOk(sqlite3_bind_int64(stmt_, bind_counter_++, static_cast<int64_t>(value)));
+  }
+
+  void Bind(const char* value) {
+    if (value != nullptr) {
+      //sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_STATIC);
+      CheckOk(sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_TRANSIENT));
+    } else {
+      BindNull();
+    }
+  }
+
+  void Bind(const std::string& value) {
+    Bind(value.c_str());
+  }
+
+  template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>>
+  void Bind(E value) {
+    Bind(static_cast<std::underlying_type_t<E>>(value));
+  }
+
+  void BindNull() {
+    CheckOk(sqlite3_bind_null(stmt_, bind_counter_++));
+  }
+
+  int Step() {
+    ++step_counter_;
+    return sqlite3_step(stmt_);
+  }
+
+  bool Step(int expected) {
+    int rc = Step();
+    if (rc != expected) {
+      LOG(ERROR) << "SQLite error: " << sqlite3_errmsg(db_.get());
+      return false;
+    }
+    return true;
+  }
+
+  // Successive ColumnAll calls start at the 0th column again.
+  template <typename T, typename ... Args>
+  void ColumnAll(T& first, Args&... rest) {
+    Column(first);
+    ColumnAll(rest...);
+    // Reset column counter back to 0
+    column_counter_ = 0;
+  }
+
+  void ColumnAll() {}
+
+  template <typename T>
+  void Column(std::optional<T>& value) {
+    T tmp;
+    Column(/*out*/tmp);
+
+    if (!tmp) {  // disambiguate 0 and NULL
+      const unsigned char* text = sqlite3_column_text(stmt_, column_counter_ - 1);
+      if (text == nullptr) {
+        value = std::nullopt;
+        return;
+      }
+    }
+    value = std::move(tmp);
+  }
+
+  template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>>
+  void Column(E& value) {
+    std::underlying_type_t<E> tmp;
+    Column(/*out*/tmp);
+    value = static_cast<E>(tmp);
+  }
+
+  void Column(bool& value) {
+    value = sqlite3_column_int(stmt_, column_counter_++);
+  }
+
+  void Column(int& value) {
+    value = sqlite3_column_int(stmt_, column_counter_++);
+  }
+
+  void Column(uint64_t& value) {
+    value = static_cast<uint64_t>(sqlite3_column_int64(stmt_, column_counter_++));
+  }
+
+  void Column(std::string& value) {
+    const unsigned char* text = sqlite3_column_text(stmt_, column_counter_++);
+
+    DCHECK(text != nullptr) << "Column should be marked NOT NULL, otherwise use optional<string>";
+    if (text == nullptr) {
+      LOG(ERROR) << "Got NULL back for column " << column_counter_-1
+                 << "; is this column marked NOT NULL?";
+      value = "(((null)))";  // Don't segfault, keep going.
+      return;
+    }
+
+    value = std::string{reinterpret_cast<const char*>(text)};
+  }
+
+  const char* ExpandedSql() {
+    char* p = sqlite3_expanded_sql(stmt_);
+    if (p == nullptr) {
+      return "(nullptr)";
+    }
+    return p;
+  }
+
+  const char* Sql() {
+    const char* p = sqlite3_sql(stmt_);
+    if (p == nullptr) {
+      return "(nullptr)";
+    }
+    return p;
+  }
+
+
+  DbStatement(DbStatement&& other)
+    : db_{other.db_}, stmt_{other.stmt_}, bind_counter_{other.bind_counter_},
+      step_counter_{other.step_counter_} {
+    other.db_ = DbHandle{nullptr};
+    other.stmt_ = nullptr;
+  }
+
+  ~DbStatement() {
+    if (stmt_ != nullptr) {
+      DCHECK_GT(step_counter_, 0) << " forgot to call Step()?";
+      sqlite3_finalize(stmt_);
+    }
+  }
+
+ private:
+  bool CheckOk(int rc, int expected = SQLITE_OK) {
+    if (rc != expected) {
+      LOG(ERROR) << "Got error for SQL query: '" << Sql() << "'"
+                 << ", expanded: '" << ExpandedSql() << "'";
+      LOG(ERROR) << "Failed SQLite api call (" << rc << "): " << sqlite3_errstr(rc);
+    }
+    return rc == expected;
+  }
+
+  DbHandle db_;
+  sqlite3_stmt* stmt_;
+  int bind_counter_ = 1;
+  int step_counter_ = 0;
+  int column_counter_ = 0;
+};
+
+class DbQueryBuilder {
+ public:
+  // Returns the row ID that was inserted last.
+  template <typename... Args>
+  static std::optional<int> Insert(DbHandle db, const std::string& sql, Args&&... args) {
+    ScopedLockDb lock{db};
+
+    sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db.get());
+    DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...);
+
+    if (!stmt.Step(SQLITE_DONE)) {
+      return std::nullopt;
+    }
+
+    last_rowid = sqlite3_last_insert_rowid(db.get());
+    DCHECK_GT(last_rowid, 0);
+
+    return static_cast<int>(last_rowid);
+  }
+
+  template <typename... Args>
+  static bool Delete(DbHandle db, const std::string& sql, Args&&... args) {
+    return ExecuteOnce(db, sql, std::forward<Args>(args)...);
+  }
+
+  template <typename... Args>
+  static bool Update(DbHandle db, const std::string& sql, Args&&... args) {
+    return ExecuteOnce(db, sql, std::forward<Args>(args)...);
+  }
+
+  template <typename... Args>
+  static bool ExecuteOnce(DbHandle db, const std::string& sql, Args&&... args) {
+    ScopedLockDb lock{db};
+
+    DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...);
+
+    if (!stmt.Step(SQLITE_DONE)) {
+      return false;
+    }
+
+    return true;
+  }
+
+  template <typename... Args>
+  static bool SelectOnce(DbStatement& stmt, Args&... args) {
+    int rc = stmt.Step();
+    switch (rc) {
+      case SQLITE_ROW:
+        stmt.ColumnAll(/*out*/args...);
+        return true;
+      case SQLITE_DONE:
+        return false;
+      default:
+        LOG(ERROR) << "Failed to step (" << rc << "): " << sqlite3_errmsg(stmt.db());
+        return false;
+    }
+  }
+};
+
+class Model {
+ public:
+  DbHandle db() const {
+    return db_;
+  }
+
+  Model(DbHandle db) : db_{db} {
+  }
+
+ private:
+  DbHandle db_;
+};
+
+class SchemaModel : public Model {
+ public:
+  static SchemaModel GetOrCreate(std::string location) {
+    int rc = sqlite3_config(SQLITE_CONFIG_LOG, ErrorLogCallback, /*data*/nullptr);
+
+    if (rc != SQLITE_OK) {
+      LOG(FATAL) << "Failed to configure logging";
+    }
+
+    sqlite3* db = nullptr;
+    bool is_deprecated = false;
+    if (location != ":memory:") {
+      // Try to open DB if it already exists.
+      rc = sqlite3_open_v2(location.c_str(), /*out*/&db, SQLITE_OPEN_READWRITE, /*vfs*/nullptr);
+
+      if (rc == SQLITE_OK) {
+        LOG(INFO) << "Opened existing database at '" << location << "'";
+        SchemaModel schema{DbHandle{db}, location};
+        if (schema.Version() == kDbVersion) {
+          return schema;
+        } else {
+          LOG(DEBUG) << "The version is old, reinit the db."
+                     << " old version is "
+                     << schema.Version()
+                     << " and new version is "
+                     << kDbVersion;
+          CleanUpFilesForDb(schema.db());
+          is_deprecated = true;
+       }
+      }
+    }
+
+    if (is_deprecated) {
+      // Remove the db and recreate it.
+      // TODO: migrate to a newer version without deleting the old one.
+      std::filesystem::remove(location.c_str());
+    }
+
+    // Create a new DB if one didn't exist already.
+    rc = sqlite3_open(location.c_str(), /*out*/&db);
+
+    if (rc != SQLITE_OK) {
+      LOG(FATAL) << "Failed to open DB: " << sqlite3_errmsg(db);
+    }
+
+    SchemaModel schema{DbHandle{db}, location};
+    schema.Reinitialize();
+    // TODO: migrate versions upwards when we rev the schema version
+
+    int old_version = schema.Version();
+    LOG(VERBOSE) << "Loaded schema version: " << old_version;
+
+    return schema;
+  }
+
+  void MarkSingleton() {
+    s_singleton_ = db();
+  }
+
+  static DbHandle GetSingleton() {
+    DCHECK(s_singleton_.has_value());
+    return *s_singleton_;
+  }
+
+  void Reinitialize() {
+    const char* sql_to_initialize = R"SQLC0D3(
+        DROP TABLE IF EXISTS schema_versions;
+        DROP TABLE IF EXISTS packages;
+        DROP TABLE IF EXISTS activities;
+        DROP TABLE IF EXISTS app_launch_histories;
+        DROP TABLE IF EXISTS raw_traces;
+        DROP TABLE IF EXISTS prefetch_files;
+)SQLC0D3";
+    char* err_msg = nullptr;
+    int rc = sqlite3_exec(db().get(),
+                          sql_to_initialize,
+                          /*callback*/nullptr,
+                          /*arg*/0,
+                          /*out*/&err_msg);
+    if (rc != SQLITE_OK) {
+      LOG(FATAL) << "Failed to drop tables: " << err_msg ? err_msg : "nullptr";
+    }
+
+    CreateSchema();
+    LOG(INFO) << "Reinitialized database at '" << location_ << "'";
+  }
+
+  int Version() {
+    std::string query = "SELECT MAX(version) FROM schema_versions;";
+    DbStatement stmt = DbStatement::Prepare(db(), query);
+
+    int return_value = 0;
+    if (!DbQueryBuilder::SelectOnce(stmt, /*out*/return_value)) {
+      LOG(ERROR) << "Failed to query schema version";
+      return -1;
+    }
+
+    return return_value;
+  }
+
+ protected:
+  SchemaModel(DbHandle db, std::string location) : Model{db}, location_(location) {
+  }
+
+ private:
+  static std::optional<DbHandle> s_singleton_;
+
+  void CreateSchema() {
+    const char* sql_to_initialize = R"SQLC0D3(
+        CREATE TABLE schema_versions(
+            version INTEGER NOT NULL
+        );
+
+        CREATE TABLE packages(
+            id INTEGER NOT NULL,
+            name TEXT NOT NULL,
+            version INTEGER NOT NULL,
+
+            PRIMARY KEY(id)
+        );
+
+        CREATE TABLE activities(
+            id INTEGER NOT NULL,
+            name TEXT NOT NULL,
+            package_id INTEGER NOT NULL,
+
+            PRIMARY KEY(id),
+            FOREIGN KEY (package_id) REFERENCES packages (id) ON DELETE CASCADE
+        );
+
+        CREATE TABLE app_launch_histories(
+            id INTEGER NOT NULL PRIMARY KEY,
+            activity_id INTEGER NOT NULL,
+            -- 1:Cold, 2:Warm, 3:Hot
+            temperature INTEGER CHECK (temperature IN (1, 2, 3)) NOT NULL,
+            trace_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL,
+            readahead_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL,
+            -- absolute timestamp since epoch
+            intent_started_ns INTEGER CHECK(intent_started_ns IS NULL or intent_started_ns >= 0),
+            -- absolute timestamp since epoch
+            total_time_ns INTEGER CHECK(total_time_ns IS NULL or total_time_ns >= 0),
+            -- absolute timestamp since epoch
+            report_fully_drawn_ns INTEGER CHECK(report_fully_drawn_ns IS NULL or report_fully_drawn_ns >= 0),
+
+            FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE
+        );
+
+        CREATE TABLE raw_traces(
+            id INTEGER NOT NULL PRIMARY KEY,
+            history_id INTEGER NOT NULL,
+            file_path TEXT NOT NULL,
+
+            FOREIGN KEY (history_id) REFERENCES app_launch_histories (id) ON DELETE CASCADE
+        );
+
+        CREATE TABLE prefetch_files(
+          id INTEGER NOT NULL PRIMARY KEY,
+          activity_id INTEGER NOT NULL,
+          file_path TEXT NOT NULL,
+
+          FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE
+        );
+)SQLC0D3";
+
+    char* err_msg = nullptr;
+    int rc = sqlite3_exec(db().get(),
+                          sql_to_initialize,
+                          /*callback*/nullptr,
+                          /*arg*/0,
+                          /*out*/&err_msg);
+
+    if (rc != SQLITE_OK) {
+      LOG(FATAL) << "Failed to create tables: " << err_msg ? err_msg : "nullptr";
+    }
+
+    const char* sql_to_insert_schema_version = R"SQLC0D3(
+      INSERT INTO schema_versions VALUES(%d)
+      )SQLC0D3";
+    rc = sqlite3_exec(db().get(),
+                      android::String8::format(sql_to_insert_schema_version,
+                                               kDbVersion),
+                      /*callback*/nullptr,
+                      /*arg*/0,
+                      /*out*/&err_msg);
+
+    if (rc != SQLITE_OK) {
+      LOG(FATAL) << "Failed to insert the schema version: "
+                 << err_msg ? err_msg : "nullptr";
+    }
+  }
+
+  static void ErrorLogCallback(void *pArg, int iErrCode, const char *zMsg) {
+    LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg;
+  }
+
+  std::string location_;
+};
+
+class PackageModel : public Model {
+ protected:
+  PackageModel(DbHandle db) : Model{db} {
+  }
+
+ public:
+  static std::optional<PackageModel> SelectById(DbHandle db, int id) {
+    ScopedLockDb lock{db};
+    int original_id = id;
+
+    std::string query = "SELECT * FROM packages WHERE id = ?1 LIMIT 1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, id);
+
+    PackageModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::vector<PackageModel> SelectByName(DbHandle db, const char* name) {
+    ScopedLockDb lock{db};
+
+    std::string query = "SELECT * FROM packages WHERE name = ?1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, name);
+
+    std::vector<PackageModel> packages;
+
+    PackageModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) {
+      packages.push_back(p);
+    }
+
+    return packages;
+  }
+
+  static std::optional<PackageModel> SelectByNameAndVersion(DbHandle db,
+                                                            const char* name,
+                                                            int version) {
+    ScopedLockDb lock{db};
+
+    std::string query =
+        "SELECT * FROM packages WHERE name = ?1 AND version = ?2 LIMIT 1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, name, version);
+
+    PackageModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::vector<PackageModel> SelectAll(DbHandle db) {
+    ScopedLockDb lock{db};
+
+    std::string query = "SELECT * FROM packages;";
+    DbStatement stmt = DbStatement::Prepare(db, query);
+
+    std::vector<PackageModel> packages;
+    PackageModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) {
+      packages.push_back(p);
+    }
+
+    return packages;
+  }
+
+  static std::optional<PackageModel> Insert(DbHandle db,
+                                            std::string name,
+                                            int version) {
+    const char* sql = "INSERT INTO packages (name, version) VALUES (?1, ?2);";
+
+    std::optional<int> inserted_row_id =
+        DbQueryBuilder::Insert(db, sql, name, version);
+    if (!inserted_row_id) {
+      return std::nullopt;
+    }
+
+    PackageModel p{db};
+    p.name = name;
+    p.version = version;
+    p.id = *inserted_row_id;
+
+    return p;
+  }
+
+  bool Delete() {
+    const char* sql = "DELETE FROM packages WHERE id = ?";
+
+    return DbQueryBuilder::Delete(db(), sql, id);
+  }
+
+  int id;
+  std::string name;
+  int version;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const PackageModel& p) {
+  os << "PackageModel{id=" << p.id << ",name=" << p.name << ",";
+  os << "version=";
+  os << p.version;
+  os << "}";
+  return os;
+}
+
+class ActivityModel : public Model {
+ protected:
+  ActivityModel(DbHandle db) : Model{db} {
+  }
+
+ public:
+  static std::optional<ActivityModel> SelectById(DbHandle db, int id) {
+    ScopedLockDb lock{db};
+    int original_id = id;
+
+    std::string query = "SELECT * FROM activities WHERE id = ? LIMIT 1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, id);
+
+    ActivityModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::optional<ActivityModel> SelectByNameAndPackageId(DbHandle db,
+                                                               const char* name,
+                                                               int package_id) {
+    ScopedLockDb lock{db};
+
+    std::string query = "SELECT * FROM activities WHERE name = ? AND package_id = ? LIMIT 1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, name, package_id);
+
+    ActivityModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::vector<ActivityModel> SelectByPackageId(DbHandle db,
+                                                      int package_id) {
+    ScopedLockDb lock{db};
+
+    std::string query = "SELECT * FROM activities WHERE package_id = ?;";
+    DbStatement stmt = DbStatement::Prepare(db, query, package_id);
+
+    std::vector<ActivityModel> activities;
+    ActivityModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) {
+      activities.push_back(p);
+    }
+
+    return activities;
+  }
+
+  static std::optional<ActivityModel> Insert(DbHandle db,
+                                             std::string name,
+                                             int package_id) {
+    const char* sql = "INSERT INTO activities (name, package_id) VALUES (?1, ?2);";
+
+    std::optional<int> inserted_row_id =
+        DbQueryBuilder::Insert(db, sql, name, package_id);
+    if (!inserted_row_id) {
+      return std::nullopt;
+    }
+
+    ActivityModel p{db};
+    p.id = *inserted_row_id;
+    p.name = name;
+    p.package_id = package_id;
+
+    return p;
+  }
+
+  // Try to select by package_name+activity_name, otherwise insert into both tables.
+  // Package version is ignored for selects.
+  static std::optional<ActivityModel> SelectOrInsert(
+      DbHandle db,
+      std::string package_name,
+      int package_version,
+      std::string activity_name) {
+    std::optional<PackageModel> package = PackageModel::SelectByNameAndVersion(db,
+                                                                               package_name.c_str(),
+                                                                               package_version);
+    if (!package) {
+      package = PackageModel::Insert(db, package_name, package_version);
+      DCHECK(package.has_value());
+    }
+
+    std::optional<ActivityModel> activity =
+        ActivityModel::SelectByNameAndPackageId(db,
+                                                activity_name.c_str(),
+                                                package->id);
+    if (!activity) {
+      activity = Insert(db, activity_name, package->id);
+      // XX: should we really return an optional here? This feels like it should never fail.
+    }
+
+    return activity;
+  }
+
+  int id;
+  std::string name;
+  int package_id;  // PackageModel::id
+};
+
+inline std::ostream& operator<<(std::ostream& os, const ActivityModel& p) {
+  os << "ActivityModel{id=" << p.id << ",name=" << p.name << ",";
+  os << "package_id=" << p.package_id << "}";
+  return os;
+}
+
+class AppLaunchHistoryModel : public Model {
+ protected:
+  AppLaunchHistoryModel(DbHandle db) : Model{db} {
+  }
+
+ public:
+  enum class Temperature : int32_t {
+    kUninitialized = -1,  // Note: Not a valid SQL value.
+    kCold = 1,
+    kWarm = 2,
+    kHot = 3,
+  };
+
+  static std::optional<AppLaunchHistoryModel> SelectById(DbHandle db, int id) {
+    ScopedLockDb lock{db};
+    int original_id = id;
+
+    std::string query = "SELECT * FROM app_launch_histories WHERE id = ? LIMIT 1;";
+    DbStatement stmt = DbStatement::Prepare(db, query, id);
+
+    AppLaunchHistoryModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt,
+                                    p.id,
+                                    p.activity_id,
+                                    p.temperature,
+                                    p.trace_enabled,
+                                    p.readahead_enabled,
+                                    p.intent_started_ns,
+                                    p.total_time_ns,
+                                    p.report_fully_drawn_ns)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  // Selects the activity history for an activity id.
+  // The requirements are:
+  // * Should be cold run.
+  // * Pefetto trace is enabled.
+  // * intent_start_ns is *NOT* null.
+  static std::vector<AppLaunchHistoryModel> SelectActivityHistoryForCompile(
+      DbHandle db,
+      int activity_id) {
+    ScopedLockDb lock{db};
+    std::string query = "SELECT * FROM app_launch_histories "
+                        "WHERE activity_id = ?1 AND"
+                        "  temperature = 1 AND"
+                        "  trace_enabled = TRUE AND"
+                        "  intent_started_ns IS NOT NULL;";
+    DbStatement stmt = DbStatement::Prepare(db, query, activity_id);
+    std::vector<AppLaunchHistoryModel> result;
+
+    AppLaunchHistoryModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt,
+                                      p.id,
+                                      p.activity_id,
+                                      p.temperature,
+                                      p.trace_enabled,
+                                      p.readahead_enabled,
+                                      p.intent_started_ns,
+                                      p.total_time_ns,
+                                      p.report_fully_drawn_ns)) {
+      result.push_back(p);
+    }
+    return result;
+  }
+
+  static std::optional<AppLaunchHistoryModel> Insert(DbHandle db,
+                                                     int activity_id,
+                                                     AppLaunchHistoryModel::Temperature temperature,
+                                                     bool trace_enabled,
+                                                     bool readahead_enabled,
+                                                     std::optional<uint64_t> intent_started_ns,
+                                                     std::optional<uint64_t> total_time_ns,
+                                                     std::optional<uint64_t> report_fully_drawn_ns)
+  {
+    const char* sql = "INSERT INTO app_launch_histories (activity_id, temperature, trace_enabled, "
+                                                        "readahead_enabled, intent_started_ns, "
+                                                        "total_time_ns, report_fully_drawn_ns) "
+                      "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7);";
+
+    std::optional<int> inserted_row_id =
+        DbQueryBuilder::Insert(db,
+                               sql,
+                               activity_id,
+                               temperature,
+                               trace_enabled,
+                               readahead_enabled,
+                               intent_started_ns,
+                               total_time_ns,
+                               report_fully_drawn_ns);
+    if (!inserted_row_id) {
+      return std::nullopt;
+    }
+
+    AppLaunchHistoryModel p{db};
+    p.id = *inserted_row_id;
+    p.activity_id = activity_id;
+    p.temperature = temperature;
+    p.trace_enabled = trace_enabled;
+    p.readahead_enabled = readahead_enabled;
+    p.intent_started_ns = intent_started_ns;
+    p.total_time_ns = total_time_ns;
+    p.report_fully_drawn_ns = report_fully_drawn_ns;
+
+    return p;
+  }
+
+  static bool UpdateReportFullyDrawn(DbHandle db,
+                                     int history_id,
+                                     uint64_t report_fully_drawn_ns)
+  {
+    const char* sql = "UPDATE app_launch_histories "
+                      "SET report_fully_drawn_ns = ?1 "
+                      "WHERE id = ?2;";
+
+    bool result = DbQueryBuilder::Update(db, sql, report_fully_drawn_ns, history_id);
+
+    if (!result) {
+      LOG(ERROR)<< "Failed to update history_id:"<< history_id
+                << ", report_fully_drawn_ns: " << report_fully_drawn_ns;
+    }
+    return result;
+  }
+
+  int id;
+  int activity_id;  // ActivityModel::id
+  Temperature temperature = Temperature::kUninitialized;
+  bool trace_enabled;
+  bool readahead_enabled;
+  std::optional<uint64_t> intent_started_ns;
+  std::optional<uint64_t> total_time_ns;
+  std::optional<uint64_t> report_fully_drawn_ns;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const AppLaunchHistoryModel& p) {
+  os << "AppLaunchHistoryModel{id=" << p.id << ","
+     << "activity_id=" << p.activity_id << ","
+     << "temperature=" << static_cast<int>(p.temperature) << ","
+     << "trace_enabled=" << p.trace_enabled << ","
+     << "readahead_enabled=" << p.readahead_enabled << ","
+     << "intent_started_ns=";
+  if (p.intent_started_ns) {
+    os << *p.intent_started_ns;
+  } else {
+    os << "(nullopt)";
+  }
+  os << ",";
+  os << "total_time_ns=";
+  if (p.total_time_ns) {
+    os << *p.total_time_ns;
+  } else {
+    os << "(nullopt)";
+  }
+  os << ",";
+  os << "report_fully_drawn_ns=";
+  if (p.report_fully_drawn_ns) {
+    os << *p.report_fully_drawn_ns;
+  } else {
+    os << "(nullopt)";
+  }
+  os << "}";
+  return os;
+}
+
+class RawTraceModel : public Model {
+ protected:
+  RawTraceModel(DbHandle db) : Model{db} {
+  }
+
+ public:
+
+  // Return raw_traces, sorted ascending by the id.
+  static std::vector<RawTraceModel> SelectByVersionedComponentName(DbHandle db,
+                                                                   VersionedComponentName vcn) {
+    ScopedLockDb lock{db};
+
+    const char* sql =
+      "SELECT raw_traces.id, raw_traces.history_id, raw_traces.file_path "
+      "FROM raw_traces "
+      "INNER JOIN app_launch_histories ON raw_traces.history_id = app_launch_histories.id "
+      "INNER JOIN activities ON activities.id = app_launch_histories.activity_id "
+      "INNER JOIN packages ON packages.id = activities.package_id "
+      "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?"
+      "ORDER BY raw_traces.id ASC";
+
+    DbStatement stmt = DbStatement::Prepare(db,
+                                            sql,
+                                            vcn.GetPackage(),
+                                            vcn.GetActivity(),
+                                            vcn.GetVersion());
+
+    std::vector<RawTraceModel> results;
+
+    RawTraceModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) {
+      results.push_back(p);
+    }
+
+    return results;
+  }
+
+  static std::optional<RawTraceModel> SelectByHistoryId(DbHandle db, int history_id) {
+    ScopedLockDb lock{db};
+
+    const char* sql =
+      "SELECT id, history_id, file_path "
+      "FROM raw_traces "
+      "WHERE history_id = ?1 "
+      "LIMIT 1;";
+
+    DbStatement stmt = DbStatement::Prepare(db, sql, history_id);
+
+    RawTraceModel p{db};
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::optional<RawTraceModel> Insert(DbHandle db,
+                                             int history_id,
+                                             std::string file_path) {
+    const char* sql = "INSERT INTO raw_traces (history_id, file_path) VALUES (?1, ?2);";
+
+    std::optional<int> inserted_row_id =
+        DbQueryBuilder::Insert(db, sql, history_id, file_path);
+    if (!inserted_row_id) {
+      return std::nullopt;
+    }
+
+    RawTraceModel p{db};
+    p.id = *inserted_row_id;
+    p.history_id = history_id;
+    p.file_path = file_path;
+
+    return p;
+  }
+
+  bool Delete() {
+    const char* sql = "DELETE FROM raw_traces WHERE id = ?";
+
+    return DbQueryBuilder::Delete(db(), sql, id);
+  }
+
+  int id;
+  int history_id;
+  std::string file_path;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const RawTraceModel& p) {
+  os << "RawTraceModel{id=" << p.id << ",history_id=" << p.history_id << ",";
+  os << "file_path=" << p.file_path << "}";
+  return os;
+}
+
+class PrefetchFileModel : public Model {
+ protected:
+  PrefetchFileModel(DbHandle db) : Model{db} {
+  }
+
+ public:
+  static std::optional<PrefetchFileModel> SelectByVersionedComponentName(
+      DbHandle db,
+      VersionedComponentName vcn) {
+    ScopedLockDb lock{db};
+
+    const char* sql =
+      "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path "
+      "FROM prefetch_files "
+      "INNER JOIN activities ON activities.id = prefetch_files.activity_id "
+      "INNER JOIN packages ON packages.id = activities.package_id "
+      "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?";
+
+    DbStatement stmt = DbStatement::Prepare(db,
+                                            sql,
+                                            vcn.GetPackage(),
+                                            vcn.GetActivity(),
+                                            vcn.GetVersion());
+
+    PrefetchFileModel p{db};
+
+    if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) {
+      return std::nullopt;
+    }
+
+    return p;
+  }
+
+  static std::vector<PrefetchFileModel> SelectAll(DbHandle db) {
+    ScopedLockDb lock{db};
+
+    std::string query =
+      "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path "
+      "FROM prefetch_files";
+    DbStatement stmt = DbStatement::Prepare(db, query);
+
+    std::vector<PrefetchFileModel> prefetch_files;
+    PrefetchFileModel p{db};
+    while (DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) {
+      prefetch_files.push_back(p);
+    }
+
+    return prefetch_files;
+  }
+
+  static std::optional<PrefetchFileModel> Insert(DbHandle db,
+                                                 int activity_id,
+                                                 std::string file_path) {
+    const char* sql = "INSERT INTO prefetch_files (activity_id, file_path) VALUES (?1, ?2);";
+
+    std::optional<int> inserted_row_id =
+        DbQueryBuilder::Insert(db, sql, activity_id, file_path);
+    if (!inserted_row_id) {
+      return std::nullopt;
+    }
+
+    PrefetchFileModel p{db};
+    p.id = *inserted_row_id;
+    p.activity_id = activity_id;
+    p.file_path = file_path;
+
+    return p;
+  }
+
+  bool Delete() {
+    const char* sql = "DELETE FROM prefetch_files WHERE id = ?";
+
+    return DbQueryBuilder::Delete(db(), sql, id);
+  }
+
+  int id;
+  int activity_id;
+  std::string file_path;
+};
+
+inline std::ostream& operator<<(std::ostream& os, const PrefetchFileModel& p) {
+  os << "PrefetchFileModel{id=" << p.id << ",activity_id=" << p.activity_id << ",";
+  os << "file_path=" << p.file_path << "}";
+  return os;
+}
+
+}  // namespace iorap::db
+
+#endif  // IORAP_SRC_DB_MODELS_H_
diff --git a/src/inode2filename/data_source.cc b/src/inode2filename/data_source.cc
new file mode 100644
index 0000000..5fcce73
--- /dev/null
+++ b/src/inode2filename/data_source.cc
@@ -0,0 +1,187 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "inode2filename/data_source.h"
+
+#include "common/cmd_utils.h"
+#include "inode2filename/inode_resolver.h"
+#include "inode2filename/search_directories.h"
+
+#include <android-base/logging.h>
+
+#include <fstream>
+#include <stdio.h>
+
+namespace rx = rxcpp;
+
+namespace iorap::inode2filename {
+
+std::vector<std::string> ToArgs(DataSourceKind data_source_kind) {
+  const char* value = nullptr;
+
+  switch (data_source_kind) {
+    case DataSourceKind::kDiskScan:
+      value = "diskscan";
+      break;
+    case DataSourceKind::kTextCache:
+      value = "textcache";
+      break;
+    case DataSourceKind::kBpf:
+      value = "bpf";
+      break;
+  }
+
+  std::vector<std::string> args;
+  iorap::common::AppendNamedArg(args, "--data-source", value);
+  return args;
+}
+
+std::vector<std::string> ToArgs(const DataSourceDependencies& deps) {
+  std::vector<std::string> args;
+
+  iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.data_source));
+  // intentionally skip system_call; it does not have a command line equivalent
+  iorap::common::AppendNamedArgRepeatedly(args, "--root", deps.root_directories);
+
+  if (deps.text_cache_filename) {
+    iorap::common::AppendNamedArg(args, "--textcache", *(deps.text_cache_filename));
+  }
+
+  return args;
+}
+
+class DiskScanDataSource : public DataSource {
+ public:
+  DiskScanDataSource(DataSourceDependencies dependencies)
+    : DataSource(std::move(dependencies)) {
+    DCHECK_NE(dependencies_.root_directories.size(), 0u) << "Root directories can't be empty";
+  }
+
+  virtual rxcpp::observable<InodeResult> EmitInodes() const override {
+    SearchDirectories searcher{/*borrow*/dependencies_.system_call};
+    return searcher.ListAllFilenames(dependencies_.root_directories);
+  }
+
+  // Since not all Inodes emitted are the ones searched for, doing additional stat(2) calls here
+  // would be redundant.
+  //
+  // We set the device number to a dummy value here (-1) so that InodeResolver
+  // can fill it in later with stat(2). This is effectively the same thing as always doing
+  // verification.
+  virtual bool ResultIncludesDeviceNumber() const { return false; };
+
+  virtual ~DiskScanDataSource() {}
+};
+
+static inline void LeftTrim(/*inout*/std::string& s) {
+  s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
+    return !std::isspace(ch);
+  }));
+}
+
+class TextCacheDataSource : public DataSource {
+ public:
+  TextCacheDataSource(DataSourceDependencies dependencies)
+    : DataSource(std::move(dependencies)) {
+    DCHECK(dependencies_.text_cache_filename.has_value()) << "Must have text cache filename";
+  }
+
+  virtual rxcpp::observable<InodeResult> EmitInodes() const override {
+    const std::string& file_name = *dependencies_.text_cache_filename;
+
+    return rx::observable<>::create<InodeResult>(
+      [file_name](rx::subscriber<InodeResult> dest) {
+        LOG(VERBOSE) << "TextCacheDataSource (lambda)";
+
+        std::ifstream ifs{file_name};
+
+        if (!ifs.is_open()) {
+          dest.on_error(rxcpp::util::make_error_ptr(
+              std::ios_base::failure("Could not open specified text cache filename")));
+          return;
+        }
+
+        while (dest.is_subscribed() && ifs) {
+          LOG(VERBOSE) << "TextCacheDataSource (read line)";
+          // TODO.
+
+          uint64_t device_number = 0;
+          uint64_t inode_number = 0;
+          uint64_t file_size = 0;
+
+          // Parse lines of form:
+          //   "$device_number $inode $filesize $filename..."
+          // This format conforms to system/extras/pagecache/pagecache.py
+
+          ifs >> device_number;
+          ifs >> inode_number;
+          ifs >> file_size;
+          (void)file_size;  // Not used in iorapd.
+
+          std::string value_filename;
+          std::getline(/*inout*/ifs, /*out*/value_filename);
+
+          if (value_filename.empty()) {
+            // Ignore empty lines.
+            continue;
+          }
+
+          // getline, unlike ifstream, does not ignore spaces.
+          // There's always at least 1 space in a textcache output file.
+          // However, drop *all* spaces on the left since filenames starting with a space are
+          // ambiguous to us.
+          LeftTrim(/*inout*/value_filename);
+
+          Inode inode = Inode::FromDeviceAndInode(static_cast<dev_t>(device_number),
+                                                  static_cast<ino_t>(inode_number));
+
+          LOG(VERBOSE) << "TextCacheDataSource (on_next) " << inode << "->" << value_filename;
+          dest.on_next(InodeResult::makeSuccess(inode, value_filename));
+        }
+
+        dest.on_completed();
+      }
+    );
+
+    // TODO: plug into the filtering and verification graph.
+  }
+
+  virtual ~TextCacheDataSource() {}
+};
+
+DataSource::DataSource(DataSourceDependencies dependencies)
+  : dependencies_{std::move(dependencies)} {
+  DCHECK(dependencies_.system_call != nullptr);
+}
+
+std::shared_ptr<DataSource> DataSource::Create(DataSourceDependencies dependencies) {
+  switch (dependencies.data_source) {
+    case DataSourceKind::kDiskScan:
+      return std::shared_ptr<DataSource>{new DiskScanDataSource{std::move(dependencies)}};
+    case DataSourceKind::kTextCache:
+      return std::shared_ptr<DataSource>{new TextCacheDataSource{std::move(dependencies)}};
+    case DataSourceKind::kBpf:
+      // TODO: BPF-based data source.
+      LOG(FATAL) << "Not implemented yet";
+      return nullptr;
+    default:
+      LOG(FATAL) << "Invalid data source value";
+      return nullptr;
+  }
+}
+
+void DataSource::StartRecording() {}
+void DataSource::StopRecording() {}
+
+}  // namespace iorap::inode2filename
diff --git a/src/inode2filename/data_source.h b/src/inode2filename/data_source.h
new file mode 100644
index 0000000..36c8383
--- /dev/null
+++ b/src/inode2filename/data_source.h
@@ -0,0 +1,74 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_
+#define IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_
+
+#include "inode2filename/inode_result.h"
+#include "inode2filename/system_call.h"
+
+#include <rxcpp/rx.hpp>
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+namespace iorap::inode2filename {
+
+enum class DataSourceKind {
+  kDiskScan,
+  kTextCache,
+  kBpf,
+};
+
+std::vector<std::string> ToArgs(DataSourceKind data_source_kind);
+
+struct DataSourceDependencies {
+  DataSourceKind data_source = DataSourceKind::kDiskScan;
+  borrowed<SystemCall*> system_call = nullptr;
+
+  // kDiskScan-specific options. Other data sources ignore these fields.
+  std::vector<std::string> root_directories;
+  // kTextCache-specific options. Other data sources ignore these fields.
+  std::optional<std::string> text_cache_filename;
+};
+
+std::vector<std::string> ToArgs(const DataSourceDependencies& deps);
+
+class DataSource : std::enable_shared_from_this<DataSource> {
+ public:
+  static std::shared_ptr<DataSource> Create(DataSourceDependencies dependencies);
+
+  virtual void StartRecording();  // XX: feels like this should be BPF-specific.
+  virtual void StopRecording();
+
+  // Emit all inode->filename mappings (i.e. an infinite lazy list).
+  // The specific order is determined by the extra dependency options.
+  //
+  // The work must terminate if all subscriptions are removed.
+  virtual rxcpp::observable<InodeResult> EmitInodes() const = 0;
+
+  // Does the InodeResult include a valid device number?
+  // If returning false, the InodeResolver fills in the missing device number with stat(2).
+  virtual bool ResultIncludesDeviceNumber() const { return true; };
+
+ protected:
+  virtual ~DataSource() {}
+  DataSource(DataSourceDependencies dependencies);
+  DataSourceDependencies dependencies_;
+};
+
+}  // namespace iorap::inode2filename
+
+#endif  // IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_
diff --git a/src/inode2filename/inode.cc b/src/inode2filename/inode.cc
index cdb2325..cc665ea 100644
--- a/src/inode2filename/inode.cc
+++ b/src/inode2filename/inode.cc
@@ -79,3 +79,9 @@ bool Inode::Parse(const std::string& str, Inode* out, std::string* error_msg) {
 }
 
 }  // namespace iorap::inode2filename
+
+// Ensure our pollution-free aliases match the typedefs in the system headers.
+static_assert(std::is_same_v<iorap::inode2filename::dev_t, dev_t>);
+static_assert(std::is_same_v<iorap::inode2filename::ino_t, ino_t>);
+
+// TODO: consider some tests for major, minor, etc.
\ No newline at end of file
diff --git a/src/inode2filename/inode.h b/src/inode2filename/inode.h
index 2dd5611..f336a14 100644
--- a/src/inode2filename/inode.h
+++ b/src/inode2filename/inode.h
@@ -23,22 +23,97 @@
 
 namespace iorap::inode2filename {
 
+// Avoid polluting headers.
+#if defined(__ANDROID__)
+#  if !defined(__LP64__)
+/* This historical accident means that we had a 32-bit dev_t on 32-bit architectures. */
+using dev_t = uint32_t;
+#  else
+using dev_t = uint64_t;
+#  endif
+using ino_t = unsigned long;
+#else
+#  if !defined(__x86_64__)
+using dev_t = unsigned long long;
+using ino_t = unsigned long long;
+#  else
+using dev_t = unsigned long;
+using ino_t = unsigned long;
+#  endif
+#endif
+
+#ifdef makedev
+#undef makedev
+#endif
+
+/** Combines `major` and `minor` into a device number. */
+constexpr inline dev_t makedev(unsigned int major, unsigned int minor) {
+  return
+      (((major) & 0xfffff000ULL) << 32) | (((major) & 0xfffULL) << 8) |
+      (((minor) & 0xffffff00ULL) << 12) | (((minor) & 0xffULL));
+}
+
+#ifdef major
+#undef major
+#endif
+
+/** Extracts the major part of a device number. */
+constexpr inline unsigned int major(dev_t dev) {
+  return
+      ((unsigned) ((((unsigned long long) (dev) >> 32) & 0xfffff000) | (((dev) >> 8) & 0xfff)));
+}
+
+#ifdef minor
+#undef minor
+#endif
+
+/** Extracts the minor part of a device number. */
+constexpr inline unsigned int minor(dev_t dev) {
+  return
+      ((unsigned) ((((dev) >> 12) & 0xffffff00) | ((dev) & 0xff)));
+};
+// Note: above definitions copied from sysmacros.h, to avoid polluting global namespace in a header.
+
+/*
+ * A convenient datum representing a (dev_t, ino_t) tuple.
+ *
+ * ino_t values may be reused across different devices (e.g. different partitions),
+ * so we need the full tuple to uniquely identify an inode on a system.
+ */
 struct Inode {
   size_t device_major;  // dev_t = makedev(major, minor)
   size_t device_minor;
   size_t inode;         // ino_t = inode
 
+  // "Major:minor:inode" OR "dev_t@inode"
   static bool Parse(const std::string& str, /*out*/Inode* out, /*out*/std::string* error_msg);
 
-  bool operator==(const Inode& rhs) const {
+  constexpr bool operator==(const Inode& rhs) const {
     return device_major == rhs.device_major &&
         device_minor == rhs.device_minor &&
         inode == rhs.inode;
   }
 
-  bool operator!=(const Inode& rhs) const {
+  constexpr bool operator!=(const Inode& rhs) const {
     return !(*this == rhs);
   }
+
+  Inode() = default;
+  constexpr Inode(size_t device_major, size_t device_minor, size_t inode)
+    : device_major{device_major}, device_minor{device_minor}, inode{inode} {
+  }
+
+  static constexpr Inode FromDeviceAndInode(dev_t dev, ino_t inode) {
+    return Inode{major(dev), minor(dev), static_cast<size_t>(inode)};
+  }
+
+  constexpr dev_t GetDevice() const {
+    return makedev(device_major, device_minor);
+  }
+
+  constexpr ino_t GetInode() const {
+    return static_cast<ino_t>(inode);
+  }
 };
 
 inline std::ostream& operator<<(std::ostream& os, const Inode& inode) {
@@ -62,4 +137,14 @@ namespace std {
   };
 }  // namespace std
 
+namespace rxcpp {
+template <class T, typename>
+struct filtered_hash;
+
+// support for the 'distinct' rx operator.
+template <>
+struct filtered_hash<iorap::inode2filename::Inode, void> : std::hash<iorap::inode2filename::Inode> {
+};
+}  // namespace rxcpp
+
 #endif  // IORAP_SRC_INODE2FILENAME_INODE_H_
diff --git a/src/inode2filename/inode_resolver.cc b/src/inode2filename/inode_resolver.cc
new file mode 100644
index 0000000..5e2945d
--- /dev/null
+++ b/src/inode2filename/inode_resolver.cc
@@ -0,0 +1,207 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "inode2filename/inode_resolver.h"
+
+#include "common/cmd_utils.h"
+#include "inode2filename/out_of_process_inode_resolver.h"
+#include "inode2filename/search_directories.h"
+
+#include <android-base/logging.h>
+
+#include <fstream>
+#include <stdio.h>
+
+namespace rx = rxcpp;
+
+namespace iorap::inode2filename {
+
+std::vector<std::string> ToArgs(ProcessMode process_mode) {
+  const char* value = nullptr;
+
+  switch (process_mode) {
+    case ProcessMode::kInProcessDirect:
+      value = "in";
+      break;
+    case ProcessMode::kInProcessIpc:
+      value = "in-ipc";
+      break;
+    case ProcessMode::kOutOfProcessIpc:
+      value = "out";
+      break;
+  }
+
+  std::vector<std::string> args;
+  iorap::common::AppendNamedArg(args, "--process-mode", value);
+  return args;
+}
+
+std::vector<std::string> ToArgs(VerifyKind verify_kind) {
+  const char* value = nullptr;
+
+  switch (verify_kind) {
+    case VerifyKind::kNone:
+      value = "none";
+      break;
+    case VerifyKind::kStat:
+      value = "stat";
+      break;
+  }
+
+  std::vector<std::string> args;
+  iorap::common::AppendNamedArg(args, "--verify", value);
+  return args;
+}
+
+std::vector<std::string> ToArgs(const InodeResolverDependencies& deps) {
+  std::vector<std::string> args = ToArgs(*static_cast<const DataSourceDependencies*>(&deps));
+  iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.process_mode));
+  iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.verify));
+
+  return args;
+}
+
+struct InodeResolver::Impl {
+  Impl(InodeResolverDependencies dependencies)
+    : dependencies_{std::move(dependencies)} {
+    DCHECK(dependencies_.system_call != nullptr);
+    data_source_ = DataSource::Create(/*downcast*/dependencies_);
+  }
+  Impl(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source)
+    : dependencies_{std::move(dependencies)} {
+    DCHECK(dependencies_.system_call != nullptr);
+    data_source_ = std::move(data_source);
+    DCHECK(data_source_ != nullptr);
+  }
+  InodeResolverDependencies dependencies_;
+  std::shared_ptr<DataSource> data_source_;
+};
+
+InodeResolver::InodeResolver(InodeResolverDependencies dependencies)
+  : impl_(new InodeResolver::Impl{std::move(dependencies)}) {
+}
+
+InodeResolver::InodeResolver(InodeResolverDependencies dependencies,
+                             std::shared_ptr<DataSource> data_source)
+  : impl_(new InodeResolver::Impl{std::move(dependencies), std::move(data_source)}) {
+}
+
+std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies) {
+  if (dependencies.process_mode == ProcessMode::kInProcessDirect) {
+    return std::shared_ptr<InodeResolver>{
+        new InodeResolver{std::move(dependencies)}};
+  } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) {
+    return std::shared_ptr<InodeResolver>{
+        new OutOfProcessInodeResolver{std::move(dependencies)}};
+  } else {
+    CHECK(false);
+  }
+  return nullptr;
+}
+
+std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies,
+                                                     std::shared_ptr<DataSource> data_source) {
+  if (dependencies.process_mode == ProcessMode::kInProcessDirect) {
+    return std::shared_ptr<InodeResolver>{
+        new InodeResolver{std::move(dependencies), std::move(data_source)}};
+  } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) {
+    CHECK(false);  // directly providing a DataSource only makes sense in-process
+  } else {
+    CHECK(false);
+  }
+  return nullptr;
+}
+
+rxcpp::observable<InodeResult>
+    InodeResolver::FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const {
+
+  // It's inefficient to search for inodes until the full search list is available,
+  // so first reduce to a vector so we can access all the inodes simultaneously.
+  return inodes.reduce(std::vector<Inode>{},
+                       [](std::vector<Inode> vec, Inode inode) {
+                         vec.push_back(inode);
+                         return vec;
+                       },
+                       [](std::vector<Inode> v) {
+                         return v;  // TODO: use an identity function
+                       })
+    .flat_map([self=shared_from_this()](std::vector<Inode> vec) {
+      // All borrowed values (e.g. SystemCall) must outlive the observable.
+      return self->FindFilenamesFromInodes(vec);
+    }
+  );
+}
+
+rxcpp::observable<InodeResult>
+InodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const {
+  const DataSource& data_source = *impl_->data_source_;
+  const InodeResolverDependencies& dependencies = impl_->dependencies_;
+
+  // Get lazy list of inodes from the data source.
+  rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes();
+
+  // Filter it according to the source+dependency requirements.
+  // Unsubscribe from 'all_inodes' early if all inodes are matched early.
+  const bool needs_device_number = !data_source.ResultIncludesDeviceNumber();
+  const bool needs_verification = dependencies.verify == VerifyKind::kStat;
+  SearchDirectories search{impl_->dependencies_.system_call};
+  return search.FilterFilenamesForSpecificInodes(all_inodes,
+                                                 inodes,
+                                                 needs_device_number,
+                                                 needs_verification);
+}
+
+rxcpp::observable<InodeResult>
+InodeResolver::EmitAll() const {
+  const DataSource& data_source = *impl_->data_source_;
+  const InodeResolverDependencies& dependencies = impl_->dependencies_;
+
+  // Get lazy list of inodes from the data source.
+  rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes();
+
+  // Apply verification and fill-in missing device numbers.
+  const bool needs_device_number = !data_source.ResultIncludesDeviceNumber();
+  const bool needs_verification = dependencies.verify == VerifyKind::kStat;
+  SearchDirectories search{impl_->dependencies_.system_call};
+  return search.EmitAllFilenames(all_inodes,
+                                 needs_device_number,
+                                 needs_verification);
+}
+
+InodeResolver::~InodeResolver() {
+  // std::unique_ptr requires complete types, but we hide the definition in the header.
+  delete impl_;
+  // XX: Does this work if we just force the dtor definition into the .cc file with a unique_ptr?
+}
+
+InodeResolverDependencies& InodeResolver::GetDependencies() {
+  return impl_->dependencies_;
+}
+
+const InodeResolverDependencies& InodeResolver::GetDependencies() const {
+  return impl_->dependencies_;
+}
+
+void InodeResolver::StartRecording() {
+  impl_->data_source_->StartRecording();
+}
+
+void InodeResolver::StopRecording() {
+  impl_->data_source_->StopRecording();
+}
+
+// TODO: refactor more code from search_directories into this file.
+// XX: do we also need a DataSink class? lets see if recording gets more complicated.
+
+}  // namespace iorap::inode2filename
diff --git a/src/inode2filename/inode_resolver.h b/src/inode2filename/inode_resolver.h
new file mode 100644
index 0000000..218640f
--- /dev/null
+++ b/src/inode2filename/inode_resolver.h
@@ -0,0 +1,121 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_
+#define IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_
+
+#include "common/expected.h"
+#include "inode2filename/data_source.h"
+#include "inode2filename/inode.h"
+#include "inode2filename/system_call.h"
+
+#include <rxcpp/rx.hpp>
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+namespace iorap::inode2filename {
+
+enum class ProcessMode {
+  // Test modes:
+  kInProcessDirect,  // Execute the code directly.
+  kInProcessIpc,     // Execute code via IPC layer using multiple threads.
+  // Shipping mode:
+  kOutOfProcessIpc,  // Execute code via fork+exec with IPC.
+
+  // Note: in-process system-wide stat(2)/readdir/etc is blocked by selinux.
+  // Attempting to call the test modes will fail with -EPERM.
+  //
+  // Use fork+exec mode in shipping configurations, which spawns inode2filename
+  // as a separate command.
+};
+
+enum class VerifyKind {
+  kNone,
+  kStat,
+};
+
+std::vector<std::string> ToArgs(VerifyKind verify_kind);
+
+struct InodeResolverDependencies : public DataSourceDependencies {
+  ProcessMode process_mode = ProcessMode::kInProcessDirect;
+  VerifyKind verify{VerifyKind::kStat};  // Filter out results that aren't up-to-date with stat(2) ?
+};
+
+std::vector<std::string> ToArgs(const InodeResolverDependencies& deps);
+
+// Create an rx-observable chain that allows searching for inode->filename mappings given
+// a set of inode keys.
+class InodeResolver : public std::enable_shared_from_this<InodeResolver> {
+ public:
+  static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies,
+                                               std::shared_ptr<DataSource> data_source);  // nonnull
+
+  // Convenience function for above: Uses DataSource::Create for the data-source.
+  static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies);
+
+  // Search the associated data source to map each inode in 'inodes' to a file path.
+  //
+  // Observes DataSource::EmitInodes(), which is unsubscribed from early once all inodes are found.
+  //
+  // Notes:
+  // * Searching does not begin until all 'inodes' are observed to avoid rescanning.
+  // * If the observable is unsubscribed to prior to completion, searching will halt.
+  //
+  // Post-condition: All emitted results are in inodes, and all inodes are in emitted results.
+  rxcpp::observable<InodeResult>
+      FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const;
+      // TODO: feels like we could turn this into a general helper?
+  // Convenience function for above.
+  virtual rxcpp::observable<InodeResult>
+      FindFilenamesFromInodes(std::vector<Inode> inodes) const;
+
+  // Enumerate *all* inodes available from the data source, associating it with a filepath.
+  //
+  // Depending on the data source (e.g. diskscan), it can take a very long time for this observable
+  // to complete. The intended use-case is for development/debugging, not for production.
+  //
+  // Observes DataSource::EmitInodes() until it reaches #on_completed.
+  //
+  // Notes:
+  // * If the observable is unsubscribed to prior to completion, searching will halt.
+  virtual rxcpp::observable<InodeResult>
+      EmitAll() const;
+
+  // Notifies the DataSource to begin recording.
+  // Some DataSources may be continuously refreshing, but only if recording is enabled.
+  // To get the most up-to-date data, toggle recording before reading the inodes out.
+  void StartRecording();  // XX: feels like this should be BPF-specific.
+
+  // Notifies the DataSource to stop recording.
+  // Some DataSources may be continuously refreshing, but only if recording is enabled.
+  // The snapshot of data returned by e.g. #EmitAll would then not change outside of recording.
+  void StopRecording();
+
+  virtual ~InodeResolver();
+ private:
+  struct Impl;
+  Impl* impl_;
+
+ protected:
+  InodeResolver(InodeResolverDependencies dependencies);
+  InodeResolver(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source);
+  InodeResolverDependencies& GetDependencies();
+  const InodeResolverDependencies& GetDependencies() const;
+};
+
+}
+
+#endif  // IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_
diff --git a/src/inode2filename/inode_result.cc b/src/inode2filename/inode_result.cc
new file mode 100644
index 0000000..139b038
--- /dev/null
+++ b/src/inode2filename/inode_result.cc
@@ -0,0 +1,62 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "inode2filename/inode_result.h"
+
+#include <string.h>
+
+namespace iorap::inode2filename {
+
+std::optional<std::string_view> InodeResult::ErrorMessage() const {
+  if (data) {
+    return std::nullopt;
+  }
+
+  const int err_no = data.error();
+  return std::string_view{
+    [=]() -> const char * {
+      switch (err_no) {
+        case InodeResult::kCouldNotFindFilename:
+          return "Could not find filename";
+        case InodeResult::kVerificationFailed:
+          return "Verification failed";
+        default:
+          return strerror(err_no);
+      }
+    }()
+  };
+}
+
+std::ostream& operator<<(std::ostream& os, const InodeResult& result) {
+  os << "InodeResult{";
+  if (result) {
+      os << "OK,";
+  } else {
+      os << "ERR,";
+  }
+
+  os << result.inode << ",";
+
+  if (result) {
+    os << "\"" << result.data.value() << "\"";
+  } else {
+    os << result.data.error();
+    os << " (" << *result.ErrorMessage() << ")";
+  }
+
+  os << "}";
+  return os;
+}
+
+}  // namespace iorap::inode2filename
\ No newline at end of file
diff --git a/src/inode2filename/inode_result.h b/src/inode2filename/inode_result.h
new file mode 100644
index 0000000..df352fb
--- /dev/null
+++ b/src/inode2filename/inode_result.h
@@ -0,0 +1,82 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_
+#define IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_
+
+#include "common/expected.h"
+#include "inode2filename/inode.h"
+#include "inode2filename/inode_result.h"
+#include "inode2filename/system_call.h"
+
+#include <rxcpp/rx.hpp>
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+namespace iorap::inode2filename {
+
+// Tuple of (Inode -> (Filename|Errno))
+struct InodeResult {
+  // We set this error when all root directories have been searched and
+  // yet we still could not find a corresponding filename for the inode under search.
+  static constexpr int kCouldNotFindFilename = ENOKEY;
+
+  // An initial inode->filename mapping was found, but subsequent verification for it failed.
+  static constexpr int kVerificationFailed = EKEYEXPIRED;
+
+  // There is always an inode, but sometimes we may fail to resolve the filename.
+  Inode inode;
+  // Value: Contains the filename (with a root directory as a prefix).
+  // Error: Contains the errno, usually one of the above, otherwise some system error.
+  iorap::expected<std::string /*filename*/, int /*errno*/> data;
+
+  static InodeResult makeSuccess(Inode inode, std::string filename) {
+    return InodeResult{inode, std::move(filename)};
+  }
+
+  static InodeResult makeFailure(Inode inode, int err_no) {
+    return InodeResult{inode, iorap::unexpected{err_no}};
+  }
+
+  constexpr explicit operator bool() const {
+    return data.has_value();
+  }
+
+  constexpr bool operator==(const InodeResult& other) const {
+    if (inode == other.inode) {
+      if (data && other.data) {
+        return *data == *other.data;
+      } else if (!data && !other.data) {
+        return data.error() == other.data.error();
+      }
+      // TODO: operator== for expected
+    }
+    return false;
+  }
+
+  constexpr bool operator!=(const InodeResult& other) const {
+    return !(*this == other);
+  }
+
+  // Returns a human-readable error message, or 'nullopt' if there was no error.
+  std::optional<std::string_view> ErrorMessage() const;
+};
+
+std::ostream& operator<<(std::ostream& os, const InodeResult& result);
+
+}  // namespace iorap::inode2filename
+
+#endif  // IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_
\ No newline at end of file
diff --git a/src/inode2filename/main.cc b/src/inode2filename/main.cc
index 2da364c..38d6eab 100644
--- a/src/inode2filename/main.cc
+++ b/src/inode2filename/main.cc
@@ -14,12 +14,18 @@
 
 #include "common/debug.h"
 #include "common/expected.h"
-#include "inode2filename/search_directories.h"
+#include "inode2filename/inode_resolver.h"
 
-using namespace iorap::inode2filename;  // NOLINT
+#include <android-base/strings.h>
+
+#include <iostream>
+#include <fstream>
+#include <string_view>
 
 #if defined(IORAP_INODE2FILENAME_MAIN)
 
+namespace iorap::inode2filename {
+
 void Usage(char** argv) {
   std::cerr << "Usage: " << argv[0] << " <options> <<inode_syntax>> [inode_syntax1 inode_syntax2 ...]" << std::endl;
   std::cerr << "" << std::endl;
@@ -28,25 +34,174 @@ void Usage(char** argv) {
   std::cerr << "  the program will terminate." << std::endl;
   std::cerr << "" << std::endl;
   std::cerr << "    Inode syntax:     ('dev_t@inode' | 'major:minor:inode')" << std::endl;
+  std::cerr << "" << std::endl;       // CLI-only flags.
   std::cerr << "    --help,-h         Print this Usage." << std::endl;
-  std::cerr << "    --root,-r         Add root directory (default '.'). Repeatable." << std::endl;
   std::cerr << "    --verbose,-v      Set verbosity (default off)." << std::endl;
   std::cerr << "    --wait,-w         Wait for key stroke before continuing (default off)." << std::endl;
+  std::cerr << "" << std::endl;       // General flags.
+  std::cerr << "    --all,-a          Enumerate all inode->filename mappings in the dataset (default off)." << std::endl;
+  std::cerr << "                      All <<inode_syntaxN>> arguments are ignored." << std::endl;
+  std::cerr << "    --data-source=,   Choose a data source (default 'diskscan')." << std::endl;
+  std::cerr << "    -ds " << std::endl;
+  std::cerr << "        diskscan      Scan disk recursively using readdir." << std::endl;
+  std::cerr << "        textcache     Use the file from the '--output-format=text'." << std::endl;
+  std::cerr << "        bpf           Query kernel BPF maps (experimental)." << std::endl;
+  std::cerr << "    --output=,-o      Choose an output file (default 'stdout')." << std::endl;
+  std::cerr << "    --output-format=, Choose an output format (default 'log')." << std::endl;
+  std::cerr << "    -of " << std::endl;
+  std::cerr << "        log           Log human-readable, non-parsable format to stdout+logcat." << std::endl;
+  std::cerr << "        textcache     Results are in the same format as system/extras/pagecache." << std::endl;
+  std::cerr << "        ipc           Results are in a binary inter-process communications format" << std::endl;
+  std::cerr << "    --process-mode=,  Choose a process mode (default 'in'). Test-oriented." << std::endl;
+  std::cerr << "    -pm " << std::endl;
+  std::cerr << "        in            Use a single process to do the work in." << std::endl;
+  std::cerr << "        out           Out-of-process work (forks into a -pm=in)." << std::endl;
+  std::cerr << "    --verify=,-vy     Verification modes for the data source (default 'stat')." << std::endl;
+  std::cerr << "        stat          Use stat(2) call to validate data inodes are up-to-date. " << std::endl;
+  std::cerr << "        none          Trust that the data-source is up-to-date without checking." << std::endl;
+  std::cerr << "" << std::endl;       // --data-source=<?> specific flags.
+  std::cerr << "    Data-source-specific commands:" << std::endl;
+  std::cerr << "      --data-source=diskscan" << std::endl;
+  std::cerr << "          --root=,-r        Add root directory (default '.'). Repeatable." << std::endl;
+  std::cerr << "      --data-source=textcache" << std::endl;
+  std::cerr << "          --textcache=,-tc  Name of file that contains the textcache." << std::endl;
+  std::cerr << "" << std::endl;       // Programmatic flags.
+  std::cerr << "    --in-fd=#         Input file descriptor. Default input is from argv." << std::endl;
+  std::cerr << "    --out-fd=#        Output file descriptor. Default stdout." << std::endl;
   exit(1);
 }
 
-static fruit::Component<SearchDirectories> GetSearchDirectoriesComponent() {
+static fruit::Component<SystemCall> GetSystemCallComponent() {
     return fruit::createComponent().bind<SystemCall, SystemCallImpl>();
 }
 
+std::optional<DataSourceKind> ParseDataSourceKind(std::string_view str) {
+  if (str == "diskscan") {
+    return DataSourceKind::kDiskScan;
+  } else if (str == "textcache") {
+    return DataSourceKind::kTextCache;
+  } else if (str == "bpf") {
+    return DataSourceKind::kBpf;
+  }
+  return std::nullopt;
+}
+
+enum class OutputFormatKind {
+  kLog,
+  kTextCache,
+  kIpc,
+};
+
+std::optional<OutputFormatKind> ParseOutputFormatKind(std::string_view str) {
+  if (str == "log") {
+    return OutputFormatKind::kLog;
+  } else if (str == "textcache") {
+    return OutputFormatKind::kTextCache;
+  } else if (str == "ipc") {
+    return OutputFormatKind::kIpc;
+  }
+  return std::nullopt;
+}
+
+std::optional<VerifyKind> ParseVerifyKind(std::string_view str) {
+  if (str == "none") {
+    return VerifyKind::kNone;
+  } else if (str == "stat") {
+    return VerifyKind::kStat;
+  }
+  return std::nullopt;
+}
+
+std::optional<ProcessMode> ParseProcessMode(std::string_view str) {
+  if (str == "in") {
+    return ProcessMode::kInProcessDirect;
+  } else if (str == "out") {
+    return ProcessMode::kOutOfProcessIpc;
+  }
+  return std::nullopt;
+}
+
+bool StartsWith(std::string_view haystack, std::string_view needle) {
+  return haystack.size() >= needle.size()
+      && haystack.compare(0, needle.size(), needle) == 0;
+}
+
+bool EndsWith(std::string_view haystack, std::string_view needle) {
+  return haystack.size() >= needle.size()
+      && haystack.compare(haystack.size() - needle.size(), haystack.npos, needle) == 0;
+}
+
+bool StartsWithOneOf(std::string_view haystack,
+                     std::string_view needle,
+                     std::string_view needle2) {
+  return StartsWith(haystack, needle) || StartsWith(haystack, needle2);
+}
+
+enum ParseError {
+  kParseSkip,
+  kParseFailed,
+};
+
+std::optional<std::string> ParseNamedArgument(std::initializer_list<std::string> names,
+                                              std::string argstr,
+                                              std::optional<std::string> arg_next,
+                                              /*inout*/
+                                              int* arg_pos) {
+  for (const std::string& name : names) {
+    {
+      // Try parsing only 'argstr' for '--foo=bar' type parameters.
+      std::vector<std::string> split_str = ::android::base::Split(argstr, "=");
+      if (split_str.size() >= 2) {
+        /*
+        std::cerr << "ParseNamedArgument(name=" << name << ", argstr='"
+                  <<  argstr << "')" << std::endl;
+        */
+
+        if (split_str[0] + "=" == name) {
+          return split_str[1];
+        }
+      }
+    }
+    //if (EndsWith(name, "=")) {
+    //  continue;
+    /*} else */ {
+      // Try parsing 'argstr arg_next' for '-foo bar' type parameters.
+      if (argstr == name) {
+        ++(*arg_pos);
+
+        if (arg_next) {
+          return arg_next;
+        } else {
+          // Missing argument, e.g. '-foo' was the last token in the argv.
+          std::cerr << "Missing " << name << " flag value." << std::endl;
+          exit(1);
+        }
+      }
+    }
+  }
+
+  return std::nullopt;
+}
+
 int main(int argc, char** argv) {
   android::base::InitLogging(argv);
   android::base::SetLogger(android::base::StderrLogger);
 
+  bool all = false;
   bool wait_for_keystroke = false;
   bool enable_verbose = false;
   std::vector<std::string> root_directories;
   std::vector<Inode> inode_list;
+  int recording_time_sec = 0;
+
+  DataSourceKind data_source = DataSourceKind::kDiskScan;
+  OutputFormatKind output_format = OutputFormatKind::kLog;
+  VerifyKind verify = VerifyKind::kStat;
+  ProcessMode process_mode = ProcessMode::kInProcessDirect;
+
+  std::optional<std::string> output_filename;
+  std::optional<int /*fd*/> in_fd, out_fd;  // input-output file descriptors [for fork+exec].
+  std::optional<std::string> text_cache_filename;
 
   if (argc == 1) {
     Usage(argv);
@@ -54,22 +209,80 @@ int main(int argc, char** argv) {
 
   for (int arg = 1; arg < argc; ++arg) {
     std::string argstr = argv[arg];
-    bool has_arg_next = (arg+1)<argc;
-    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+    std::optional<std::string> arg_next;
+    if ((arg + 1) < argc) {
+      arg_next = argv[arg+1];
+    }
 
     if (argstr == "--help" || argstr == "-h") {
       Usage(argv);
-    } else if (argstr == "--root" || argstr == "-r") {
-      if (!has_arg_next) {
-        std::cerr << "Missing --root <value>" << std::endl;
-        return 1;
-      }
-      root_directories.push_back(arg_next);
-      ++arg;
+    } else if (auto val = ParseNamedArgument({"--root=", "-r"}, argstr, arg_next, /*inout*/&arg);
+               val) {
+      root_directories.push_back(*val);
     } else if (argstr == "--verbose" || argstr == "-v") {
       enable_verbose = true;
     } else if (argstr == "--wait" || argstr == "-w") {
       wait_for_keystroke = true;
+    }
+     else if (argstr == "--all" || argstr == "-a") {
+      all = true;
+    } else if (auto val = ParseNamedArgument({"--data-source=", "-ds"},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      auto ds = ParseDataSourceKind(*val);
+      if (!ds) {
+        std::cerr << "Invalid --data-source=<value>" << std::endl;
+        return 1;
+      }
+      data_source = *ds;
+    } else if (auto val = ParseNamedArgument({"--output=", "-o"},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      output_filename = *val;
+    } else if (auto val = ParseNamedArgument({"--process-mode=", "-pm"},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      auto pm = ParseProcessMode(*val);
+      if (!pm) {
+        std::cerr << "Invalid --process-mode=<value>" << std::endl;
+        return 1;
+      }
+      process_mode = *pm;
+    }
+    else if (auto val = ParseNamedArgument({"--output-format=", "-of"},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      auto of = ParseOutputFormatKind(*val);
+      if (!of) {
+        std::cerr << "Missing --output-format=<value>" << std::endl;
+        return 1;
+      }
+      output_format = *of;
+    } else if (auto val = ParseNamedArgument({"--verify=", "-vy="},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      auto vy = ParseVerifyKind(*val);
+      if (!vy) {
+        std::cerr << "Invalid --verify=<value>" << std::endl;
+        return 1;
+      }
+      verify = *vy;
+    } else if (auto val = ParseNamedArgument({"--textcache=", "-tc"},
+                                             argstr,
+                                             arg_next,
+                                             /*inout*/&arg);
+               val) {
+      text_cache_filename = *val;
     } else {
       Inode maybe_inode{};
 
@@ -94,10 +307,23 @@ int main(int argc, char** argv) {
     root_directories.push_back(".");
   }
 
-  if (inode_list.size() == 0) {
-    DCHECK_EQ(true, false);
-    std::cerr << "Provide at least one inode." << std::endl;
+  if (inode_list.size() == 0 && !all) {
+    std::cerr << "Provide at least one inode. Or use --all to dump everything." << std::endl;
     return 1;
+  } else if (all && inode_list.size() > 0) {
+    std::cerr << "[WARNING]: --all flag ignores all inodes passed on command line." << std::endl;
+  }
+
+  std::ofstream fout;
+  if (output_filename) {
+    fout.open(*output_filename);
+    if (!fout) {
+      std::cerr << "Failed to open output file for writing: \"" << *output_filename << "\"";
+      return 1;
+    }
+  } else {
+    fout.open("/dev/null");  // have to open *something* otherwise rdbuf fails.
+    fout.basic_ios<char>::rdbuf(std::cout.rdbuf());
   }
 
   if (enable_verbose) {
@@ -109,7 +335,12 @@ int main(int argc, char** argv) {
     for (auto& inode_num : inode_list) {
       LOG(VERBOSE) << "Searching for inode " << inode_num;
     }
+
+    LOG(VERBOSE) << "Dumping all inodes? " << all;
   }
+  // else use
+  // $> ANDROID_LOG_TAGS='*:d' iorap.inode2filename <args>
+  // which will enable arbitrary log levels.
 
   // Useful to attach a debugger...
   // 1) $> inode2filename -w <args>
@@ -120,33 +351,90 @@ int main(int argc, char** argv) {
     std::cin >> wait_for_keystroke;
   }
 
-  fruit::Injector<SearchDirectories> injector(GetSearchDirectoriesComponent);
-  SearchDirectories* search_directories = injector.get<SearchDirectories*>();
+  fruit::Injector<SystemCall> injector(GetSystemCallComponent);
+
+  InodeResolverDependencies ir_dependencies;
+  // Passed from command-line.
+  ir_dependencies.data_source = data_source;
+  ir_dependencies.process_mode = process_mode;
+  ir_dependencies.root_directories = root_directories;
+  ir_dependencies.text_cache_filename = text_cache_filename;
+  ir_dependencies.verify = verify;
+  // Hardcoded.
+  ir_dependencies.system_call = injector.get<SystemCall*>();
+
+  std::shared_ptr<InodeResolver> inode_resolver =
+      InodeResolver::Create(ir_dependencies);
 
-  auto/*observable[2]*/ [inode_results, connectable] =
-      search_directories->FindFilenamesFromInodesPair(
-          std::move(root_directories),
-          std::move(inode_list),
-          SearchMode::kInProcessDirect);
+  inode_resolver->StartRecording();
+  sleep(recording_time_sec);  // TODO: add cli flag for this when we add something that needs it.
+  inode_resolver->StopRecording();
 
-  int return_code = 1;
-  inode_results.subscribe([&return_code](const InodeResult& result) {
+  auto/*observable<InodeResult>*/ inode_results = all
+      ? inode_resolver->EmitAll()
+      : inode_resolver->FindFilenamesFromInodes(std::move(inode_list));
+
+  int return_code = 2;
+  inode_results.subscribe(
+      /*on_next*/[&return_code, output_format, &fout](const InodeResult& result) {
     if (result) {
-      LOG(DEBUG) << "Inode match: " << result.inode << ", " << result.data.value();
-      std::cout << "Inode match: " << result.inode << ", " << result.data.value() << std::endl;
+      LOG(DEBUG) << "Inode match: " << result;
+      if (output_format == OutputFormatKind::kLog) {
+        fout << "\033[1;32m[OK]\033[0m  "
+             << result.inode
+             << " \"" << result.data.value() << "\"" << std::endl;
+      } else if (output_format == OutputFormatKind::kIpc) {
+        fout << "K "
+             << result.inode
+             << " " << result.data.value() << std::endl;
+      } else if (output_format == OutputFormatKind::kTextCache) {
+        // Same format as TextCacheDataSource (system/extras/pagecache/pagecache.py -d)
+        //   "$device_number $inode $filesize $filename..."
+        const Inode& inode = result.inode;
+        fout << inode.GetDevice() << " "
+             << inode.GetInode()
+             << " -1 "  // always -1 for filesize, since we don't track what it is.
+             << result.data.value() << "\n";  // don't use endl which flushes, for faster writes.
+      } else {
+        LOG(FATAL) << "Not implemented this kind of --output-format";
+      }
+
       return_code = 0;
     } else {
-      LOG(WARNING) << "Failed to match inode: " << result.inode;
+      LOG(DEBUG) << "Failed to match inode: " << result;
+      if (output_format == OutputFormatKind::kLog) {
+        fout << "\033[1;31m[ERR]\033[0m "
+             << result.inode
+             << " '" << *result.ErrorMessage() << "'" << std::endl;
+      } else if (output_format == OutputFormatKind::kIpc) {
+        fout << "E "
+             << result.inode
+             << " " << result.data.error() << std::endl;
+      }
+      else if (output_format == OutputFormatKind::kTextCache) {
+        // Don't add bad results to the textcache. They are dropped.
+      } else {
+        LOG(FATAL) << "Not implemented this kind of --output-format";
+      }
     }
+  }, /*on_error*/[&return_code](rxcpp::util::error_ptr error) {
+    // Usually occurs very early on before we see the first result.
+    // In this case the error is terminal so we just end up exiting out soon.
+    return_code = 3;
+    LOG(ERROR) << "Critical error: " << rxcpp::util::what(error);
   });
 
-  // Normally #subscribe would start emitting items immediately, but this does nothing yet
-  // because one of the nodes in the flow graph was published. Published streams make the entire
-  // downstream inert until #connect is called.
-  connectable->connect();
-
-  // 0 -> found at least a single match, 1 -> could not find any matches.
+  // 0 -> found at least a single match,
+  // 1 -> bad parameters,
+  // 2 -> could not find any matches,
+  // 3 -> rxcpp on_error.
   return return_code;
 }
 
+}  // namespace iorap::inode2filename
+
+int main(int argc, char** argv) {
+  return ::iorap::inode2filename::main(argc, argv);
+}
+
 #endif
diff --git a/src/inode2filename/out_of_process_inode_resolver.cc b/src/inode2filename/out_of_process_inode_resolver.cc
new file mode 100644
index 0000000..88ce6e0
--- /dev/null
+++ b/src/inode2filename/out_of_process_inode_resolver.cc
@@ -0,0 +1,425 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/cmd_utils.h"
+#include "inode2filename/inode_resolver.h"
+#include "inode2filename/out_of_process_inode_resolver.h"
+
+#include <android-base/file.h>
+#include <android-base/logging.h>
+#include <android-base/unique_fd.h>
+
+#include <sstream>
+#include <stdio.h>
+#include <unistd.h>
+
+namespace rx = rxcpp;
+
+namespace iorap::inode2filename {
+
+using ::android::base::unique_fd;
+using ::android::base::borrowed_fd;
+
+static const char* GetCommandFileName() {
+  // Avoid ENOENT by execve by specifying the absolute path of inode2filename.
+#ifdef __ANDROID__
+  return "/system/bin/iorap.inode2filename";
+#else
+  static const char* file_name = nullptr;
+
+  if (file_name == nullptr) {
+    char* out_dir = getenv("ANDROID_HOST_OUT");
+    static std::string file_name_str = out_dir;
+    if (out_dir != nullptr) {
+      file_name_str += "/bin/";
+    } else {
+      // Assume it's in the same directory as the binary we are in.
+      std::string self_path;
+      CHECK(::android::base::Readlink("/proc/self/exe", /*out*/&self_path));
+
+      std::string self_dir = ::android::base::Dirname(self_path);
+      file_name_str = self_dir + "/";
+    }
+    file_name_str += "iorap.inode2filename";
+
+    file_name = file_name_str.c_str();
+  }
+
+  return file_name;
+#endif
+}
+
+std::error_code ErrorCodeFromErrno() {
+  int err = errno;
+  return std::error_code(err, std::system_category());
+}
+
+std::ios_base::failure IosBaseFailureWithErrno(const char* message) {
+  std::error_code ec = ErrorCodeFromErrno();
+  return std::ios_base::failure(message, ec);
+}
+
+static constexpr bool kDebugFgets = false;
+
+// This always contains the 'newline' character at the end of the string.
+// If there is not, the string is both empty and we hit EOF (or an error occurred).
+std::string FgetsWholeLine(FILE* stream,
+                           bool* eof) {
+  DCHECK(stream != nullptr);
+  DCHECK(eof != nullptr);
+
+  char buf[1024];
+
+  std::string str;
+  *eof = false;
+
+  while (true) {
+    memset(buf, '\0', sizeof(buf));
+
+    char* out = fgets(&buf[0], sizeof(buf), stream);
+
+    if (out == nullptr) {
+      // either EOF or error.
+
+      *eof = true;
+      if (feof(stream)) {
+        return str;
+      } else {
+        // error! :(
+        PLOG(ERROR) << "failed to fgets";
+        return str;
+      }
+    }
+
+    if (kDebugFgets) {
+      std::string dbg;
+
+      for (size_t i = 0; i < sizeof(buf); ++i) {
+        if (buf[i] == '\0') {
+          break;
+        }
+
+        int val = buf[i];
+
+        dbg += "," + std::to_string(val);
+      }
+
+      LOG(DEBUG) << "fgets ascii: " << dbg;
+    }
+
+    str += buf;
+
+    // fgets always reads at most count-1 characters.
+    // the last character is always '\0'
+    // the second-to-last character would be \n if we read the full line,
+    // and any other character otherwise.
+    if (!str.empty() && str.back() == '\n') {
+      // we read the whole line: do not need to call fgets again.
+      break;
+    }
+  }
+
+  return str;
+}
+
+static inline void LeftTrim(/*inout*/std::string& s) {
+  s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) {
+    return !std::isspace(ch);
+  }));
+}
+
+// Parses an --output-format=ipc kind of line into an InodeResult.
+// Returns nullopt if parsing failed.
+std::optional<InodeResult> ParseFromLine(const std::string& line) {
+  // inode <- INT:INT:INT
+  // line_ok <- 'K ' inode ' ' STRING
+  // line_err <- 'E ' inode ' ' INT
+  //
+  // result <- line_ok | line_err
+
+  std::stringstream ss{line};
+
+  bool result_ok = false;
+
+  std::string ok_or_error;
+  ss >> ok_or_error;
+
+  if (ss.fail()) {
+    return std::nullopt;
+  }
+
+  if (ok_or_error == "K") {
+    result_ok = true;
+  } else if (ok_or_error == "E") {
+    result_ok = false;
+  } else {
+    return std::nullopt;
+  }
+
+  std::string inode_s;
+  ss >> inode_s;
+
+  if (ss.fail()) {
+    return std::nullopt;
+  }
+
+  Inode inode;
+
+  std::string inode_parse_error_msg;
+  if (!Inode::Parse(inode_s, /*out*/&inode, /*out*/&inode_parse_error_msg)) {
+    return std::nullopt;
+  }
+
+  if (result_ok == false) {
+    int error_code;
+    ss >> error_code;
+
+    if (ss.fail()) {
+      return std::nullopt;
+    }
+
+    return InodeResult::makeFailure(inode, error_code);
+  } else if (result_ok == true) {
+    std::string rest_of_line;
+
+    // parse " string with potential spaces[\n]"
+    // into "string with potential spaces"
+    std::getline(/*inout*/ss, /*out*/rest_of_line);
+    LeftTrim(/*inout*/rest_of_line);
+
+    if (ss.fail()) {
+      return std::nullopt;
+    }
+
+    const std::string& file_name = rest_of_line;
+
+    return InodeResult::makeSuccess(inode, file_name);
+  }
+
+  return std::nullopt;
+}
+
+struct OutOfProcessInodeResolver::Impl {
+  Impl() {
+  }
+
+ private:
+  // Create the argv we will pass to the forked inode2filename, corresponds to #EmitAll.
+  std::vector<std::string> CreateArgvAll(const InodeResolverDependencies& deps) const {
+    std::vector<std::string> argv;
+    argv.push_back("--all");
+
+    return CreateArgv(deps, std::move(argv));
+  }
+
+  // Create the argv we will pass to the forked inode2filename, corresponds to
+  // #FindFilenamesFromInodes.
+  std::vector<std::string> CreateArgvFind(const InodeResolverDependencies& deps,
+                                          const std::vector<Inode>& inodes) const {
+    std::vector<std::string> argv;
+    iorap::common::AppendArgsRepeatedly(argv, inodes);
+
+    return CreateArgv(deps, std::move(argv));
+  }
+
+  std::vector<std::string> CreateArgv(const InodeResolverDependencies& deps,
+                                      std::vector<std::string> append_argv) const {
+    InodeResolverDependencies deps_oop = deps;
+    deps_oop.process_mode = ProcessMode::kInProcessDirect;
+
+    std::vector<std::string> argv = ToArgs(deps_oop);
+
+    argv.push_back("--output-format=ipc");
+
+    if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.log.verbose", false)) {
+      argv.push_back("--verbose");
+    }
+
+    iorap::common::AppendArgsRepeatedly(argv, std::move(append_argv));
+
+    return argv;
+  }
+
+ public:
+  // fork+exec into inode2filename with 'inodes' as the search list.
+  // Each result is parsed into a dest#on_next(result).
+  // If a fatal error occurs, dest#on_error is called once and no other callbacks are called.
+  void EmitFromCommandFind(rxcpp::subscriber<InodeResult>& dest,
+                           const InodeResolverDependencies& deps,
+                           const std::vector<Inode>& inodes) {
+    // Trivial case: complete immediately.
+    // Executing inode2filename with empty search list will just print the --help menu.
+    if (inodes.empty()) {
+      dest.on_completed();
+    }
+
+    std::vector<std::string> argv = CreateArgvFind(deps, inodes);
+    EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), inodes.size());
+  }
+
+  // fork+exec into inode2filename with --all (listing *all* inodes).
+  // Each result is parsed into a dest#on_next(result).
+  // If a fatal error occurs, dest#on_error is called once and no other callbacks are called.
+  void EmitFromCommandAll(rxcpp::subscriber<InodeResult>& dest,
+                           const InodeResolverDependencies& deps) {
+    std::vector<std::string> argv = CreateArgvAll(deps);
+    EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), /*result_count*/std::nullopt);
+  }
+
+ private:
+  void EmitFromCommandWithArgv(rxcpp::subscriber<InodeResult>& dest,
+                               std::vector<std::string> argv_vec,
+                               std::optional<size_t> result_count) {
+    unique_fd pipe_reader, pipe_writer;
+    if (!::android::base::Pipe(/*out*/&pipe_reader, /*out*/&pipe_writer)) {
+      dest.on_error(
+          rxcpp::util::make_error_ptr(
+              IosBaseFailureWithErrno("Failed to create out-going pipe for inode2filename")));
+      return;
+    }
+
+    pid_t child = fork();
+    if (child == -1) {
+      dest.on_error(
+          rxcpp::util::make_error_ptr(
+              IosBaseFailureWithErrno("Failed to fork process for inode2filename")));
+      return;
+    } else if (child > 0) {  // we are the caller of this function
+      LOG(DEBUG) << "forked into a process for inode2filename , pid = " << child;
+    } else {
+      // we are the child that was forked
+
+      const char* kCommandFileName = GetCommandFileName();
+
+      std::stringstream argv; // for debugging.
+      for (std::string arg : argv_vec) {
+        argv  << arg << ' ';
+      }
+      LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str();
+
+      // Redirect only stdout. stdin is unused, stderr is same as parent.
+      if (dup2(pipe_writer.get(), STDOUT_FILENO) == -1) {
+        // Trying to call #on_error does not make sense here because we are in a forked process,
+        // the only thing we can do is crash definitively.
+        PLOG(FATAL) << "Failed to dup2 for inode2filename";
+      }
+
+      std::unique_ptr<const char *[]> argv_ptr =
+                common::VecToArgv(kCommandFileName, argv_vec);
+
+      if (execve(kCommandFileName,
+                 (char **)argv_ptr.get(),
+                 /*envp*/nullptr) == -1) {
+        // Trying to call #on_error does not make sense here because we are in a forked process,
+        // the only thing we can do is crash definitively.
+        PLOG(FATAL) << "Failed to execve process for inode2filename";
+      }
+      // This should never return.
+    }
+
+    // Immediately close the writer end of the pipe because we never use it.
+    pipe_writer.reset();
+
+    // Convert pipe(reader) file descriptor into FILE*.
+    std::unique_ptr<FILE, int(*)(FILE*)> file_reader{
+        ::android::base::Fdopen(std::move(pipe_reader), /*mode*/"r"), fclose};
+    if (!file_reader) {
+      dest.on_error(
+          rxcpp::util::make_error_ptr(
+              IosBaseFailureWithErrno("Failed to fdopen for inode2filename")));
+      return;
+    }
+
+    size_t actual_result_count = 0;
+
+    bool file_eof = false;
+    while (!file_eof) {
+      std::string inode2filename_line = FgetsWholeLine(file_reader.get(), /*out*/&file_eof);
+
+      if (inode2filename_line.empty()) {
+        if (!file_eof) {
+          // Ignore empty lines.
+          LOG(WARNING) << "inode2filename: got empty line";
+        }
+        continue;
+      }
+
+      LOG(DEBUG) << "inode2filename output-line: " << inode2filename_line;
+
+      std::optional<InodeResult> res = ParseFromLine(inode2filename_line);
+      if (!res) {
+        std::string error_msg = "Invalid output: ";
+        error_msg += inode2filename_line;
+        dest.on_error(
+            rxcpp::util::make_error_ptr(std::ios_base::failure(error_msg)));
+        return;
+      }
+      dest.on_next(*res);
+
+      ++actual_result_count;
+    }
+
+    LOG(DEBUG) << "inode2filename output-eof";
+
+    // Ensure that the # of inputs into the rx stream match the # of outputs.
+    // This is validating the post-condition of FindFilenamesFromInodes.
+    if (result_count && actual_result_count != *result_count) {
+      std::stringstream ss;
+      ss << "Invalid number of results, expected: " << *result_count;
+      ss << ", actual: " << actual_result_count;
+
+      dest.on_error(
+          rxcpp::util::make_error_ptr(std::ios_base::failure(ss.str())));
+      return;
+    }
+
+    CHECK(child > 0);  // we are in the parent process, parse the IPC output of inode2filename
+    dest.on_completed();
+  }
+};
+
+rxcpp::observable<InodeResult>
+    OutOfProcessInodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const {
+  return rxcpp::observable<>::create<InodeResult>(
+    [self=std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this()),
+     inodes=std::move(inodes)](
+        rxcpp::subscriber<InodeResult> s) {
+      self->impl_->EmitFromCommandFind(s, self->GetDependencies(), inodes);
+  });
+}
+
+rxcpp::observable<InodeResult>
+    OutOfProcessInodeResolver::EmitAll() const {
+  auto self = std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this());
+  CHECK(self != nullptr);
+  CHECK(self->impl_ != nullptr);
+
+  return rxcpp::observable<>::create<InodeResult>(
+    [self](rxcpp::subscriber<InodeResult> s) {
+      CHECK(self != nullptr);
+      CHECK(self->impl_ != nullptr);
+      self->impl_->EmitFromCommandAll(s, self->GetDependencies());
+  });
+}
+
+OutOfProcessInodeResolver::OutOfProcessInodeResolver(InodeResolverDependencies dependencies)
+  : InodeResolver{std::move(dependencies)}, impl_{new Impl{}} {
+}
+
+OutOfProcessInodeResolver::~OutOfProcessInodeResolver() {
+  // std::unique_ptr requires complete types, but we hide the definition in the header.
+  delete impl_;
+}
+
+}  // namespace iorap::inode2filename
diff --git a/src/inode2filename/out_of_process_inode_resolver.h b/src/inode2filename/out_of_process_inode_resolver.h
new file mode 100644
index 0000000..c9f4291
--- /dev/null
+++ b/src/inode2filename/out_of_process_inode_resolver.h
@@ -0,0 +1,44 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_
+#define IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_
+
+#include "common/expected.h"
+#include "inode2filename/inode_resolver.h"
+
+namespace iorap::inode2filename {
+
+// Create an InodeResolver that fork+exec+pipes into the binary 'iorap.inode2filename'
+// and transmits the results back via an IPC mechanism.
+//
+// This is instantiated by InodeResolver::Create + ProcessMode::kOutOfProcessIpc
+class OutOfProcessInodeResolver : public InodeResolver {
+ public:
+  virtual rxcpp::observable<InodeResult>
+      FindFilenamesFromInodes(std::vector<Inode> inodes) const override;
+
+  virtual rxcpp::observable<InodeResult>
+      EmitAll() const override;
+
+  OutOfProcessInodeResolver(InodeResolverDependencies dependencies);
+  ~OutOfProcessInodeResolver();
+ private:
+  struct Impl;
+  Impl* impl_;
+};
+
+}
+
+#endif  // IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_
diff --git a/src/inode2filename/search_directories.cc b/src/inode2filename/search_directories.cc
index 2ec09d9..1d31671 100644
--- a/src/inode2filename/search_directories.cc
+++ b/src/inode2filename/search_directories.cc
@@ -53,11 +53,45 @@ using android::base::StringPrintf;  // NOLINT
 
 namespace iorap::inode2filename {
 
+#define DEBUG_INODE_SET 0
+
 // A multimap of 'ino_t -> List[Inode]' (where the value Inodes have the same ino_t as the key).
 //
 // A flat list of Inodes is turned into the above map, then keys can be removed one at a time
 // until the InodeSet eventually becomes empty.
 struct InodeSet {
+
+  InodeSet() = default;
+#if DEBUG_INODE_SET
+  InodeSet(const InodeSet& other) {
+    LOG(INFO) << "InodeSet-copyctor";
+    set_ = other.set_;
+  }
+
+  InodeSet(InodeSet&& other) {
+    LOG(INFO) << "InodeSet-movector";
+    set_ = std::move(other.set_);
+  }
+
+  InodeSet& operator=(const InodeSet& other) {
+    LOG(INFO) << "InodeSet-opassign-copy";
+    set_ = other.set_;
+    return *this;
+  }
+
+  InodeSet& operator=(InodeSet&& other) {
+    LOG(INFO) << "InodeSet-opassign-move";
+    set_ = std::move(other.set_);
+    return *this;
+  }
+#else
+  InodeSet(InodeSet&& other) = default;
+  InodeSet& operator=(InodeSet&& other) = default;
+  // Copying InodeSet can be very expensive, refuse to even allow compiling such code.
+  InodeSet(const InodeSet& other) = delete;
+  InodeSet& operator=(const InodeSet& other) = delete;
+#endif
+
   struct ValueRange {
     auto/*Iterable<Inode>*/ begin() {
       return begin_;
@@ -149,7 +183,7 @@ struct InodeSet {
     });
 
     if (found != inode_list.end()) {
-      const Inode& inode = found->second;
+      Inode inode = found->second;
       LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode;
       DCHECK(found->second.inode == stat_buf.st_ino);
       // Erase the inode from the list. This is important.
@@ -160,6 +194,33 @@ struct InodeSet {
     return std::nullopt;
   }
 
+  // Match all fields of an Inode against another Inode.
+  //
+  // The returned Inode (if any) is removed from the InodeSet; it will not be returned by
+  // FindInodeList in future calls.
+  std::optional<Inode> FindAndRemoveInodeInList(ValueRange inode_list,
+                                                const Inode& inode) {
+    LOG(VERBOSE) << "FindAndRemoveInodeInList " << inode_list << ", "
+                 << inode << "}";
+
+    auto /*iterator*/ found = std::find_if(inode_list.begin(),
+                                           inode_list.end(),
+                                           [&](const std::pair<ino_t, Inode>& pair) {
+      return inode == pair.second;
+    });
+
+    if (found != inode_list.end()) {
+      Inode inode = found->second;
+      LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode;
+      DCHECK_EQ(found->second, inode);
+      // Erase the inode from the list. This is important.
+      set_.erase(found);
+      return inode;
+    }
+
+    return std::nullopt;
+  }
+
   // TODO: equality and string operators for testing/logging.
  private:
   // Explanation: readdir returns a 'file' -> 'ino_t inode' mapping.
@@ -641,7 +702,8 @@ struct StatError {
 };
 
 std::ostream& operator<<(std::ostream& os, const StatError& e) {
-  os << "StatError{" << e.err_no << "," << e.path_name << "}";
+  os << "StatError{" << e.err_no << "," << e.path_name
+     << ": " << strerror(e.err_no) << "}";
   return os;
 }
 
@@ -685,6 +747,12 @@ struct SearchState {
   // The InodeSet removes any matching 'Inode'.
   std::optional<SearchMatch> match;
 
+  SearchState() = default;
+  SearchState(SearchState&& other) = default;
+
+  // Do not copy this because copying InodeSet is excruciatingly slow.
+  SearchState(const SearchState& other) = delete;
+
   // TODO: make sure this doesn't copy [inodes], as that would be unnecessarily expensive.
 };
 
@@ -727,18 +795,19 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
   // DirectoryEntry is missing the dev_t portion, so we may need to call scan(2) again
   // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match.
   // InodeSet lets us optimally avoid calling scan(2).
-  SearchState initial;
-  initial.inode_set = InodeSet::OfList(inode_search_list);
+  std::shared_ptr<SearchState> initial = std::make_shared<SearchState>();
+  initial->inode_set = InodeSet::OfList(inode_search_list);
 
   auto/*[observable<SearchState>,Connectable]*/ search_state_results = find_all_subdir_entries.scan(
       std::move(initial),
-      [system_call=system_call](SearchState search_state, const DirectoryEntry& dir_entry) {
+      [system_call=system_call](std::shared_ptr<SearchState> search_state,
+                                const DirectoryEntry& dir_entry) {
         LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#Scan "
-                     << dir_entry << ", state: " << search_state;
+                     << dir_entry << ", state: " << *search_state;
 
-        search_state.match = std::nullopt;
+        search_state->match = std::nullopt;
 
-        InodeSet* inodes = &search_state.inode_set;
+        InodeSet* inodes = &search_state->inode_set;
 
         // Find all the possible inodes across different devices.
         InodeSet::ValueRange inode_list = inodes->FindInodeList(dir_entry.d_ino);
@@ -754,23 +823,23 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
           std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf);
 
           if (inode) {
-            search_state.match = SearchMatch{inode.value(), dir_entry.filename};
+            search_state->match = SearchMatch{inode.value(), dir_entry.filename};
           }
         });
 
-        return search_state;  // implicit move.
+        return search_state;
       }
   // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon
   // as we find every single inode we care about.
-  ).take_while([](const SearchState& state) {
+  ).take_while([](std::shared_ptr<SearchState> state) {
       // Also emit the last item that caused the search set to go empty.
-      bool cond = !state.inode_set.Empty() || state.match;
+      bool cond = !state->inode_set.Empty() || state->match;
 
       if (WOULD_LOG(VERBOSE)) {
         static int kCounter = 0;
         LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while (" << kCounter++ <<
                      ",is_empty:"
-                     << state.inode_set.Empty() << ", match:" << state.match.has_value();
+                     << state->inode_set.Empty() << ", match:" << state->match.has_value();
       }
       // Minor O(1) implementation inefficiency:
       // (Too minor to fix but it can be strange if looking at the logs or readdir traces).
@@ -781,8 +850,8 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
       // This means that for cond to go to false, we would've read one extra item and then discarded
       // it. If that item was the first child of a directory, that means we essentially did
       // one redundant pass of doing a readdir.
-      //
       // In other words if the search set goes to empty while the current item is a directory,
+      //
       // it will definitely readdir on it at least once as we try to get the first child in
       // OnTreeTraversal.
       //
@@ -792,7 +861,7 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
 
       if (!cond) {
         LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while "
-                     << "should now terminate for " << state;
+                     << "should now terminate for " << *state;
       }
 
       return cond;
@@ -813,8 +882,10 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
   //    The 'matched_inode_values' only touches the match.
   // Either stream can 'std::move' from those fields because they don't move each other's fields.
   auto/*observable<InodeResult>*/ matched_inode_values = search_state_results
-      .filter([](const SearchState& search_state) { return search_state.match.has_value(); })
-      .map([](SearchState& search_state) { return std::move(search_state.match.value()); })
+      .filter([](std::shared_ptr<SearchState> search_state) {
+                  return search_state->match.has_value(); })
+      .map([](std::shared_ptr<SearchState> search_state) {
+                  return std::move(search_state->match.value()); })
                      // observable<SearchMatch>
       .map([](SearchMatch search_match) {
           return InodeResult::makeSuccess(search_match.inode, std::move(search_match.filename));
@@ -823,10 +894,10 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
   auto/*observable<?>*/ unmatched_inode_values = search_state_results
       // The 'last' SearchState is the one that contains all the remaining inodes.
       .take_last(1)  // observable<SearchState>
-      .flat_map([](const SearchState& search_state) {
+      .flat_map([](std::shared_ptr<SearchState> search_state) {
           LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- flat_map";
           // Aside: Could've used a move here if the inodes weren't so lightweight already.
-          return search_state.inode_set.IterateValues(); })
+          return search_state->inode_set.IterateValues(); })
                      // observable<Inode>
       .map([](const Inode& inode) {
           LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- map";
@@ -848,11 +919,10 @@ auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInode
   return std::make_pair(all_inode_results, search_state_results);
 }
 
-
 rxcpp::observable<InodeResult> SearchDirectories::FindFilenamesFromInodes(
     std::vector<std::string> root_directories,
     std::vector<Inode> inode_list,
-    SearchMode mode) {
+    SearchMode mode) const {
   DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet";
 
   auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes(
@@ -860,7 +930,7 @@ rxcpp::observable<InodeResult> SearchDirectories::FindFilenamesFromInodes(
       std::move(inode_list),
       system_call_);
 
-  return inode_results;
+  return inode_results.ref_count(connectable);
 }
 
 // I think we could avoid this with auto_connect, which rxcpp doesn't seem to have.
@@ -898,7 +968,7 @@ std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<SearchDirectories::RxA
     SearchDirectories::FindFilenamesFromInodesPair(
         std::vector<std::string> root_directories,
         std::vector<Inode> inode_list,
-        SearchMode mode) {
+        SearchMode mode) const {
   DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet";
 
   auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes(
@@ -912,4 +982,385 @@ std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<SearchDirectories::RxA
   return {inode_results, std::move(connectable_ptr)};
 }
 
+rxcpp::observable<InodeResult>
+    SearchDirectories::FindFilenamesFromInodes(std::vector<std::string> root_directories,
+                                               rxcpp::observable<Inode> inodes,
+                                               SearchMode mode) const {
+
+  // It's inefficient to search for inodes until the full search list is available,
+  // so first reduce to a vector so we can access all the inodes simultaneously.
+  return inodes.reduce(std::vector<Inode>{},
+                       [](std::vector<Inode> vec, Inode inode) {
+                         vec.push_back(inode);
+                         return vec;
+                       },
+                       [](std::vector<Inode> v){
+                         return v;  // TODO: use an identity function
+                       })
+    .flat_map([root_directories=std::move(root_directories), mode, self=*this]
+          (std::vector<Inode> vec) {
+      // All borrowed values (e.g. SystemCall) must outlive the observable.
+      return self.FindFilenamesFromInodes(root_directories, vec, mode);
+    }
+  );
+}
+
+auto/*[observable<InodeResult>]*/ EmitAllInodesFromDirectories(
+    std::vector<std::string> root_dirs,
+    borrowed<SystemCall*> system_call) {
+
+  // Create a (lazy) observable that will emit each DirectoryEntry that is a recursive subchild
+  // of root_dirs. Emission will be stopped when its unsubscribed from.
+  //
+  // This is done by calling readdir(3) lazily.
+  auto/*obs<DirectoryEntry>*/ find_all_subdir_entries = ([&]() {
+    DirectoryEntry sentinel = DirectoryEntry::CreateSentinel(std::move(root_dirs));
+    auto/*obs<DirectoryEntryResult*/ results = sentinel.GetSubTreePreOrderEntries(system_call);
+
+    // Drop any errors by logging them to logcat. "Unwrap" the expected into the underlying data.
+    auto/*obs<DirectoryEntry*>*/ expected_drop_errors = MapExpectedOrLogError(std::move(results));
+    return expected_drop_errors;
+  })();
+
+  // Fill in -1 for the dev_t since readdir only returns the ino_t.
+  // The caller of this function is expected to call stat(2) later on to fill in
+  // the full data.
+  return find_all_subdir_entries.map([](DirectoryEntry e) {
+    return InodeResult::makeSuccess(Inode::FromDeviceAndInode(-1, e.d_ino), std::move(e.filename));
+  });
+}
+
+rxcpp::observable<InodeResult>
+    SearchDirectories::ListAllFilenames(std::vector<std::string> root_directories) const {
+  // TODO: refactor implementation into DiskScanDataSource.
+  return EmitAllInodesFromDirectories(std::move(root_directories),
+                                      /*borrowed*/system_call_);
+}
+
+struct FilterState {
+  // Emit 'match' Inodes corresponding to the ones here.
+  InodeSet inode_set;
+
+  // An inode matching one of the ones in inode_set was discovered in the most-recently
+  // emitted SearchState.
+  //
+  // The InodeSet removes any matching 'Inode'.
+  std::optional<InodeResult> match;
+
+  FilterState() = default;
+  FilterState(FilterState&& other) = default;
+
+  // Copying the InodeSet is expensive, so forbid any copies.
+  FilterState(const FilterState& other) = delete;
+};
+
+std::ostream& operator<<(std::ostream& os, const FilterState& s) {
+  os << "FilterState{match:";
+  // Print the 'match' first. The InodeSet could be very large so it could be truncated in logs.
+  if (s.match) {
+    os << s.match.value();
+  } else {
+    os << "(none)";
+  }
+  os << ", inode_set:" << s.inode_set << "}";
+  return os;
+}
+
+rxcpp::observable<InodeResult> SearchDirectories::FilterFilenamesForSpecificInodes(
+    rxcpp::observable<InodeResult> all_inodes,
+    std::vector<Inode> inode_list,
+    bool missing_device_number,   // missing dev_t portion?
+    bool needs_verification) const {
+  // TODO: refactor into InodeResolver
+
+  borrowed<SystemCall*> system_call = system_call_;
+
+  // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again
+  // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match.
+  // InodeSet lets us optimally avoid calling scan(2).
+  std::shared_ptr<FilterState> initial = std::make_shared<FilterState>();
+  initial->inode_set = InodeSet::OfList(inode_list);
+
+  auto/*[observable<FilterState>,Connectable]*/ filter_state_results = all_inodes.scan(
+      std::move(initial),
+      [system_call, missing_device_number]
+          (std::shared_ptr<FilterState> filter_state, InodeResult inode_result) {
+        LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#Scan "
+                     << inode_result << ", state: " << *filter_state;
+
+        filter_state->match = std::nullopt;
+
+        InodeSet* inodes = &filter_state->inode_set;
+
+        // Find all the possible (dev_t, ino_t) potential needles given an ino_t in the haystack.
+        InodeSet::ValueRange inode_list = inodes->FindInodeList(inode_result.inode.inode);
+
+        // This inode result doesn't correspond to any inodes we are searching for.
+        if (!inode_list) {
+          // Drop the result and keep going.
+          return filter_state;
+        }
+
+        if (missing_device_number) {
+          // Need to fill in dev_t by calling stat(2).
+          VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) {
+            StatResult maybe_stat = Stat(filename, system_call);
+            VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) {
+              // Try to match the specific inode. Usually this will not result in a match (nullopt).
+              std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf);
+
+              if (inode) {
+                filter_state->match = InodeResult::makeSuccess(inode.value(), std::move(filename));
+              }
+            });
+
+            // Note: stat errors are logged here to make the error closer to the occurrence.
+            // In theory, we could just return it as an InodeResult but then the error would
+            // just get logged elsewhere.
+          });
+        } else {
+            // Trust the dev_t in InodeResult is valid. Later passes can verify it.
+
+            // Try to match the specific inode. Usually this will not result in a match (nullopt).
+            std::optional<Inode> inode =
+                inodes->FindAndRemoveInodeInList(inode_list, inode_result.inode);
+
+            if (inode) {
+              filter_state->match = inode_result;
+            }
+
+            // Note that the InodeResult doesn't necessarily need to have a valid filename here.
+            // If the earlier pass returned an error-ed result, this will forward the error code.
+        }
+
+        return filter_state;
+      }
+  // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon
+  // as we find every single inode we care about.
+  ).take_while([](std::shared_ptr<FilterState> state) {
+      // Also emit the last item that caused the search set to go empty.
+      bool cond = !state->inode_set.Empty() || state->match;
+
+      if (WOULD_LOG(VERBOSE)) {
+        static int kCounter = 0;
+        LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while (" << kCounter++ <<
+                     ",is_empty:"
+                     << state->inode_set.Empty() << ", match:" << state->match.has_value();
+      }
+      // Minor O(1) implementation inefficiency:
+      // (Too minor to fix but it can be strange if looking at the logs or readdir traces).
+      //
+      // Note, because we return 'true' after the search set went empty,
+      // the overall stream graph still pulls from filter_state_results exactly once more:
+      //
+      // This means that for cond to go to false, we would've read one extra item and then discarded
+      // it. If that item was the first child of a directory, that means we essentially did
+      // one redundant pass of doing a readdir.
+      // In other words if the search set goes to empty while the current item is a directory,
+      //
+      // it will definitely readdir on it at least once as we try to get the first child in
+      // OnTreeTraversal.
+      //
+      // This could be fixed with a 'take_until(Predicate)' operator which doesn't discard
+      // the last item when the condition becomes false. However rxcpp seems to lack this operator,
+      // whereas RxJava has it.
+
+      if (!cond) {
+        LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while "
+                     << "should now terminate for " << *state;
+      }
+
+      return cond;
+  }).publish();
+  // The publish here is mandatory. The stream is consumed twice (once by matched and once by
+  // unmatched streams). Without the publish, once all items from 'matched' were consumed it would
+  // start another instance of 'filter_state_results' (i.e. it appears as if the search
+  // is restarted).
+  //
+  // By using 'publish', the filter_state_results is effectively shared by both downstream nodes.
+  // Note that this also requires the subscriber to additionally call #connect on the above stream,
+  // otherwise no work will happen.
+
+  // Lifetime notes:
+  //
+  // The the 'FilterState' is emitted into both below streams simultaneously.
+  //    The 'unmatched_inode_values' only touches the inode_set.
+  //    The 'matched_inode_values' only touches the match.
+  // Either stream can 'std::move' from those fields because they don't move each other's fields.
+  auto/*observable<InodeResult>*/ matched_inode_values = filter_state_results
+      .filter([](std::shared_ptr<FilterState> filter_state) {
+                  return filter_state->match.has_value(); })
+      .map([](std::shared_ptr<FilterState> filter_state) {
+                  return std::move(filter_state->match.value()); });
+                     // observable<InodeResult>
+
+  auto/*observable<?>*/ unmatched_inode_values = filter_state_results
+      // The 'last' FilterState is the one that contains all the remaining inodes.
+      .take_last(1)  // observable<FilterState>
+      .flat_map([](std::shared_ptr<FilterState> filter_state) {
+          LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- flat_map";
+          // Aside: Could've used a move here if the inodes weren't so lightweight already.
+          return filter_state->inode_set.IterateValues(); })
+                     // observable<Inode>
+      .map([](const Inode& inode) {
+          LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- map";
+          return InodeResult::makeFailure(inode, InodeResult::kCouldNotFindFilename);
+      });
+                     // observable<InodeResult>
+
+  // The matched and unmatched InodeResults are emitted together.
+  //   Use merge, not concat, because we need both observables to be subscribed to simultaneously.
+
+  auto/*observable<InodeResult*/ all_inode_results =
+      matched_inode_values.merge(unmatched_inode_values);
+
+  // Verify the inode results by calling stat(2).
+  // Unverified results are turned into an error.
+
+  auto/*observable<InodeResult>*/ verified_inode_results =
+    all_inode_results.map([needs_verification, system_call](InodeResult result) {
+      if (!needs_verification || !result) {
+        // Skip verification if requested, or if the result didn't have a filename.
+        return result;
+      }
+
+      const std::string& filename = result.data.value();
+      StatResult maybe_stat = Stat(filename, system_call);
+
+      if (maybe_stat)
+      {
+        if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) {
+          return result;
+        } else {
+          LOG(WARNING)
+              << "FilterFilenamesForSpecificInodes#verified fail out-of-date inode: " << result;
+          return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed);
+        }
+      } else {
+        // Forward stat errors directly, as it could be a missing security rule,
+        // but turn -ENOENT into casual verification errors.
+        const StatError& err = maybe_stat.error();
+        int error_code = err.err_no;
+        if (err.err_no == ENOENT) {
+          error_code = InodeResult::kVerificationFailed;
+
+          // TODO: Don't LOG(WARNING) here because this could be very common if we
+          // access the data much much later after the initial results were read in.
+          LOG(WARNING)
+              << "FilterFilenamesForSpecificInodes#verified fail out-of-date filename: " << result;
+        } else {
+          LOG(ERROR)
+              << "FilterFilenamesForSpecificInodes#verified stat(2) failure: " << err;
+        }
+
+        return InodeResult::makeFailure(result.inode, error_code);
+      }
+    });
+
+  // Now that all mid-stream observables have been connected, turn the Connectable observable
+  // into a regular observable.
+  return verified_inode_results.ref_count(filter_state_results);
+}
+
+rxcpp::observable<InodeResult> SearchDirectories::EmitAllFilenames(
+    rxcpp::observable<InodeResult> all_inodes,
+    bool missing_device_number,   // missing dev_t portion?
+    bool needs_verification) const {
+  // TODO: refactor into InodeResolver
+
+  borrowed<SystemCall*> system_call = system_call_;
+
+  // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again
+  // to confirm the dev_t.
+
+  using EmitAllState = std::optional<InodeResult>;
+
+  auto/*[observable<FilterState>,Connectable]*/ all_inode_results = all_inodes.map(
+      [system_call, missing_device_number](InodeResult inode_result) {
+        LOG(VERBOSE) << "EmitAllFilenames#map "
+                     << inode_result;
+
+        // Could fail if the device number is missing _and_ stat(2) fails.
+        EmitAllState match = std::nullopt;
+
+        if (missing_device_number) {
+          // Need to fill in dev_t by calling stat(2).
+          VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) {
+            StatResult maybe_stat = Stat(filename, system_call);
+            VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) {
+              Inode inode = Inode::FromDeviceAndInode(stat_buf.st_dev, stat_buf.st_ino);
+              match = InodeResult::makeSuccess(inode, std::move(filename));
+            });
+
+            // Note: stat errors are logged here to make the error closer to the occurrence.
+            // In theory, we could just return it as an InodeResult but then the error would
+            // just get logged elsewhere.
+          });
+        } else {
+            // Trust the dev_t in InodeResult is valid. Later passes can verify it.
+            match = std::move(inode_result);
+
+            // Note that the InodeResult doesn't necessarily need to have a valid filename here.
+            // If the earlier pass returned an error-ed result, this will forward the error code.
+        }
+
+        return match;  // implicit move.
+      }
+  );
+
+  auto/*observable<InodeResult>*/ matched_inode_values = all_inode_results
+      .filter([](const EmitAllState& filter_state) { return filter_state.has_value(); })
+      .map([](EmitAllState& filter_state) { return std::move(filter_state.value()); });
+                     // observable<InodeResult>
+
+  // Verify the inode results by calling stat(2).
+  // Unverified results are turned into an error.
+
+  auto/*observable<InodeResult>*/ verified_inode_results =
+    matched_inode_values.map([needs_verification, system_call](InodeResult result) {
+      if (!needs_verification || !result) {
+        // Skip verification if requested, or if the result didn't have a filename.
+        return result;
+      }
+
+      const std::string& filename = result.data.value();
+      StatResult maybe_stat = Stat(filename, system_call);
+
+      if (maybe_stat)
+      {
+        if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) {
+          return result;
+        } else {
+          LOG(WARNING)
+              << "EmitAllFilenames#verified fail out-of-date inode: " << result;
+          return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed);
+        }
+      } else {
+        // Forward stat errors directly, as it could be a missing security rule,
+        // but turn -ENOENT into casual verification errors.
+        const StatError& err = maybe_stat.error();
+        int error_code = err.err_no;
+        if (err.err_no == ENOENT) {
+          error_code = InodeResult::kVerificationFailed;
+
+          // TODO: Don't LOG(WARNING) here because this could be very common if we
+          // access the data much much later after the initial results were read in.
+          LOG(WARNING)
+              << "EmitAllFilenames#verified fail out-of-date filename: " << result;
+        } else {
+          LOG(ERROR)
+              << "EmitAllFilenames#verified stat(2) failure: " << err;
+        }
+
+        return InodeResult::makeFailure(result.inode, error_code);
+      }
+    });
+
+  // TODO: refactor this function some more with the Find(inode_set) equivalent.
+
+  // Now that all mid-stream observables have been connected, turn the Connectable observable
+  // into a regular observable.
+  return verified_inode_results;
+}
+
 }  // namespace iorap::inode2filename
diff --git a/src/inode2filename/search_directories.h b/src/inode2filename/search_directories.h
index 8156574..0b4af78 100644
--- a/src/inode2filename/search_directories.h
+++ b/src/inode2filename/search_directories.h
@@ -15,52 +15,17 @@
 #ifndef IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_
 #define IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_
 
-#include "common/expected.h"
 #include "inode2filename/inode.h"
 #include "inode2filename/system_call.h"
+#include "inode2filename/inode_resolver.h"
 
 #include <fruit/fruit.h>
 
 #include <rxcpp/rx.hpp>
 namespace iorap::inode2filename {
 
-// Tuple of (Inode -> (Filename|Errno))
-struct InodeResult {
-  // We set this error when all root directories have been searched and
-  // yet we still could not find a corresponding filename for the inode under search.
-  static constexpr int kCouldNotFindFilename = -ENOENT;
-
-  Inode inode;
-  // Value: Contains the filename (with a root directory as a prefix).
-  // Error: Contains the errno, usually -ENOENT or perhaps a security error.
-  iorap::expected<std::string /*filename*/, int /*errno*/> data;
-
-  static InodeResult makeSuccess(Inode inode, std::string filename) {
-    return InodeResult{inode, std::move(filename)};
-  }
-
-  static InodeResult makeFailure(Inode inode, int err_no) {
-    return InodeResult{inode, iorap::unexpected{err_no}};
-  }
-
-  constexpr operator bool() const {
-    return data.has_value();
-  }
-};
-
-enum class SearchMode {
-  // Test modes:
-  kInProcessDirect,  // Execute the code directly.
-  kInProcessIpc,     // Execute code via IPC layer using multiple threads.
-  // Shipping mode:
-  kOutOfProcessIpc,  // Execute code via fork+exec with IPC.
-
-  // Note: in-process system-wide stat(2)/readdir/etc is blocked by selinux.
-  // Attempting to call the test modes will fail with -EPERM.
-  //
-  // Use fork+exec mode in shipping configurations, which spawns inode2filename
-  // as a separate command.
-};
+// TODO: rename.
+using SearchMode = ProcessMode;
 
 struct SearchDirectories {
   // Type-erased subset of rxcpp::connectable_observable<?>
@@ -100,7 +65,7 @@ struct SearchDirectories {
   rxcpp::observable<InodeResult>
       FindFilenamesFromInodes(std::vector<std::string> root_directories,
                               std::vector<Inode> inode_list,
-                              SearchMode mode);
+                              SearchMode mode) const;
 
   // Create a cold observable of inode results (a lazy stream) corresponding
   // to the inode search list.
@@ -126,7 +91,37 @@ struct SearchDirectories {
   std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<RxAnyConnectable>>
       FindFilenamesFromInodesPair(std::vector<std::string> root_directories,
                                   std::vector<Inode> inode_list,
-                                  SearchMode mode);
+                                  SearchMode mode) const;
+
+  // No items on the output stream will be emitted until 'inodes' completes.
+  //
+  // The current algorithm is a naive DFS, so if it began too early it would either
+  // miss the search items or require traversal restarts.
+  //
+  // See above for more details.
+  rxcpp::observable<InodeResult>
+      FindFilenamesFromInodes(std::vector<std::string> root_directories,
+                              rxcpp::observable<Inode> inodes,
+                              SearchMode mode) const;
+
+  rxcpp::observable<InodeResult>
+      ListAllFilenames(std::vector<std::string> root_directories) const;
+
+  rxcpp::observable<InodeResult> FilterFilenamesForSpecificInodes(
+      // haystack that will be subscribed to until all in inode_list are found.
+      rxcpp::observable<InodeResult> all_inodes,
+      // key list: traverse all_inodes until we emit all results from inode_list.
+      std::vector<Inode> inode_list,
+      // all_inodes have a missing device number: use stat(2) to fill it in.
+      bool missing_device_number,
+      bool needs_verification) const;
+
+  rxcpp::observable<InodeResult> EmitAllFilenames(
+      // haystack that will be subscribed to until all in inode_list are found.
+      rxcpp::observable<InodeResult> all_inodes,
+      // all_inodes have a missing device number: use stat(2) to fill it in.
+      bool missing_device_number,
+      bool needs_verification) const;
 
   // Any borrowed parameters here can also be borrowed by the observables returned by the above
   // member functions.
diff --git a/src/iorapd/main.cc b/src/iorapd/main.cc
index ab73fe5..8d806ba 100644
--- a/src/iorapd/main.cc
+++ b/src/iorapd/main.cc
@@ -16,6 +16,8 @@
 
 #include "binder/iiorap_impl.h"
 #include "common/debug.h"
+#include "common/loggers.h"
+#include "db/models.h"
 #include "manager/event_manager.h"
 
 #include <android-base/logging.h>
@@ -27,27 +29,6 @@
 
 static constexpr const char* kServiceName = iorap::binder::IIorapImpl::getServiceName();
 
-// Log to both Stderr and Logd for convenience when running this from the command line.
-class StderrAndLogdLogger {
- public:
-  explicit StderrAndLogdLogger(android::base::LogId default_log_id = android::base::MAIN)
-    : logd_(default_log_id) {
-  }
-
-  void operator()(::android::base::LogId id,
-                  ::android::base::LogSeverity sev,
-                  const char* tag,
-                  const char* file,
-                  unsigned int line,
-                  const char* message) {
-    logd_(id, sev, tag, file, line, message);
-    StderrLogger(id, sev, tag, file, line, message);
-  }
-
- private:
-  ::android::base::LogdLogger logd_;
-};
-
 int main(int /*argc*/, char** argv) {
   if (android::base::GetBoolProperty("iorapd.log.verbose", iorap::kIsDebugBuild)) {
     // Show verbose logs if the property is enabled or if we are a debug build.
@@ -55,23 +36,35 @@ int main(int /*argc*/, char** argv) {
   }
 
   // Logs go to system logcat.
-  android::base::InitLogging(argv, StderrAndLogdLogger{android::base::SYSTEM});
+  android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM});
 
+  LOG(INFO) << kServiceName << " (the prefetchening) firing up";
   {
-    android::ScopedTrace trace_main{ATRACE_TAG_PACKAGE_MANAGER, "main"};
-    LOG(INFO) << kServiceName << " (the prefetchening) firing up";
+    android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::db_init"};
+    iorap::db::SchemaModel db_schema =
+        iorap::db::SchemaModel::GetOrCreate(
+            android::base::GetProperty("iorapd.db.location",
+                                       "/data/misc/iorapd/sqlite.db"));
+    db_schema.MarkSingleton();
+  }
 
-    android::ScopedTrace trace_start{ATRACE_TAG_PACKAGE_MANAGER, "IorapNativeService::start"};
+  std::shared_ptr<iorap::manager::EventManager> event_manager;
+  {
+    android::ScopedTrace trace_start{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::start"};
 
     // TODO: use fruit for this DI.
-    auto /*std::shared_ptr<EventManager>*/ event_manager =
+    event_manager =
         iorap::manager::EventManager::Create();
-    if (!iorap::binder::IIorapImpl::Start(std::move(event_manager))) {
+    if (!iorap::binder::IIorapImpl::Start(event_manager)) {
       LOG(ERROR) << "Unable to start IorapNativeService";
       exit(1);
     }
   }
 
+  // This must be logged after all other initialization has finished.
+  LOG(INFO) << kServiceName << " (the prefetchening) readied up";
+
+  event_manager->Join();  // TODO: shutdown somewhere?
   // Block until something else shuts down the binder service.
   android::IPCThreadState::self()->joinThreadPool();
   LOG(INFO) << kServiceName << " shutting down";
diff --git a/src/maintenance/controller.cc b/src/maintenance/controller.cc
new file mode 100644
index 0000000..daaf08f
--- /dev/null
+++ b/src/maintenance/controller.cc
@@ -0,0 +1,620 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "compiler/compiler.h"
+#include "maintenance/controller.h"
+
+#include "common/cmd_utils.h"
+#include "common/debug.h"
+#include "common/expected.h"
+#include "common/trace.h"
+
+#include "db/models.h"
+#include "inode2filename/inode.h"
+#include "inode2filename/search_directories.h"
+#include "prefetcher/read_ahead.h"
+
+#include <android-base/file.h>
+#include <utils/Printer.h>
+
+#include <chrono>
+#include <ctime>
+#include <iostream>
+#include <filesystem>
+#include <fstream>
+#include <limits>
+#include <mutex>
+#include <optional>
+#include <vector>
+#include <string>
+#include <sys/wait.h>
+
+namespace iorap::maintenance {
+
+const constexpr int64_t kCompilerCheckIntervalMs = 10;
+static constexpr size_t kMinTracesForCompilation = 1;
+
+struct LastJobInfo {
+  time_t last_run_ns_{0};
+  size_t activities_last_compiled_{0};
+};
+
+LastJobInfo last_job_info_;
+std::mutex last_job_info_mutex_;
+
+// Gets the path of output compiled trace.
+db::CompiledTraceFileModel CalculateNewestFilePath(
+    const std::string& package_name,
+    const std::string& activity_name,
+    int version) {
+   db::VersionedComponentName versioned_component_name{
+     package_name, activity_name, version};
+
+   db::CompiledTraceFileModel output_file =
+       db::CompiledTraceFileModel::CalculateNewestFilePath(versioned_component_name);
+
+   return output_file;
+}
+
+using ArgString = const char*;
+
+static constexpr const char kCommandFileName[] = "/system/bin/iorap.cmd.compiler";
+
+int Exec::Execve(const std::string& pathname,
+                 std::vector<std::string>& argv_vec,
+                 char *const envp[]) {
+  std::unique_ptr<ArgString[]> argv_ptr =
+      common::VecToArgv(kCommandFileName, argv_vec);
+
+  return execve(pathname.c_str(), (char**)argv_ptr.get(), envp);
+}
+
+pid_t Exec::Fork() {
+  return fork();
+}
+
+// Represents the parameters used when fork+exec compiler.
+struct CompilerForkParameters {
+  std::vector<std::string> input_pbs;
+  std::vector<uint64_t> timestamp_limit_ns;
+  std::string output_proto;
+  ControllerParameters controller_params;
+
+  CompilerForkParameters(const std::vector<compiler::CompilationInput>& perfetto_traces,
+                         const std::string& output_proto,
+                         ControllerParameters controller_params) :
+    output_proto(output_proto), controller_params(controller_params) {
+        for (compiler::CompilationInput perfetto_trace : perfetto_traces) {
+          input_pbs.push_back(perfetto_trace.filename);
+          timestamp_limit_ns.push_back(perfetto_trace.timestamp_limit_ns);
+        }
+  }
+};
+
+std::vector<std::string> MakeCompilerParams(const CompilerForkParameters& params) {
+    std::vector<std::string> argv;
+    ControllerParameters controller_params = params.controller_params;
+
+    common::AppendArgsRepeatedly(argv, params.input_pbs);
+    common::AppendArgsRepeatedly(argv, "--timestamp_limit_ns", params.timestamp_limit_ns);
+
+    if (controller_params.output_text) {
+      argv.push_back("--output-text");
+    }
+
+    common::AppendArgs(argv, "--output-proto", params.output_proto);
+
+    if (controller_params.inode_textcache) {
+      common::AppendArgs(argv, "--inode-textcache", *controller_params.inode_textcache);
+    }
+
+    if (controller_params.verbose) {
+      argv.push_back("--verbose");
+    }
+
+    return argv;
+}
+
+// Sets a watch dog for the given pid and kill it if timeout.
+std::thread SetTimeoutWatchDog(pid_t pid, int64_t timeout_ms, std::atomic<bool>& cancel_watchdog) {
+  std::thread watchdog_thread{[pid, timeout_ms, &cancel_watchdog]() {
+    std::chrono::time_point start = std::chrono::system_clock::now();
+    std::chrono::milliseconds timeout(timeout_ms);
+    while (!cancel_watchdog) {
+      int status = kill(pid, 0);
+      if (status != 0) {
+        LOG(DEBUG) << "Process (" << pid << ") doesn't exist now.";
+        break;
+      }
+      std::chrono::time_point cur = std::chrono::system_clock::now();
+      if (cur - start > timeout) {
+        LOG(INFO) << "Process (" << pid << ") is timeout!";
+        LOG(INFO) << "start time: "
+                   << std::chrono::system_clock::to_time_t(start)
+                   << " end time: "
+                   << std::chrono::system_clock::to_time_t(cur)
+                   << " timeout: "
+                   << timeout_ms;
+        kill(pid, SIGKILL);
+        break;
+      }
+      usleep(kCompilerCheckIntervalMs * 1000);
+    }
+  }};
+
+  return watchdog_thread;
+}
+
+bool StartViaFork(const CompilerForkParameters& params) {
+  const ControllerParameters& controller_params = params.controller_params;
+  pid_t child = controller_params.exec->Fork();
+
+  if (child == -1) {
+    LOG(FATAL) << "Failed to fork a process for compilation";
+  } else if (child > 0) {  // we are the caller of this function
+    LOG(DEBUG) << "forked into a process for compilation , pid = " << child;
+
+    int64_t compiler_timeout_ms =
+        android::base::GetIntProperty("iorapd.maintenance.compiler_timeout_ms",
+                                       /*default*/ 10 * 60 * 1000); // 10 min
+    std::atomic<bool> cancel_watchdog(false);
+    std::thread watchdog_thread = SetTimeoutWatchDog(child, compiler_timeout_ms, cancel_watchdog);
+    int wstatus;
+    waitpid(child, /*out*/&wstatus, /*options*/0);
+
+    // Terminate the thread after the compiler process is killed or done.
+    LOG(DEBUG) << "Terminate the watch dog thread.";
+    cancel_watchdog = true;
+    watchdog_thread.join();
+
+    if (!WIFEXITED(wstatus)) {
+      LOG(ERROR) << "Child terminated abnormally, status: " << WEXITSTATUS(wstatus);
+      return false;
+    }
+
+    int status = WEXITSTATUS(wstatus);
+    LOG(DEBUG) << "Child terminated, status: " << status;
+    if (status == 0) {
+      LOG(DEBUG) << "Iorap compilation succeeded";
+      return true;
+    } else {
+      LOG(ERROR) << "Iorap compilation failed";
+      return false;
+    }
+  } else {
+    // we are the child that was forked.
+    std::vector<std::string> argv_vec = MakeCompilerParams(params);
+    std::unique_ptr<ArgString[]> argv_ptr =
+        common::VecToArgv(kCommandFileName, argv_vec);
+
+    std::stringstream argv; // for debugging.
+    for (std::string arg : argv_vec) {
+      argv  << arg << ' ';
+    }
+    LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str();
+
+    controller_params.exec->Execve(kCommandFileName,
+                                          argv_vec,
+                                         /*envp*/nullptr);
+    // This should never return.
+  }
+  return false;
+}
+
+// Gets the perfetto trace infos in the histories.
+std::vector<compiler::CompilationInput> GetPerfettoTraceInfo(
+    const db::DbHandle& db,
+    const std::vector<db::AppLaunchHistoryModel>& histories) {
+  std::vector<compiler::CompilationInput> perfetto_traces;
+
+  for(db::AppLaunchHistoryModel history : histories) {
+    // Get perfetto trace.
+    std::optional<db::RawTraceModel> raw_trace =
+        db::RawTraceModel::SelectByHistoryId(db, history.id);
+    if (!raw_trace) {
+      // This is normal: non-cold launches do not have traces.
+      continue;
+    }
+
+    uint64_t timestamp_limit = std::numeric_limits<uint64_t>::max();
+    // Get corresponding timestamp limit.
+    if (history.report_fully_drawn_ns) {
+      timestamp_limit = *history.report_fully_drawn_ns;
+    } else if (history.total_time_ns) {
+      timestamp_limit = *history.total_time_ns;
+    } else {
+      LOG(DEBUG) << " No timestamp exists. Using the max value.";
+    }
+    perfetto_traces.push_back({raw_trace->file_path, timestamp_limit});
+  }
+  return perfetto_traces;
+}
+
+// Helper struct for printing vector.
+template <class T>
+struct VectorPrinter {
+  std::vector<T>& values;
+};
+
+std::ostream& operator<<(std::ostream& os,
+                      const struct compiler::CompilationInput& perfetto_trace) {
+  os << "file_path: " << perfetto_trace.filename << " "
+     << "timestamp_limit: " << perfetto_trace.timestamp_limit_ns;
+  return os;
+}
+
+template <class T>
+std::ostream& operator<<(std::ostream& os, const struct VectorPrinter<T>& printer) {
+  os << "[\n";
+  for (T i : printer.values) {
+    os << i << ",\n";
+  }
+  os << "]\n";
+  return os;
+}
+
+// Compiled the perfetto traces for an activity.
+bool CompileActivity(const db::DbHandle& db,
+                     int package_id,
+                     const std::string& package_name,
+                     const std::string& activity_name,
+                     int version,
+                     const ControllerParameters& params) {
+  ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER,
+                                           "Compile activity %s",
+                                           activity_name.c_str());
+
+  LOG(DEBUG) << "CompileActivity: " << package_name << "/" << activity_name << "@" << version;
+
+  db::CompiledTraceFileModel output_file =
+      CalculateNewestFilePath(package_name, activity_name, version);
+
+  std::string file_path = output_file.FilePath();
+
+  if (!params.recompile) {
+    if (std::filesystem::exists(file_path)) {
+      LOG(DEBUG) << "compiled trace exists in " << file_path;
+
+      db::VersionedComponentName vcn{package_name, activity_name, version};
+      std::optional<db::PrefetchFileModel> prefetch_file =
+          db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn);
+      if (prefetch_file) {
+        return true;
+      } else {
+        LOG(WARNING) << "Missing corresponding prefetch_file db row for " << vcn;
+        // let it go and compile again. we'll insert the prefetch_file at the bottom.
+      }
+    }
+  }
+
+  std::optional<db::ActivityModel> activity =
+      db::ActivityModel::SelectByNameAndPackageId(db, activity_name.c_str(), package_id);
+  if (!activity) {
+    LOG(ERROR) << "Cannot find activity for package_id: " << package_id
+               <<" activity_name: " <<activity_name;
+    return false;
+  }
+
+  int activity_id = activity->id;
+
+  std::vector<db::AppLaunchHistoryModel> histories =
+      db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity_id);
+
+  {
+    std::vector<compiler::CompilationInput> perfetto_traces =
+        GetPerfettoTraceInfo(db, histories);
+
+    if (perfetto_traces.size() < params.min_traces) {
+      LOG(DEBUG) << "The number of perfetto traces is " << perfetto_traces.size()
+                 <<", which is less than " << params.min_traces;
+      return false;
+    }
+
+    {
+      std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_};
+      last_job_info_.activities_last_compiled_++;
+    }
+
+    // Show the compilation config.
+    LOG(DEBUG) << "Try to compiled package_id: " << package_id
+               << " package_name: " << package_name
+               << " activity_name: " << activity_name
+               << " version: " << version
+               << " file_path: " << file_path
+               << " verbose: " << params.verbose
+               << " perfetto_traces: "
+               << VectorPrinter<compiler::CompilationInput>{perfetto_traces};
+    if (params.inode_textcache) {
+      LOG(DEBUG) << "inode_textcache: " << *params.inode_textcache;
+    }
+
+    CompilerForkParameters compiler_params{perfetto_traces, file_path, params};
+
+    if (!output_file.MkdirWithParents()) {
+      LOG(ERROR) << "Compile activity failed. Failed to mkdirs " << file_path;
+      return false;
+    }
+
+    ScopedFormatTrace atrace_compile_fork(ATRACE_TAG_PACKAGE_MANAGER,
+                                          "Fork+exec iorap.cmd.compiler",
+                                          activity_name.c_str());
+    if (!StartViaFork(compiler_params)) {
+      LOG(ERROR) << "Compilation failed for package_id:" << package_id
+                 << " activity_name: " << activity_name;
+      return false;
+    }
+  }
+
+  std::optional<db::PrefetchFileModel> compiled_trace =
+      db::PrefetchFileModel::Insert(db, activity_id, file_path);
+  if (!compiled_trace) {
+    LOG(ERROR) << "Cannot insert compiled trace activity_id: " << activity_id
+               << " file_path: " << file_path;
+    return false;
+  }
+  return true;
+}
+
+// Compiled the perfetto traces for activities in an package.
+bool CompilePackage(const db::DbHandle& db,
+                    const std::string& package_name,
+                    int version,
+                    const ControllerParameters& params) {
+  ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER,
+                                           "Compile package %s",
+                                           package_name.c_str());
+
+  std::optional<db::PackageModel> package =
+        db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version);
+
+  if (!package) {
+    LOG(ERROR) << "Cannot find package for package_name: "
+               << package_name
+               << " and version "
+               << version;
+    return false;
+  }
+
+  std::vector<db::ActivityModel> activities =
+      db::ActivityModel::SelectByPackageId(db, package->id);
+
+  bool ret = true;
+  for (db::ActivityModel activity : activities) {
+    if (!CompileActivity(db, package->id, package->name, activity.name, version, params)) {
+      ret = false;
+    }
+  }
+  return ret;
+}
+
+// Compiled the perfetto traces for packages in a device.
+bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params) {
+  {
+    std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_};
+    last_job_info_.activities_last_compiled_ = 0;
+  }
+
+  std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db);
+  bool ret = true;
+  for (db::PackageModel package : packages) {
+    if (!CompilePackage(db, package.name, package.version, params)) {
+      ret = false;
+    }
+  }
+
+  {
+    std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_};
+    last_job_info_.last_run_ns_ = time(nullptr);
+  }
+
+  return ret;
+}
+
+// Compiled the perfetto traces for a single package in a device.
+bool CompileSingleAppOnDevice(const db::DbHandle& db,
+                              const ControllerParameters& params,
+                              const std::string& package_name) {
+  std::vector<db::PackageModel> packages = db::PackageModel::SelectByName(db, package_name.c_str());
+  bool ret = true;
+  for (db::PackageModel package : packages) {
+    if (!CompilePackage(db, package.name, package.version, params)) {
+      ret = false;
+    }
+  }
+
+  return ret;
+}
+
+bool Compile(const std::string& db_path, const ControllerParameters& params) {
+  iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path);
+  db::DbHandle db{db_schema.db()};
+  return CompileAppsOnDevice(db, params);
+}
+
+bool Compile(const std::string& db_path,
+             const std::string& package_name,
+             int version,
+             const ControllerParameters& params) {
+  iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path);
+  db::DbHandle db{db_schema.db()};
+  return CompilePackage(db, package_name, version, params);
+}
+
+bool Compile(const std::string& db_path,
+             const std::string& package_name,
+             const std::string& activity_name,
+             int version,
+             const ControllerParameters& params) {
+  iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path);
+  db::DbHandle db{db_schema.db()};
+
+  std::optional<db::PackageModel> package =
+      db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version);
+
+  if (!package) {
+    LOG(ERROR) << "Cannot find package with name "
+               << package_name
+               << " and version "
+               << version;
+    return false;
+  }
+  return CompileActivity(db, package->id, package_name, activity_name, version, params);
+}
+
+static std::string TimeToString(time_t the_time) {
+  tm tm_buf{};
+  tm* tm_ptr = localtime_r(&the_time, &tm_buf);
+
+  if (tm_ptr != nullptr) {
+    char time_buffer[256];
+    strftime(time_buffer, sizeof(time_buffer), "%a %b %d %H:%M:%S %Y", tm_ptr);
+    return std::string{time_buffer};
+  } else {
+    return std::string{"(nullptr)"};
+  }
+}
+
+static std::string GetTimestampForPrefetchFile(const db::PrefetchFileModel& prefetch_file) {
+  std::filesystem::path path{prefetch_file.file_path};
+
+  std::error_code ec{};
+  auto last_write_time = std::filesystem::last_write_time(path, /*out*/ec);
+  if (ec) {
+    return std::string("Failed to get last write time: ") + ec.message();
+  }
+
+  time_t time = decltype(last_write_time)::clock::to_time_t(last_write_time);
+
+  std::string time_str = TimeToString(time);
+  return time_str;
+}
+
+void DumpPackageActivity(const db::DbHandle& db,
+                         ::android::Printer& printer,
+                         const db::PackageModel& package,
+                         const db::ActivityModel& activity) {
+  int package_id = package.id;
+  const std::string& package_name = package.name;
+  int package_version = package.version;
+  const std::string& activity_name = activity.name;
+  db::VersionedComponentName vcn{package_name, activity_name, package_version};
+
+  // com.google.Settings/com.google.Settings.ActivityMain@1234567890
+  printer.printFormatLine("  %s/%s@%d",
+                          package_name.c_str(),
+                          activity_name.c_str(),
+                          package_version);
+
+  std::optional<db::PrefetchFileModel> prefetch_file =
+      db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn);
+
+  std::vector<db::AppLaunchHistoryModel> histories =
+      db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity.id);
+  std::vector<compiler::CompilationInput> perfetto_traces =
+        GetPerfettoTraceInfo(db, histories);
+
+  if (prefetch_file) {
+    bool exists_on_disk = std::filesystem::exists(prefetch_file->file_path);
+
+    std::optional<size_t> prefetch_byte_sum =
+        prefetcher::ReadAhead::PrefetchSizeInBytes(prefetch_file->file_path);
+
+    if (exists_on_disk) {
+      printer.printFormatLine("    Compiled Status: Usable compiled trace");
+    } else {
+      printer.printFormatLine("    Compiled Status: Prefetch file deleted from disk.");
+    }
+
+    if (prefetch_byte_sum) {
+      printer.printFormatLine("      Bytes to be prefetched: %zu", *prefetch_byte_sum);
+    } else {
+      printer.printFormatLine("      Bytes to be prefetched: (bad file path)" );
+    }
+
+    printer.printFormatLine("      Time compiled: %s",
+                            GetTimestampForPrefetchFile(*prefetch_file).c_str());
+    printer.printFormatLine("      %s", prefetch_file->file_path.c_str());
+  } else {
+    size_t size = perfetto_traces.size();
+
+    if (size >= kMinTracesForCompilation) {
+      printer.printFormatLine("    Compiled Status: Raw traces pending compilation (%zu)",
+                              perfetto_traces.size());
+    } else {
+      size_t remaining = kMinTracesForCompilation - size;
+      printer.printFormatLine("    Compiled Status: Need %zu more traces for compilation",
+                              remaining);
+    }
+  }
+
+  printer.printFormatLine("    Raw traces:");
+  printer.printFormatLine("      Trace count: %zu", perfetto_traces.size());
+
+  for (compiler::CompilationInput& compilation_input : perfetto_traces) {
+    std::string& raw_trace_file_name = compilation_input.filename;
+
+    printer.printFormatLine("      %s", raw_trace_file_name.c_str());
+  }
+}
+
+void DumpPackage(const db::DbHandle& db,
+                 ::android::Printer& printer,
+                 db::PackageModel package) {
+  std::vector<db::ActivityModel> activities =
+      db::ActivityModel::SelectByPackageId(db, package.id);
+
+  for (db::ActivityModel& activity : activities) {
+    DumpPackageActivity(db, printer, package, activity);
+  }
+}
+
+void DumpAllPackages(const db::DbHandle& db, ::android::Printer& printer) {
+  printer.printLine("Package history in database:");
+
+  std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db);
+  for (db::PackageModel package : packages) {
+    DumpPackage(db, printer, package);
+  }
+
+  printer.printLine("");
+}
+
+void Dump(const db::DbHandle& db, ::android::Printer& printer) {
+  bool locked = last_job_info_mutex_.try_lock();
+
+  LastJobInfo info = last_job_info_;
+
+  printer.printFormatLine("Background job:");
+  if (!locked) {
+    printer.printLine("""""  (possible deadlock)");
+  }
+  if (info.last_run_ns_ != time_t{0}) {
+    std::string time_str = TimeToString(info.last_run_ns_);
+
+    printer.printFormatLine("  Last run at: %s", time_str.c_str());
+  } else {
+    printer.printFormatLine("  Last run at: (None)");
+  }
+  printer.printFormatLine("  Activities last compiled: %zu", info.activities_last_compiled_);
+
+  printer.printLine("");
+
+  if (locked) {
+    last_job_info_mutex_.unlock();
+  }
+
+  DumpAllPackages(db, printer);
+}
+
+}  // namespace iorap::maintenance
diff --git a/src/maintenance/controller.h b/src/maintenance/controller.h
new file mode 100644
index 0000000..dbe6df9
--- /dev/null
+++ b/src/maintenance/controller.h
@@ -0,0 +1,113 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_
+#define IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_
+
+#include "db/file_models.h"
+#include "inode2filename/inode_resolver.h"
+
+#include <string>
+#include <vector>
+
+namespace android {
+class Printer;
+}  // namespace android
+
+namespace iorap::maintenance {
+
+// Enabling mock for testing purpose.
+class IExec {
+ public:
+  virtual int Execve(const std::string& pathname,
+                     std::vector<std::string>& argv_vec,
+                     char *const envp[]) = 0;
+  virtual int Fork() = 0;
+  virtual ~IExec() = default;
+};
+
+class Exec : public IExec {
+ public:
+   virtual int Execve(const std::string& pathname,
+                      std::vector<std::string>& argv_vec,
+                      char *const envp[]);
+   virtual int Fork();
+};
+
+// Represents the parameters used for compilation controller.
+struct ControllerParameters {
+  bool output_text;
+  // The path of inode2filepath file.
+  std::optional<std::string> inode_textcache;
+  bool verbose;
+  bool recompile;
+  uint64_t min_traces;
+  std::shared_ptr<IExec> exec;
+
+  ControllerParameters(bool output_text,
+                       std::optional<std::string> inode_textcache,
+                       bool verbose,
+                       bool recompile,
+                       uint64_t min_traces,
+                       std::shared_ptr<IExec> exec) :
+    output_text(output_text),
+    inode_textcache(inode_textcache),
+    verbose(verbose),
+    recompile(recompile),
+    min_traces(min_traces),
+    exec(exec) {
+  }
+};
+
+// Control the compilation of perfetto traces in the sqlite db.
+//
+// The strategy now is to compile all the existing perfetto traces for an activity
+// and skip ones if the number of perfetto traces is less than the min_traces.
+//
+// By default, the program doesn't replace the existing compiled trace, it just
+// return true. To force replace the existing compiled trace, set `force` to true.
+//
+// The timestamp limit of the each perfetto trace is determined by `report_fully_drawn_ns`
+// timestamp. If it doesn't exists, use `total_time_ns`. If neither of them exists,
+// use the max.
+
+// Compile all activities of all packages in the database.
+bool Compile(const std::string& db_path, const ControllerParameters& params);
+
+// Compile all activities in the package.
+// If the version is not given, an arbitrary package that has the same name is used.
+bool Compile(const std::string& db_path,
+             const std::string& package_name,
+             int version,
+             const ControllerParameters& params);
+
+// Compile trace for the activity.
+// If the version is not given, an arbitrary package has the same name is used.
+bool Compile(const std::string& db_path,
+             const std::string& package_name,
+             const std::string& activity_name,
+             int version,
+             const ControllerParameters& params);
+// Visible for testing.
+bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params);
+
+bool CompileSingleAppOnDevice(const db::DbHandle& db,
+                              const ControllerParameters& params,
+                              const std::string& package_name);
+
+void Dump(const db::DbHandle& db, ::android::Printer& printer);
+
+} // iorap::maintenance
+
+#endif  // IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_
diff --git a/src/maintenance/db_cleaner.cc b/src/maintenance/db_cleaner.cc
new file mode 100644
index 0000000..56eff70
--- /dev/null
+++ b/src/maintenance/db_cleaner.cc
@@ -0,0 +1,66 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "maintenance/db_cleaner.h"
+
+#include <android-base/file.h>
+
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "db/clean_up.h"
+#include "db/file_models.h"
+#include "db/models.h"
+
+namespace iorap::maintenance {
+
+// Enable foreign key restriction.
+const constexpr char* kForeignKeyOnSql = "PRAGMA foreign_keys = ON;";
+
+void CleanUpDatabase(const db::DbHandle& db,
+                     std::shared_ptr<binder::PackageVersionMap> version_map) {
+  std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db);
+  // Enable cascade deletion.
+  if (!db::DbQueryBuilder::ExecuteOnce(db, kForeignKeyOnSql)) {
+    LOG(ERROR) << "Fail to turn on foreign key restraint!";
+  }
+
+  for (db::PackageModel package : packages) {
+    std::optional<int64_t> version = version_map->Find(package.name);
+    if (!version) {
+      LOG(DEBUG) << "Fail to find version for package " << package.name
+                 << " with version " << package.version
+                 << ". The package manager may be down.";
+      continue;
+    }
+    // Package is cleanup if it
+    // * is not in the version map, it may be uninstalled
+    // * has an different version with the latest one
+    if (*version != package.version) {
+      db::CleanUpFilesForPackage(db, package.id, package.name, package.version);
+      if (!package.Delete()) {
+        LOG(ERROR) << "Fail to delete package " << package.name
+                   << " with version " << package.version;
+      }
+    }
+  }
+}
+
+}  // namespace iorap::maintenance
diff --git a/src/maintenance/db_cleaner.h b/src/maintenance/db_cleaner.h
new file mode 100644
index 0000000..1168bd5
--- /dev/null
+++ b/src/maintenance/db_cleaner.h
@@ -0,0 +1,34 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_
+#define IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_
+
+#include <android/content/pm/IPackageManagerNative.h>
+
+#include <string>
+#include <vector>
+
+#include "binder/package_version_map.h"
+#include "db/file_models.h"
+
+namespace iorap::maintenance {
+
+// Clean up the database.
+// Remove all relevant data for old-version packages.
+void CleanUpDatabase(const db::DbHandle& db,
+                     std::shared_ptr<binder::PackageVersionMap> version_map);
+}  // namespace iorap::maintenance
+
+#endif  // IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_
diff --git a/src/maintenance/main.cc b/src/maintenance/main.cc
new file mode 100644
index 0000000..16b2a35
--- /dev/null
+++ b/src/maintenance/main.cc
@@ -0,0 +1,194 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/debug.h"
+#include "compiler/compiler.h"
+#include "maintenance/controller.h"
+#include "db/clean_up.h"
+
+#include <android-base/parseint.h>
+#include <android-base/properties.h>
+#include <android-base/logging.h>
+
+#include <iostream>
+#include <optional>
+
+#if defined(IORAP_MAINTENANCE_MAIN)
+
+namespace iorap::maintenance {
+
+void Usage(char** argv) {
+  std::cerr << "Usage: " << argv[0] << " <path of sqlite db>" << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Compile the perfetto trace for an package and activity." << std::endl;
+  std::cerr << "  The info of perfetto trace is stored in the sqlite db." << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Optional flags:" << std::endl;
+  std::cerr << "    --package $,-p $           Package name." << std::endl;
+  std::cerr << "    --version $,-ve $          Package version." << std::endl;
+  std::cerr << "    --activity $,-a $          Activity name." << std::endl;
+  std::cerr << "    --inode-textcache $,-it $  Resolve inode->filename from textcache." << std::endl;
+  std::cerr << "    --help,-h                  Print this Usage." << std::endl;
+  std::cerr << "    --recompile,-r             Force re-compilation, which replace the existing compiled trace ." << std::endl;
+  std::cerr << "    --purge-package,-pp        Purge all files associated with a package." << std::endl;
+  std::cerr << "    --verbose,-v               Set verbosity (default off)." << std::endl;
+  std::cerr << "    --output-text,-ot          Output ascii text instead of protobuf (default off)." << std::endl;
+  std::cerr << "    --min_traces,-mt           The min number of perfetto traces needed for compilation (default 1)."
+            << std::endl;
+  exit(1);
+}
+
+
+int Main(int argc, char** argv){
+  android::base::InitLogging(argv);
+  android::base::SetLogger(android::base::StderrLogger);
+
+  if (argc == 1) {
+    // Need at least 1 input file to do anything.
+    Usage(argv);
+  }
+
+  std::vector<std::string> arg_input_filenames;
+  std::optional<std::string> arg_package;
+  std::optional<std::string> arg_purge_package;
+  int arg_version = -1;
+  std::optional<std::string> arg_activity;
+  std::optional<std::string> arg_inode_textcache;
+  bool recompile = false;
+  bool enable_verbose = false;
+  bool arg_output_text = false;
+  uint64_t arg_min_traces = 1;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+    bool has_arg_next = (arg+1)<argc;
+    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+
+    if (argstr == "--help" || argstr == "-h") {
+      Usage(argv);
+    } else if (argstr == "--package" || argstr == "-p") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --package <value>" << std::endl;
+        return 1;
+      }
+      arg_package = arg_next;
+      ++arg;
+    } else if (argstr == "--version" || argstr == "-ve") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --version <value>" << std::endl;
+        return 1;
+      }
+      int version;
+      if (!android::base::ParseInt<int>(arg_next, &version)) {
+        std::cerr << "Invalid --version " << arg_next << std::endl;
+        return 1;
+      }
+      arg_version = version;
+      ++arg;
+    } else if (argstr == "--activity" || argstr == "-a") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --activity <value>" << std::endl;
+        return 1;
+      }
+      arg_activity = arg_next;
+      ++arg;
+    } else if (argstr == "--inode-textcache" || argstr == "-it") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --inode-textcache <value>" << std::endl;
+        return 1;
+      }
+      arg_inode_textcache = arg_next;
+      ++arg;
+    } else if (argstr == "--purge-package" || argstr == "-pp") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --purge-package <value>" << std::endl;
+        return 1;
+      }
+      arg_purge_package = arg_next;
+      ++arg;
+    }
+    else if (argstr == "--verbose" || argstr == "-v") {
+      enable_verbose = true;
+    } else if (argstr == "--recompile" || argstr == "-r") {
+      recompile = true;
+    } else if (argstr == "--output-text" || argstr == "-ot") {
+      arg_output_text = true;
+    } else if (argstr == "--min_traces" || argstr == "-mt") {
+      if (!has_arg_next) {
+        std::cerr << "Missing --min_traces <value>" << std::endl;
+        return 1;
+      }
+      arg_min_traces = std::stoul(arg_next);
+      ++arg;
+    } else {
+      arg_input_filenames.push_back(argstr);
+    }
+  }
+
+  if (arg_input_filenames.empty()) {
+    LOG(ERROR) << "Missing filename to a sqlite database.";
+    Usage(argv);
+  } else if (arg_input_filenames.size() > 1) {
+    LOG(ERROR) << "More than one filename to a sqlite database.";
+    Usage(argv);
+  }
+
+  std::string db_path = arg_input_filenames[0];
+
+  if (enable_verbose) {
+    android::base::SetMinimumLogSeverity(android::base::VERBOSE);
+
+    LOG(VERBOSE) << "Verbose check";
+    LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild;
+  } else {
+    android::base::SetMinimumLogSeverity(android::base::DEBUG);
+  }
+
+  if (arg_purge_package) {
+    db::CleanUpFilesForPackage(db_path, *arg_purge_package);
+    return 0;
+    // Don't do any more work because SchemaModel can only be created once.
+  }
+
+  maintenance::ControllerParameters params{
+    arg_output_text,
+    arg_inode_textcache,
+    enable_verbose,
+    recompile,
+    arg_min_traces,
+    std::make_shared<Exec>()};
+
+  int ret_code = 0;
+  if (arg_package && arg_activity) {
+    ret_code = !Compile(std::move(db_path),
+                        std::move(*arg_package),
+                        std::move(*arg_activity),
+                        arg_version,
+                        params);
+  } else if (arg_package) {
+    ret_code = !Compile(std::move(db_path), std::move(*arg_package), arg_version, params);
+  } else {
+    ret_code = !Compile(std::move(db_path), params);
+  }
+  return ret_code;
+}
+
+} // iorap::maintenance
+
+int main(int argc, char** argv) {
+  return ::iorap::maintenance::Main(argc, argv);
+}
+
+
+#endif  // IORAP_MAINTENANCE_MAIN
diff --git a/src/manager/event_manager.cc b/src/manager/event_manager.cc
index 3b75b89..bae676b 100644
--- a/src/manager/event_manager.cc
+++ b/src/manager/event_manager.cc
@@ -14,94 +14,94 @@
  * limitations under the License.
  */
 
+#include "binder/package_version_map.h"
 #include "common/debug.h"
 #include "common/expected.h"
+#include "common/printer.h"
+#include "common/rx_async.h"
+#include "common/trace.h"
+#include "db/app_component_name.h"
+#include "db/file_models.h"
+#include "db/models.h"
+#include "maintenance/controller.h"
+#include "maintenance/db_cleaner.h"
 #include "manager/event_manager.h"
 #include "perfetto/rx_producer.h"
+#include "prefetcher/read_ahead.h"
+#include "prefetcher/task_id.h"
 
+#include <android-base/chrono_utils.h>
+#include <android-base/strings.h>
 #include <android-base/properties.h>
 #include <rxcpp/rx.hpp>
+#include <server_configurable_flags/get_flags.h>
+#include <utils/misc.h>
+#include <utils/Trace.h>
 
 #include <atomic>
+#include <filesystem>
 #include <functional>
+#include <type_traits>
+#include <unordered_map>
 
 using rxcpp::observe_on_one_worker;
 
 namespace iorap::manager {
 
-using binder::RequestId;
 using binder::AppLaunchEvent;
+using binder::DexOptEvent;
+using binder::JobScheduledEvent;
+using binder::RequestId;
+using binder::TaskResult;
+
+using common::AsyncPool;
+using common::RxAsync;
+
 using perfetto::PerfettoStreamCommand;
 using perfetto::PerfettoTraceProto;
 
-struct AppComponentName {
-  std::string package;
-  std::string activity_name;
+using db::AppComponentName;
 
-  static bool HasAppComponentName(const std::string& s) {
-    return s.find('/') != std::string::npos;
-  }
+static std::atomic<bool> s_tracing_allowed{false};
+static std::atomic<bool> s_readahead_allowed{false};
+static std::atomic<uint64_t> s_min_traces{3};
 
-  // "com.foo.bar/.A" -> {"com.foo.bar", ".A"}
-  static AppComponentName FromString(const std::string& s) {
-    constexpr const char delimiter = '/';
-    std::string package = s.substr(0, delimiter);
+struct PackageBlacklister {
+  // "x.y.z;foo.bar.baz" colon-separated list of substrings
+  PackageBlacklister(std::string blacklist_string) {
+    LOG(VERBOSE) << "Configuring package blacklister with string: " << blacklist_string;
 
-    std::string activity_name = s;
-    activity_name.erase(0, s.find(delimiter) + sizeof(delimiter));
+    std::vector<std::string> split = ::android::base::Split(blacklist_string, ";");
 
-    return {std::move(package), std::move(activity_name)};
-  }
-
-  // {"com.foo.bar", ".A"} -> "com.foo.bar/.A"
-  std::string ToString() const {
-    return package + "/" + activity_name;
+    // Ignore any l/r whitespace or empty strings.
+    for (const std::string& s : split) {
+      std::string t = ::android::base::Trim(s);
+      if (!t.empty()) {
+        LOG(INFO) << "Blacklisted package: " << t << "; will not optimize.";
+        packages_.push_back(t);
+      }
+    }
   }
 
-  /*
-   * '/' is encoded into %2F
-   * '%' is encoded into %25
-   *
-   * This allows the component name to be be used as a file name
-   * ('/' is illegal due to being a path separator) with minimal
-   * munging.
-   */
+  PackageBlacklister() = default;
 
-  // "com.foo.bar%2F.A%25" -> {"com.foo.bar", ".A%"}
-  static AppComponentName FromUrlEncodedString(const std::string& s) {
-    std::string cpy = s;
-    Replace(cpy, "%2F", "/");
-    Replace(cpy, "%25", "%");
+  bool IsBlacklisted(const std::string& package_name) const {
+    return std::find(packages_.begin(), packages_.end(), package_name) != packages_.end();
+  }
 
-    return FromString(cpy);
+  bool IsBlacklisted(const AppComponentName& component_name) const {
+    return IsBlacklisted(component_name.package);
   }
 
-  // {"com.foo.bar", ".A%"} -> "com.foo.bar%2F.A%25"
-  std::string ToUrlEncodedString() const {
-    std::string s = ToString();
-    Replace(s, "%", "%25");
-    Replace(s, "/", "%2F");
-    return s;
+  bool IsBlacklisted(const std::optional<AppComponentName>& component_name) const {
+    return component_name.has_value() && IsBlacklisted(component_name->package);
   }
 
  private:
-  static bool Replace(std::string& str, const std::string& from, const std::string& to) {
-    // TODO: call in a loop to replace all occurrences, not just the first one.
-    const size_t start_pos = str.find(from);
-    if (start_pos == std::string::npos) {
-      return false;
-    }
-
-    str.replace(start_pos, from.length(), to);
-
-    return true;
-}
+  std::vector<std::string> packages_;
 };
 
-std::ostream& operator<<(std::ostream& os, const AppComponentName& name) {
-  os << name.ToString();
-  return os;
-}
+using PackageVersionMap = std::unordered_map<std::string, int64_t>;
 
 // Main logic of the #OnAppLaunchEvent scan method.
 //
@@ -112,26 +112,74 @@ std::ostream& operator<<(std::ostream& os, const AppComponentName& name) {
 // of #scan to another.
 struct AppLaunchEventState {
   std::optional<AppComponentName> component_name_;
-
+  // Sequence ID is shared amongst the same app launch sequence,
+  // but changes whenever a new app launch sequence begins.
+  size_t sequence_id_ = static_cast<size_t>(-1);
+  std::optional<AppLaunchEvent::Temperature> temperature_;
+
+  // Push data to perfetto rx chain for associating
+  // the raw_trace with the history_id.
+  std::optional<rxcpp::subscriber<int>> history_id_subscriber_;
+  rxcpp::observable<int> history_id_observable_;
+
+  std::optional<uint64_t> intent_started_ns_;
+  std::optional<uint64_t> total_time_ns_;
+
+  // Used by kReportFullyDrawn to find the right history_id.
+  // We assume no interleaving between different sequences.
+  // This assumption is checked in the Java service code.
+  std::optional<uint64_t> recent_history_id_;
+
+  // labeled as 'shared' due to rx not being able to handle move-only objects.
+  // lifetime: in practice equivalent to unique_ptr.
+  std::shared_ptr<prefetcher::ReadAhead> read_ahead_;
+  bool allowed_readahead_{true};
+  bool is_read_ahead_{false};
+  std::optional<prefetcher::TaskId> read_ahead_task_;
+
+  bool allowed_tracing_{true};
   bool is_tracing_{false};
   std::optional<rxcpp::composite_subscription> rx_lifetime_;
   std::vector<rxcpp::composite_subscription> rx_in_flight_;
 
+  PackageBlacklister package_blacklister_{};
+
   borrowed<perfetto::RxProducerFactory*> perfetto_factory_;  // not null
   borrowed<observe_on_one_worker*> thread_;  // not null
   borrowed<observe_on_one_worker*> io_thread_;  // not null
+  borrowed<AsyncPool*> async_pool_;  // not null
+
+  std::shared_ptr<binder::PackageVersionMap> version_map_;
 
   explicit AppLaunchEventState(borrowed<perfetto::RxProducerFactory*> perfetto_factory,
+                               bool allowed_readahead,
+                               bool allowed_tracing,
+                               PackageBlacklister package_blacklister,
                                borrowed<observe_on_one_worker*> thread,
-                               borrowed<observe_on_one_worker*> io_thread) {
+                               borrowed<observe_on_one_worker*> io_thread,
+                               borrowed<AsyncPool*> async_pool,
+                               std::shared_ptr<binder::PackageVersionMap> version_map)
+    : read_ahead_{std::make_shared<prefetcher::ReadAhead>()}
+  {
     perfetto_factory_ = perfetto_factory;
     DCHECK(perfetto_factory_ != nullptr);
 
+    allowed_readahead_ = allowed_readahead;
+    allowed_tracing_ = allowed_tracing;
+
+    package_blacklister_ = package_blacklister;
+
     thread_ = thread;
     DCHECK(thread_ != nullptr);
 
     io_thread_ = io_thread;
     DCHECK(io_thread_ != nullptr);
+
+    async_pool_ = async_pool;
+    DCHECK(async_pool_ != nullptr);
+
+    version_map_ = version_map;
+    DCHECK(version_map_ != nullptr);
   }
 
   // Updates the values in this struct only as a side effect.
@@ -141,11 +189,57 @@ struct AppLaunchEventState {
   void OnNewEvent(const AppLaunchEvent& event) {
     LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent: " << event;
 
+    android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER,
+                                       "IorapNativeService::OnAppLaunchEvent"};
+
     using Type = AppLaunchEvent::Type;
 
+    DCHECK_GE(event.sequence_id, 0);
+    sequence_id_ = static_cast<size_t>(event.sequence_id);
+    allowed_readahead_ = s_readahead_allowed;
+    allowed_tracing_ = s_tracing_allowed;
+
     switch (event.type) {
       case Type::kIntentStarted: {
+        const std::string& package_name = event.intent_proto->component().package_name();
+        const std::string& class_name = event.intent_proto->component().class_name();
+        AppComponentName component_name{package_name, class_name};
+        component_name = component_name.Canonicalize();
+        component_name_ = component_name;
+
+        if (package_blacklister_.IsBlacklisted(component_name)) {
+          LOG(DEBUG) << "kIntentStarted: package " << component_name.package
+                     << " ignored due to blacklisting.";
+          break;
+        }
+
+        // Create a new history ID chain for each new app start-up sequence.
+        auto history_id_observable = rxcpp::observable<>::create<int>(
+          [&](rxcpp::subscriber<int> subscriber) {
+            history_id_subscriber_ = std::move(subscriber);
+            LOG(VERBOSE) << " set up the history id subscriber ";
+          })
+          .tap([](int history_id) { LOG(VERBOSE) << " tap rx history id = " << history_id; })
+          .replay(1);  // Remember the history id in case we subscribe too late.
+
+        history_id_observable_ = history_id_observable;
+
+        // Immediately turn observable hot, creating the subscriber.
+        history_id_observable.connect();
+
         DCHECK(!IsTracing());
+
+        // The time should be set before perfetto tracing.
+        // Record the timestamp even no perfetto tracing is triggered,
+        // because the tracing may start in the following ActivityLaunched
+        // event. Otherwise, there will be no starting timestamp and
+        // trace without starting timestamp is not considered for compilation.
+        if (event.timestamp_nanos >= 0) {
+          intent_started_ns_ = event.timestamp_nanos;
+        } else {
+          LOG(WARNING) << "Negative event timestamp: " << event.timestamp_nanos;
+        }
+
         // Optimistically start tracing if we have the activity in the intent.
         if (!event.intent_proto->has_component()) {
           // Can't do anything if there is no component in the proto.
@@ -153,51 +247,93 @@ struct AppLaunchEventState {
           break;
         }
 
-        const std::string& package_name = event.intent_proto->component().package_name();
-        const std::string& class_name = event.intent_proto->component().class_name();
-        AppComponentName component_name{package_name, class_name};
-
-        component_name_ = component_name;
-        rx_lifetime_ = StartTracing(std::move(component_name));
+        if (allowed_readahead_) {
+          StartReadAhead(sequence_id_, component_name);
+        }
+        if (allowed_tracing_ && !IsReadAhead()) {
+          rx_lifetime_ = StartTracing(std::move(component_name));
+        }
 
         break;
       }
       case Type::kIntentFailed:
+        if (package_blacklister_.IsBlacklisted(component_name_)) {
+          LOG(VERBOSE) << "kIntentFailed: package " << component_name_->package
+                       << " ignored due to blacklisting.";
+          break;
+        }
+
         AbortTrace();
+        AbortReadAhead();
+
+        if (history_id_subscriber_) {
+          history_id_subscriber_->on_error(rxcpp::util::make_error_ptr(
+            std::ios_base::failure("Aborting due to intent failed")));
+          history_id_subscriber_ = std::nullopt;
+        }
+
         break;
       case Type::kActivityLaunched: {
+        const std::string& title = event.activity_record_proto->identifier().title();
+        if (!AppComponentName::HasAppComponentName(title)) {
+          // Proto comment claim this is sometimes a window title.
+          // We need the actual 'package/component' here, so just ignore it if it's a title.
+          LOG(WARNING) << "App launched without a component name: " << event;
+          break;
+        }
+
+        AppComponentName component_name = AppComponentName::FromString(title);
+        component_name = component_name.Canonicalize();
+        component_name_ = component_name;
+
+        if (package_blacklister_.IsBlacklisted(component_name_)) {
+          LOG(VERBOSE) << "kActivityLaunched: package " << component_name_->package
+                       << " ignored due to blacklisting.";
+          break;
+        }
+
         // Cancel tracing for warm/hot.
         // Restart tracing if the activity was unexpected.
 
         AppLaunchEvent::Temperature temperature = event.temperature;
+        temperature_ = temperature;
         if (temperature != AppLaunchEvent::Temperature::kCold) {
           LOG(DEBUG) << "AppLaunchEventState#OnNewEvent aborting trace due to non-cold temperature";
+
           AbortTrace();
-        } else if (!IsTracing()) {  // and the temperature is Cold.
+          AbortReadAhead();
+        } else if (!IsTracing() && !IsReadAhead()) {  // and the temperature is Cold.
           // Start late trace when intent didn't have a component name
           LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent need to start new trace";
 
-          const std::string& title = event.activity_record_proto->identifier().title();
-          if (!AppComponentName::HasAppComponentName(title)) {
-            // Proto comment claim this is sometimes a window title.
-            // We need the actual 'package/component' here, so just ignore it if it's a title.
-            LOG(WARNING) << "App launched without a component name: " << event;
-            break;
+          if (allowed_readahead_ && !IsReadAhead()) {
+            StartReadAhead(sequence_id_, component_name);
+          }
+          if (allowed_tracing_ && !IsTracing() && !IsReadAhead()) {
+            rx_lifetime_ = StartTracing(std::move(component_name));
           }
-
-          AppComponentName component_name = AppComponentName::FromString(title);
-
-          component_name_ = component_name;
-          rx_lifetime_ = StartTracing(std::move(component_name));
         } else {
           // FIXME: match actual component name against intent component name.
           // abort traces if they don't match.
 
-          LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already tracing";
+          if (allowed_tracing_) {
+            LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already tracing";
+          }
+          LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already doing readahead";
         }
         break;
       }
       case Type::kActivityLaunchFinished:
+        if (package_blacklister_.IsBlacklisted(component_name_)) {
+          LOG(VERBOSE) << "kActivityLaunchFinished: package " << component_name_->package
+                       << " ignored due to blacklisting.";
+          break;
+        }
+
+        if (event.timestamp_nanos >= 0) {
+           total_time_ns_ = event.timestamp_nanos;
+        }
+        RecordDbLaunchHistory();
         // Finish tracing and collect trace buffer.
         //
         // TODO: this happens automatically when perfetto finishes its
@@ -205,22 +341,138 @@ struct AppLaunchEventState {
         if (IsTracing()) {
           MarkPendingTrace();
         }
+        FinishReadAhead();
         break;
       case Type::kActivityLaunchCancelled:
+        if (package_blacklister_.IsBlacklisted(component_name_)) {
+          LOG(VERBOSE) << "kActivityLaunchCancelled: package " << component_name_->package
+                       << " ignored due to blacklisting.";
+          break;
+        }
+
         // Abort tracing.
         AbortTrace();
+        AbortReadAhead();
+        break;
+      case Type::kReportFullyDrawn: {
+        if (package_blacklister_.IsBlacklisted(component_name_)) {
+          LOG(VERBOSE) << "kReportFullyDrawn: package " << component_name_->package
+                       << " ignored due to blacklisting.";
+          break;
+        }
+
+        if (!recent_history_id_) {
+          LOG(WARNING) << "Dangling kReportFullyDrawn event";
+          return;
+        }
+        UpdateReportFullyDrawn(*recent_history_id_, event.timestamp_nanos);
+        recent_history_id_ = std::nullopt;
         break;
+      }
       default:
         DCHECK(false) << "invalid type: " << event;  // binder layer should've rejected this.
         LOG(ERROR) << "invalid type: " << event;  // binder layer should've rejected this.
     }
   }
 
+  // Is there an in-flight readahead task currently?
+  bool IsReadAhead() const {
+    return read_ahead_task_.has_value();
+  }
+
+  // Gets the compiled trace.
+  // If a compiled trace exists in sqlite, use that one. Otherwise, try
+  // to find a prebuilt one.
+  std::optional<std::string> GetCompiledTrace(const AppComponentName& component_name) {
+    ScopedFormatTrace atrace_get_compiled_trace(ATRACE_TAG_ACTIVITY_MANAGER, "GetCompiledTrace");
+    // Firstly, try to find the compiled trace from sqlite.
+    android::base::Timer timer{};
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+    std::optional<int> version =
+        version_map_->GetOrQueryPackageVersion(component_name.package);
+    if (!version) {
+      LOG(DEBUG) << "The version is NULL, maybe package manager is down.";
+      return std::nullopt;
+    }
+    db::VersionedComponentName vcn{component_name.package,
+                                   component_name.activity_name,
+                                   *version};
+
+    std::optional<db::PrefetchFileModel> compiled_trace =
+          db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn);
+
+    std::chrono::milliseconds duration_ms = timer.duration();
+    LOG(DEBUG) << "EventManager: Looking up compiled trace done in "
+               << duration_ms.count() // the count of ticks.
+               << "ms.";
+
+    if (compiled_trace) {
+      if (std::filesystem::exists(compiled_trace->file_path)) {
+        return compiled_trace->file_path;
+      } else {
+        LOG(DEBUG) << "Compiled trace in sqlite doesn't exists. file_path: "
+                   << compiled_trace->file_path;
+      }
+    }
+
+    LOG(DEBUG) << "Cannot find compiled trace in sqlite for package_name: "
+               << component_name.package
+               << " activity_name: "
+               << component_name.activity_name;
+
+    // If sqlite doesn't have the compiled trace, try the prebuilt path.
+    std::string file_path = "/product/iorap-trace/";
+    file_path += component_name.ToMakeFileSafeEncodedPkgString();
+    file_path += ".compiled_trace.pb";
+
+    if (std::filesystem::exists(file_path)) {
+      return file_path;
+    }
+
+    LOG(DEBUG) << "Prebuilt compiled trace doesn't exists. file_path: "
+               << file_path;
+
+    return std::nullopt;
+  }
+
+  void StartReadAhead(size_t id, const AppComponentName& component_name) {
+    DCHECK(allowed_readahead_);
+    DCHECK(!IsReadAhead());
+
+    std::optional<std::string> file_path = GetCompiledTrace(component_name);
+    if (!file_path) {
+      LOG(VERBOSE) << "Cannot find a compiled trace.";
+      return;
+    }
+
+    prefetcher::TaskId task{id, *file_path};
+    read_ahead_->BeginTask(task);
+    // TODO: non-void return signature?
+
+    read_ahead_task_ = std::move(task);
+  }
+
+  void FinishReadAhead() {
+    // if no readahead task exist, do nothing.
+    if (!IsReadAhead()){
+      return;
+    }
+
+    read_ahead_->FinishTask(*read_ahead_task_);
+    read_ahead_task_ = std::nullopt;
+  }
+
+  void AbortReadAhead() {
+    FinishReadAhead();
+  }
+
   bool IsTracing() const {
     return is_tracing_;
   }
 
-  rxcpp::composite_subscription StartTracing(AppComponentName component_name) {
+  std::optional<rxcpp::composite_subscription> StartTracing(
+      AppComponentName component_name) {
+    DCHECK(allowed_tracing_);
     DCHECK(!IsTracing());
 
     auto /*observable<PerfettoStreamCommand>*/ perfetto_commands =
@@ -245,30 +497,73 @@ struct AppLaunchEventState {
 
     rxcpp::composite_subscription lifetime;
 
-    trace_proto_stream
+    auto stream_via_threads = trace_proto_stream
       .tap([](const PerfettoTraceProto& trace_proto) {
              LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (1)";
            })
+      .combine_latest(history_id_observable_)
       .observe_on(*thread_)   // All work prior to 'observe_on' is handled on thread_.
       .subscribe_on(*thread_)   // All work prior to 'observe_on' is handled on thread_.
       .observe_on(*io_thread_)  // Write data on an idle-class-priority thread.
-      .tap([](const PerfettoTraceProto& trace_proto) {
+      .tap([](std::tuple<PerfettoTraceProto, int> trace_tuple) {
              LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (2)";
-           })
-      .as_blocking()  // TODO: remove.
-      .subscribe(/*out*/lifetime,
-        /*on_next*/[component_name]
-        (PerfettoTraceProto trace_proto) {
-          std::string file_path = "/data/misc/iorapd/";
-          file_path += component_name.ToUrlEncodedString();
-          file_path += ".perfetto_trace.pb";
+           });
 
-          // TODO: timestamp each file into a subdirectory.
+    std::optional<int> version =
+        version_map_->GetOrQueryPackageVersion(component_name_->package);
+    if (!version) {
+      LOG(DEBUG) << "The version is NULL, maybe package manager is down.";
+      return std::nullopt;
+    }
+    db::VersionedComponentName versioned_component_name{component_name.package,
+                                                        component_name.activity_name,
+                                                        *version};
+    lifetime = RxAsync::SubscribeAsync(*async_pool_,
+        std::move(stream_via_threads),
+        /*on_next*/[versioned_component_name]
+        (std::tuple<PerfettoTraceProto, int> trace_tuple) {
+          PerfettoTraceProto& trace_proto = std::get<0>(trace_tuple);
+          int history_id = std::get<1>(trace_tuple);
+
+          db::PerfettoTraceFileModel file_model =
+            db::PerfettoTraceFileModel::CalculateNewestFilePath(versioned_component_name);
+
+          std::string file_path = file_model.FilePath();
+
+          ScopedFormatTrace atrace_write_to_file(ATRACE_TAG_ACTIVITY_MANAGER,
+                                                 "Perfetto Write Trace To File %s",
+                                                 file_path.c_str());
+
+          if (!file_model.MkdirWithParents()) {
+            LOG(ERROR) << "Cannot save TraceBuffer; failed to mkdirs " << file_path;
+            return;
+          }
 
           if (!trace_proto.WriteFullyToFile(file_path)) {
             LOG(ERROR) << "Failed to save TraceBuffer to " << file_path;
           } else {
             LOG(INFO) << "Perfetto TraceBuffer saved to file: " << file_path;
+
+            ScopedFormatTrace atrace_update_raw_traces_table(
+                ATRACE_TAG_ACTIVITY_MANAGER,
+                "update raw_traces table history_id = %d",
+                history_id);
+            db::DbHandle db{db::SchemaModel::GetSingleton()};
+            std::optional<db::RawTraceModel> raw_trace =
+                db::RawTraceModel::Insert(db, history_id, file_path);
+
+            if (!raw_trace) {
+              LOG(ERROR) << "Failed to insert raw_traces for " << file_path;
+            } else {
+              LOG(VERBOSE) << "Inserted into db: " << *raw_trace;
+
+              ScopedFormatTrace atrace_delete_older_files(
+                  ATRACE_TAG_ACTIVITY_MANAGER,
+                  "Delete older trace files for package");
+
+              // Ensure we don't have too many files per-app.
+              db::PerfettoTraceFileModel::DeleteOlderFiles(db, versioned_component_name);
+            }
           }
         },
         /*on_error*/[](rxcpp::util::error_ptr err) {
@@ -282,12 +577,23 @@ struct AppLaunchEventState {
 
   void AbortTrace() {
     LOG(VERBOSE) << "AppLaunchEventState - AbortTrace";
+
+    // if the tracing is not running, do nothing.
+    if (!IsTracing()){
+      return;
+    }
+
     is_tracing_ = false;
     if (rx_lifetime_) {
       // TODO: it would be good to call perfetto Destroy.
 
+      rx_in_flight_.erase(std::remove(rx_in_flight_.begin(),
+                                      rx_in_flight_.end(), *rx_lifetime_),
+                          rx_in_flight_.end());
+
       LOG(VERBOSE) << "AppLaunchEventState - AbortTrace - Unsubscribe";
       rx_lifetime_->unsubscribe();
+
       rx_lifetime_.reset();
     }
   }
@@ -309,8 +615,187 @@ struct AppLaunchEventState {
       LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace - lifetime was empty";
     }
 
+    is_tracing_ = false;
     // FIXME: how do we clear this vector?
   }
+
+  void RecordDbLaunchHistory() {
+    std::optional<db::AppLaunchHistoryModel> history = InsertDbLaunchHistory();
+
+    // RecordDbLaunchHistory happens-after kIntentStarted
+    if (!history_id_subscriber_.has_value()) {
+      LOG(WARNING) << "Logic error? Should always have a subscriber here.";
+      return;
+    }
+
+    // Ensure that the history id rx chain is terminated either with an error or with
+    // the newly inserted app_launch_histories.id
+    if (!history) {
+      history_id_subscriber_->on_error(rxcpp::util::make_error_ptr(
+          std::ios_base::failure("Failed to insert history id")));
+      recent_history_id_ = std::nullopt;
+    } else {
+      // Note: we must have already subscribed, or this value will disappear.
+      LOG(VERBOSE) << "history_id_subscriber on_next history_id=" << history->id;
+      history_id_subscriber_->on_next(history->id);
+      history_id_subscriber_->on_completed();
+
+      recent_history_id_ = history->id;
+    }
+    history_id_subscriber_ = std::nullopt;
+  }
+
+  std::optional<db::AppLaunchHistoryModel> InsertDbLaunchHistory() {
+    // TODO: deferred queue into a different lower priority thread.
+    if (!component_name_ || !temperature_) {
+      LOG(VERBOSE) << "Skip RecordDbLaunchHistory, no component name available.";
+
+      return std::nullopt;
+    }
+
+    android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER,
+                               "IorapNativeService::RecordDbLaunchHistory"};
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+
+    using namespace iorap::db;
+
+    std::optional<int> version =
+        version_map_->GetOrQueryPackageVersion(component_name_->package);
+    if (!version) {
+      LOG(DEBUG) << "The version is NULL, maybe package manager is down.";
+      return std::nullopt;
+    }
+    std::optional<ActivityModel> activity =
+        ActivityModel::SelectOrInsert(db,
+                                      component_name_->package,
+                                      *version,
+                                      component_name_->activity_name);
+
+    if (!activity) {
+      LOG(WARNING) << "Failed to query activity row for : " << *component_name_;
+      return std::nullopt;
+    }
+
+    auto temp = static_cast<db::AppLaunchHistoryModel::Temperature>(*temperature_);
+
+    std::optional<AppLaunchHistoryModel> alh =
+        AppLaunchHistoryModel::Insert(db,
+                                      activity->id,
+                                      temp,
+                                      IsTracing(),
+                                      IsReadAhead(),
+                                      intent_started_ns_,
+                                      total_time_ns_,
+                                      // ReportFullyDrawn event normally occurs after this. Need update later.
+                                      /* report_fully_drawn_ns= */ std::nullopt);
+    //Repo
+    if (!alh) {
+      LOG(WARNING) << "Failed to insert app_launch_histories row";
+      return std::nullopt;
+    }
+
+    LOG(VERBOSE) << "RecordDbLaunchHistory: " << *alh;
+    return alh;
+  }
+
+  void UpdateReportFullyDrawn(int history_id, uint64_t timestamp_ns) {
+    LOG(DEBUG) << "Update kReportFullyDrawn for history_id:"
+               << history_id
+               << " timestamp_ns: "
+               << timestamp_ns;
+
+    android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER,
+                               "IorapNativeService::UpdateReportFullyDrawn"};
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+
+    bool result =
+        db::AppLaunchHistoryModel::UpdateReportFullyDrawn(db,
+                                                          history_id,
+                                                          timestamp_ns);
+
+    if (!result) {
+      LOG(WARNING) << "Failed to update app_launch_histories row";
+    }
+  }
+};
+
+struct AppLaunchEventDefender {
+  binder::AppLaunchEvent::Type last_event_type_{binder::AppLaunchEvent::Type::kUninitialized};
+
+  enum class Result {
+    kAccept,      // Pass-through the new event.
+    kOverwrite,   // Overwrite the new event with a different event.
+    kReject       // Completely reject the new event, it will not be delivered.
+  };
+
+  Result OnAppLaunchEvent(binder::RequestId request_id,
+                          const binder::AppLaunchEvent& event,
+                          binder::AppLaunchEvent* overwrite) {
+    using Type = binder::AppLaunchEvent::Type;
+    CHECK(overwrite != nullptr);
+
+    // Ensure only legal transitions are allowed.
+    switch (last_event_type_) {
+      case Type::kUninitialized:
+      case Type::kIntentFailed:
+      case Type::kActivityLaunchCancelled:
+      case Type::kReportFullyDrawn: {  // From a terminal state, only go to kIntentStarted
+        if (event.type != Type::kIntentStarted) {
+          LOG(WARNING) << "Rejecting transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = Type::kUninitialized;
+          return Result::kReject;
+        } else {
+          LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = event.type;
+          return Result::kAccept;
+        }
+      }
+      case Type::kIntentStarted: {
+        if (event.type == Type::kIntentFailed ||
+            event.type == Type::kActivityLaunched) {
+          LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = event.type;
+          return Result::kAccept;
+        } else {
+          LOG(WARNING) << "Overwriting transition from kIntentStarted to "
+                       << event.type << " into kIntentFailed";
+          last_event_type_ = Type::kIntentFailed;
+
+          *overwrite = event;
+          overwrite->type = Type::kIntentFailed;
+          return Result::kOverwrite;
+        }
+      }
+      case Type::kActivityLaunched: {
+        if (event.type == Type::kActivityLaunchFinished ||
+            event.type == Type::kActivityLaunchCancelled) {
+          LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = event.type;
+          return Result::kAccept;
+        } else {
+          LOG(WARNING) << "Overwriting transition from kActivityLaunched to "
+                       << event.type << " into kActivityLaunchCancelled";
+          last_event_type_ = Type::kActivityLaunchCancelled;
+
+          *overwrite = event;
+          overwrite->type = Type::kActivityLaunchCancelled;
+          return Result::kOverwrite;
+        }
+      }
+      case Type::kActivityLaunchFinished: {
+        if (event.type == Type::kIntentStarted ||
+            event.type == Type::kReportFullyDrawn) {
+          LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = event.type;
+          return Result::kAccept;
+        } else {
+          LOG(WARNING) << "Rejecting transition from " << last_event_type_ << " to " << event.type;
+          last_event_type_ = Type::kUninitialized;
+          return Result::kReject;
+        }
+      }
+    }
+  }
 };
 
 // Convert callback pattern into reactive pattern.
@@ -387,6 +872,70 @@ struct AppLaunchEventSubject {
   std::optional<rxcpp::subscriber<RefWrapper>> subscriber_;
 };
 
+// Convert callback pattern into reactive pattern.
+struct JobScheduledEventSubject {
+  JobScheduledEventSubject() {}
+
+  void Subscribe(rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>> subscriber) {
+    DCHECK(ready_ != true) << "Cannot Subscribe twice";
+
+    subscriber_ = std::move(subscriber);
+
+    // Release edge of synchronizes-with AcquireIsReady.
+    ready_.store(true);
+  }
+
+  void OnNext(RequestId request_id, JobScheduledEvent e) {
+    if (!AcquireIsReady()) {
+      return;
+    }
+
+    if (!subscriber_->is_subscribed()) {
+      return;
+    }
+
+    subscriber_->on_next(std::pair<RequestId, JobScheduledEvent>{std::move(request_id), std::move(e)});
+
+  }
+
+  void OnCompleted() {
+    if (!AcquireIsReady()) {
+      return;
+    }
+
+    subscriber_->on_completed();
+  }
+
+ private:
+  bool AcquireIsReady() {
+    // Synchronizes-with the release-edge in Subscribe.
+    // This can happen much later, only once the subscription actually happens.
+
+    // However, as far as I know, 'rxcpp::subscriber' is not thread safe,
+    // (but the observable chain itself can be made thread-safe via #observe_on, etc).
+    // so we must avoid reading it until it has been fully synchronized.
+    //
+    // TODO: investigate rxcpp subscribers and see if we can get rid of this atomics,
+    // to make it simpler.
+    return ready_.load();
+  }
+
+  // TODO: also track the RequestId ?
+
+  std::atomic<bool> ready_{false};
+
+  std::optional<rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>>> subscriber_;
+};
+
+std::ostream& operator<<(std::ostream& os, const android::content::pm::PackageChangeEvent& event) {
+  os << "PackageChangeEvent{";
+  os << "packageName=" << event.packageName << ",";
+  os << "version=" << event.version << ",";
+  os << "lastUpdateTimeMillis=" << event.lastUpdateTimeMillis;
+  os << "}";
+  return os;
+}
+
 class EventManager::Impl {
  public:
   Impl(/*borrow*/perfetto::RxProducerFactory& perfetto_factory)
@@ -394,15 +943,36 @@ class EventManager::Impl {
       worker_thread_(rxcpp::observe_on_new_thread()),
       worker_thread2_(rxcpp::observe_on_new_thread()),
       io_thread_(perfetto::ObserveOnNewIoThread()) {
+    // Try to create version map
+    RetryCreateVersionMap();
 
-    // TODO: read all properties from one config class.
-    tracing_allowed_ = ::android::base::GetBoolProperty("iorapd.perfetto.enable", /*default*/false);
+    iorap::common::StderrLogPrinter printer{"iorapd"};
+    RefreshSystemProperties(printer);
 
-    if (tracing_allowed_) {
-      rx_lifetime_ = InitializeRxGraph();
-    } else {
-      LOG(WARNING) << "Tracing disabled by iorapd.perfetto.enable=false";
-    }
+    rx_lifetime_ = InitializeRxGraph();
+    rx_lifetime_jobs_ = InitializeRxGraphForJobScheduledEvents();
+
+    android::add_sysprop_change_callback(&Impl::OnSyspropChanged, /*priority*/-10000);
+  }
+
+  void RetryCreateVersionMap() {
+    android::base::Timer timer{};
+    version_map_ = binder::PackageVersionMap::Create();
+    std::chrono::milliseconds duration_ms = timer.duration();
+    LOG(DEBUG) << "Got versions for "
+               << version_map_->Size()
+               << " packages in "
+               << duration_ms.count()
+               << "ms";
+  }
+
+  void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) {
+    DCHECK(callbacks_.expired());
+    callbacks_ = callbacks;
+  }
+
+  void Join() {
+    async_pool_.Join();
   }
 
   bool OnAppLaunchEvent(RequestId request_id,
@@ -411,11 +981,75 @@ class EventManager::Impl {
                  << "request_id=" << request_id.request_id << ","
                  << event;
 
-    app_launch_event_subject_.OnNext(event);
+    // Filter any incoming events through a defender that enforces
+    // that all state transitions are as contractually documented in
+    // ActivityMetricsLaunchObserver's javadoc.
+    AppLaunchEvent overwrite_event{};
+    AppLaunchEventDefender::Result result =
+        app_launch_event_defender_.OnAppLaunchEvent(request_id, event, /*out*/&overwrite_event);
+
+    switch (result) {
+      case AppLaunchEventDefender::Result::kAccept:
+        app_launch_event_subject_.OnNext(event);
+        return true;
+      case AppLaunchEventDefender::Result::kOverwrite:
+        app_launch_event_subject_.OnNext(overwrite_event);
+        return false;
+      case AppLaunchEventDefender::Result::kReject:
+        // Intentionally left-empty: we drop the event completely.
+        return false;
+    }
+
+    // In theory returns BAD_VALUE to the other side of this binder connection.
+    // In practice we use 'oneway' flags so this doesn't matter on a regular build.
+    return false;
+  }
+
+  bool OnDexOptEvent(RequestId request_id,
+                     const DexOptEvent& event) {
+    LOG(VERBOSE) << "EventManager::OnDexOptEvent("
+                 << "request_id=" << request_id.request_id << ","
+                 << event.package_name
+                 << ")";
+
+    return PurgePackage(event.package_name);
+  }
+
+  bool OnJobScheduledEvent(RequestId request_id,
+                           const JobScheduledEvent& event) {
+    LOG(VERBOSE) << "EventManager::OnJobScheduledEvent("
+                 << "request_id=" << request_id.request_id << ",event=TODO).";
+
+    job_scheduled_event_subject_.OnNext(std::move(request_id), event);
 
+    return true;  // No errors.
+  }
+
+  bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event) {
+    LOG(DEBUG) << "Received " << event;
+    if (event.isDeleted) {
+      // Do nothing if the package is deleted rignt now.
+      // The package will be removed from db during maintenance.
+      return true;
+    }
+    // Update the version map.
+    if (version_map_->Update(event.packageName, event.version)) {
+      return true;
+    }
+
+    // Sometimes a package is updated without any version change.
+    // Clean it up in this case.
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+    db::CleanUpFilesForPackage(db, event.packageName, event.version);
     return true;
   }
 
+  void Dump(/*borrow*/::android::Printer& printer) {
+    ::iorap::prefetcher::ReadAhead::Dump(printer);
+    ::iorap::perfetto::PerfettoConsumerImpl::Dump(/*borrow*/printer);
+    ::iorap::maintenance::Dump(db::SchemaModel::GetSingleton(), printer);
+  }
+
   rxcpp::composite_subscription InitializeRxGraph() {
     LOG(VERBOSE) << "EventManager::InitializeRxGraph";
 
@@ -426,7 +1060,21 @@ class EventManager::Impl {
 
     rxcpp::composite_subscription lifetime;
 
-    AppLaunchEventState initial_state{&perfetto_factory_, &worker_thread2_, &io_thread_};
+    if (!tracing_allowed_) {
+      LOG(WARNING) << "Tracing disabled by system property";
+    }
+    if (!readahead_allowed_) {
+      LOG(WARNING) << "Readahead disabled by system property";
+    }
+
+    AppLaunchEventState initial_state{&perfetto_factory_,
+                                      readahead_allowed_,
+                                      tracing_allowed_,
+                                      package_blacklister_,
+                                      &worker_thread2_,
+                                      &io_thread_,
+                                      &async_pool_,
+                                      version_map_};
     app_launch_events_
       .subscribe_on(worker_thread_)
       .scan(std::move(initial_state),
@@ -442,12 +1090,227 @@ class EventManager::Impl {
     return lifetime;
   }
 
+  // Runs the maintenance code to compile perfetto traces to compiled
+  // trace.
+  void StartMaintenance(bool output_text,
+                        std::optional<std::string> inode_textcache,
+                        bool verbose,
+                        bool recompile,
+                        uint64_t min_traces) {
+    ScopedFormatTrace atrace_bg_scope(ATRACE_TAG_PACKAGE_MANAGER,
+                                      "Background Job Scope");
+
+    {
+      ScopedFormatTrace atrace_update_versions(ATRACE_TAG_PACKAGE_MANAGER,
+                                               "Update package versions map cache");
+      // Update the version map.
+      version_map_->UpdateAll();
+    }
+
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+    {
+      ScopedFormatTrace atrace_cleanup_db(ATRACE_TAG_PACKAGE_MANAGER,
+                                          "Clean up obsolete data in database");
+      // Cleanup the obsolete data in the database.
+      maintenance::CleanUpDatabase(db, version_map_);
+    }
+
+    {
+      ScopedFormatTrace atrace_compile_apps(ATRACE_TAG_PACKAGE_MANAGER,
+                                            "Compile apps on device");
+      // Compilation
+      maintenance::ControllerParameters params{
+        output_text,
+        inode_textcache,
+        verbose,
+        recompile,
+        min_traces,
+        std::make_shared<maintenance::Exec>()};
+
+      LOG(DEBUG) << "StartMaintenance: min_traces=" << min_traces;
+      maintenance::CompileAppsOnDevice(db, params);
+    }
+  }
+
+  rxcpp::composite_subscription InitializeRxGraphForJobScheduledEvents() {
+    LOG(VERBOSE) << "EventManager::InitializeRxGraphForJobScheduledEvents";
+
+    using RequestAndJobEvent = std::pair<RequestId, JobScheduledEvent>;
+
+    job_scheduled_events_ = rxcpp::observable<>::create<RequestAndJobEvent>(
+      [&](rxcpp::subscriber<RequestAndJobEvent> subscriber) {
+        job_scheduled_event_subject_.Subscribe(std::move(subscriber));
+      });
+
+    rxcpp::composite_subscription lifetime;
+
+    job_scheduled_events_
+      .observe_on(worker_thread_)  // async handling.
+      .tap([this](const RequestAndJobEvent& e) {
+        LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(1) - job begins";
+        this->NotifyProgress(e.first, TaskResult{TaskResult::State::kBegan});
+
+        StartMaintenance(/*output_text=*/false,
+                         /*inode_textcache=*/std::nullopt,
+                         /*verbose=*/false,
+                         /*recompile=*/false,
+                         s_min_traces);
+
+        // TODO: probably this shouldn't be emitted until most of the usual DCHECKs
+        // (for example, validate a job isn't already started, the request is not reused, etc).
+        // In this way we could block from the client until it sees 'kBegan' and Log.wtf otherwise.
+      })
+      .tap([](const RequestAndJobEvent& e) {
+        // TODO. Actual work.
+        LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(2) - job is being processed";
+
+        // TODO: abort functionality for in-flight jobs.
+        //
+        // maybe something like scan that returns an observable<Job> + flat map to that job.
+        // then we could unsubscribe from the scan to do a partial abort? need to try it and see if it works.
+        //
+        // other option is to create a new outer subscription for each job id which seems less ideal.
+      })
+      .subscribe(/*out*/lifetime,
+        /*on_next*/
+        [this](const RequestAndJobEvent& e) {
+          LOG(VERBOSE) << "EventManager#JobScheduledEvent#subscribe - job completed";
+          this->NotifyComplete(e.first, TaskResult{TaskResult::State::kCompleted});
+        }
+#if 0
+        ,
+        /*on_error*/
+        [](rxcpp::util::error_ptr err) {
+          LOG(ERROR) << "Scheduled job event failed: " << rxcpp::util::what(err);
+
+          //std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock();
+          //if (callbacks != nullptr) {
+            // FIXME: How do we get the request ID back out of the error? Seems like a problem.
+            // callbacks->OnComplete(, TaskResult{TaskResult::kError});
+            // We may have to wrap with an iorap::expected instead of using on_error.
+          //}
+
+          // FIXME: need to add a 'OnErrorResumeNext' operator?
+          DCHECK(false) << "forgot to implement OnErrorResumeNext";
+        }
+#endif
+      );
+
+    // TODO: error output should happen via an observable.
+
+    return lifetime;
+  }
+
+  void NotifyComplete(RequestId request_id, TaskResult result) {
+      std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock();
+      if (callbacks != nullptr) {
+        callbacks->OnComplete(std::move(request_id), std::move(result));
+      } else {
+        LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early";
+      }
+  }
+
+  void NotifyProgress(RequestId request_id, TaskResult result) {
+      std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock();
+      if (callbacks != nullptr) {
+        callbacks->OnProgress(std::move(request_id), std::move(result));
+      } else {
+        LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early";
+      }
+  }
+
+  static void OnSyspropChanged() {
+    LOG(DEBUG) << "OnSyspropChanged";
+  }
+
+  void RefreshSystemProperties(::android::Printer& printer) {
+    // TODO: read all properties from one config class.
+    // PH properties do not work if they contain ".". "_" was instead used here.
+    const char* ph_namespace = "runtime_native_boot";
+    tracing_allowed_ = server_configurable_flags::GetServerConfigurableFlag(
+        ph_namespace,
+        "iorap_perfetto_enable",
+        ::android::base::GetProperty("iorapd.perfetto.enable", /*default*/"true")) == "true";
+    s_tracing_allowed = tracing_allowed_;
+    printer.printFormatLine("iorapd.perfetto.enable = %s", tracing_allowed_ ? "true" : "false");
+
+    readahead_allowed_ = server_configurable_flags::GetServerConfigurableFlag(
+        ph_namespace,
+        "iorap_readahead_enable",
+        ::android::base::GetProperty("iorapd.readahead.enable", /*default*/"true")) == "true";
+    s_readahead_allowed = readahead_allowed_;
+    printer.printFormatLine("iorapd.readahead.enable = %s", s_readahead_allowed ? "true" : "false");
+
+    s_min_traces =
+        ::android::base::GetUintProperty<uint64_t>("iorapd.maintenance.min_traces", /*default*/1);
+    uint64_t min_traces = s_min_traces;
+    printer.printFormatLine("iorapd.maintenance.min_traces = %" PRIu64, min_traces);
+
+    package_blacklister_ = PackageBlacklister{
+        /* Colon-separated string list of blacklisted packages, e.g.
+         * "foo.bar.baz;com.fake.name" would blacklist {"foo.bar.baz", "com.fake.name"} packages.
+         *
+         * Blacklisted packages are ignored by iorapd.
+         */
+        server_configurable_flags::GetServerConfigurableFlag(
+            ph_namespace,
+            "iorap_blacklisted_packages",
+            ::android::base::GetProperty("iorapd.blacklist_packages",
+                                         /*default*/""))
+    };
+
+    LOG(DEBUG) << "RefreshSystemProperties";
+  }
+
+  bool PurgePackage(::android::Printer& printer, const std::string& package_name) {
+    (void)printer;
+    return PurgePackage(package_name);
+  }
+
+  bool PurgePackage(const std::string& package_name) {
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+    db::CleanUpFilesForPackage(db, package_name);
+    LOG(DEBUG) << "PurgePackage: " << package_name;
+    return true;
+  }
+
+  bool CompilePackage(::android::Printer& printer, const std::string& package_name) {
+    (void)printer;
+
+    ScopedFormatTrace atrace_compile_app(ATRACE_TAG_PACKAGE_MANAGER,
+                                         "Compile one app on device");
+
+    maintenance::ControllerParameters params{
+      /*output_text*/false,
+      /*inode_textcache*/std::nullopt,
+      WOULD_LOG(VERBOSE),
+      /*recompile*/false,
+      s_min_traces,
+      std::make_shared<maintenance::Exec>()};
+
+    db::DbHandle db{db::SchemaModel::GetSingleton()};
+    bool res = maintenance::CompileSingleAppOnDevice(db, std::move(params), package_name);
+    LOG(DEBUG) << "CompilePackage: " << package_name;
+
+    return res;
+  }
+
+  bool readahead_allowed_{true};
+
   perfetto::RxProducerFactory& perfetto_factory_;
   bool tracing_allowed_{true};
 
+  PackageBlacklister package_blacklister_{};
+
+  std::weak_ptr<TaskResultCallbacks> callbacks_;  // avoid cycles with weakptr.
+
   using AppLaunchEventRefWrapper = AppLaunchEventSubject::RefWrapper;
   rxcpp::observable<AppLaunchEventRefWrapper> app_launch_events_;
   AppLaunchEventSubject app_launch_event_subject_;
+  AppLaunchEventDefender app_launch_event_defender_;
+
+  rxcpp::observable<std::pair<RequestId, JobScheduledEvent>> job_scheduled_events_;
+  JobScheduledEventSubject job_scheduled_event_subject_;
 
   rxcpp::observable<RequestId> completed_requests_;
 
@@ -456,8 +1319,14 @@ class EventManager::Impl {
   observe_on_one_worker worker_thread2_;
   // low priority idle-class thread for IO operations.
   observe_on_one_worker io_thread_;
+  // async futures pool for async rx operations.
+  AsyncPool async_pool_;
+
+  rxcpp::composite_subscription rx_lifetime_;  // app launch events
+  rxcpp::composite_subscription rx_lifetime_jobs_;  // job scheduled events
 
-  rxcpp::composite_subscription rx_lifetime_;
+  // package version map
+  std::shared_ptr<binder::PackageVersionMap> version_map_;
 
 //INTENTIONAL_COMPILER_ERROR_HERE:
   // FIXME:
@@ -488,9 +1357,47 @@ std::shared_ptr<EventManager> EventManager::Create(perfetto::RxProducerFactory&
   return p;
 }
 
+void EventManager::SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) {
+  return impl_->SetTaskResultCallbacks(std::move(callbacks));
+}
+
+void EventManager::Join() {
+  return impl_->Join();
+}
+
 bool EventManager::OnAppLaunchEvent(RequestId request_id,
                                     const AppLaunchEvent& event) {
   return impl_->OnAppLaunchEvent(request_id, event);
 }
 
+bool EventManager::OnDexOptEvent(RequestId request_id,
+                                 const DexOptEvent& event) {
+  return impl_->OnDexOptEvent(request_id, event);
+}
+
+bool EventManager::OnJobScheduledEvent(RequestId request_id,
+                                       const JobScheduledEvent& event) {
+  return impl_->OnJobScheduledEvent(request_id, event);
+}
+
+bool EventManager::OnPackageChanged(const android::content::pm::PackageChangeEvent& event) {
+  return impl_->OnPackageChanged(event);
+}
+
+void EventManager::Dump(/*borrow*/::android::Printer& printer) {
+  return impl_->Dump(printer);
+}
+
+void EventManager::RefreshSystemProperties(::android::Printer& printer) {
+  return impl_->RefreshSystemProperties(printer);
+}
+
+bool EventManager::PurgePackage(::android::Printer& printer, const std::string& package_name) {
+  return impl_->PurgePackage(printer, package_name);
+}
+
+bool EventManager::CompilePackage(::android::Printer& printer, const std::string& package_name) {
+  return impl_->CompilePackage(printer, package_name);
+}
+
 }  // namespace iorap::manager
diff --git a/src/manager/event_manager.h b/src/manager/event_manager.h
index 5355296..fa09eec 100644
--- a/src/manager/event_manager.h
+++ b/src/manager/event_manager.h
@@ -18,21 +18,50 @@
 #define IORAP_MANAGER_EVENT_MANAGER_H_
 
 #include "binder/app_launch_event.h"
+#include "binder/dexopt_event.h"
+#include "binder/job_scheduled_event.h"
 #include "binder/request_id.h"
+#include "binder/task_result.h"
+
+#include <android/content/pm/PackageChangeEvent.h>
 
 #include <memory>
 
+namespace android {
+class Printer;
+}  // namespace android
+
 namespace iorap::perfetto {
-  struct RxProducerFactory;
+struct RxProducerFactory;
 }  // namespace iorap::perfetto
 
 namespace iorap::manager {
 
+// These callbacks are invoked by the EventManager to provide asynchronous notification for the status
+// of an event handler.
+//
+// Calling 'On_Event' in EventManager should be considered merely to start the task.
+// Calling 'OnComplete' here is considered to terminate the request (either with a success or error).
+// OnProgress is optional, but if it is called it must be called prior to 'OnComplete'.
+//
+// All callbacks for the same request-id are sequentially consistent.
+class TaskResultCallbacks {
+ public:
+  virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {}
+  virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {}
+
+  virtual ~TaskResultCallbacks() {}
+};
+
 class EventManager {
  public:
   static std::shared_ptr<EventManager> Create();
   static std::shared_ptr<EventManager> Create(
       /*borrow*/perfetto::RxProducerFactory& perfetto_factory);
+  void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks);
+
+  // Joins any background threads created by EventManager.
+  void Join();
 
   // Handles an AppLaunchEvent:
   //
@@ -45,6 +74,41 @@ class EventManager {
   bool OnAppLaunchEvent(binder::RequestId request_id,
                         const binder::AppLaunchEvent& event);
 
+  // Handles a DexOptEvent:
+  //
+  // Clean up the invalidate traces after package is updated by dexopt.
+  bool OnDexOptEvent(binder::RequestId request_id,
+                     const binder::DexOptEvent& event);
+
+
+  // Handles a JobScheduledEvent:
+  //
+  // * Start/stop background jobs (typically for idle maintenance).
+  // * For example, this could kick off a background compiler.
+  bool OnJobScheduledEvent(binder::RequestId request_id,
+                           const binder::JobScheduledEvent& event);
+
+  // Handles a PackageChangeEvent:
+  //
+  // * The package manager service send this event for package install
+  //   update or delete.
+  bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event);
+
+  // Print to adb shell dumpsys (for bugreport info).
+  void Dump(/*borrow*/::android::Printer& printer);
+
+  // A dumpsys --refresh-properties command signaling that we should
+  // refresh our system properties.
+  void RefreshSystemProperties(::android::Printer& printer);
+
+  // A dumpsys --purge-package <name> command signaling
+  // that all db rows and files associated with a package should be deleted.
+  bool PurgePackage(::android::Printer& printer, const std::string& package_name);
+
+  // A dumpsys --compile-package <name> command signaling
+  // that a package should be recompiled.
+  bool CompilePackage(::android::Printer& printer, const std::string& package_name);
+
   class Impl;
  private:
   std::unique_ptr<Impl> impl_;
diff --git a/src/perfetto/perfetto_consumer.cc b/src/perfetto/perfetto_consumer.cc
new file mode 100644
index 0000000..4fe677b
--- /dev/null
+++ b/src/perfetto/perfetto_consumer.cc
@@ -0,0 +1,608 @@
+// Copyright (C) 2020 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "perfetto/perfetto_consumer.h"
+
+#include "common/trace.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <utils/Looper.h>
+#include <utils/Printer.h>
+
+#include <limits>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <sstream>
+#include <vector>
+
+#include <inttypes.h>
+#include <time.h>
+
+namespace iorap::perfetto {
+
+using State = PerfettoConsumer::State;
+using Handle = PerfettoConsumer::Handle;
+static constexpr Handle kInvalidHandle = PerfettoConsumer::kInvalidHandle;
+using OnStateChangedCb = PerfettoConsumer::OnStateChangedCb;
+using TraceBuffer = PerfettoConsumer::TraceBuffer;
+
+enum class StateKind {
+  kUncreated,
+  kCreated,
+  kStartedTracing,
+  kReadTracing,
+  kTimedOutDestroyed,  // same as kDestroyed but timed out.
+  kDestroyed,          // calling kDestroyed before timing out.
+};
+
+std::ostream& operator<<(std::ostream& os, StateKind kind) {
+  switch (kind) {
+    case StateKind::kUncreated:
+      os << "kUncreated";
+      break;
+    case StateKind::kCreated:
+      os << "kCreated";
+      break;
+    case StateKind::kStartedTracing:
+      os << "kStartedTracing";
+      break;
+    case StateKind::kReadTracing:
+      os << "kReadTracing";
+      break;
+    case StateKind::kTimedOutDestroyed:
+      os << "kTimedOutDestroyed";
+      break;
+    case StateKind::kDestroyed:
+      os << "kDestroyed";
+      break;
+    default:
+      os << "(invalid)";
+      break;
+  }
+  return os;
+}
+
+std::string ToString(StateKind kind) {
+  std::stringstream ss;
+  ss << kind;
+  return ss.str();
+}
+
+static constexpr uint64_t kSecToNano = 1000000000LL;
+
+static uint64_t GetTimeNanoseconds() {
+  struct timespec now;
+  clock_gettime(CLOCK_REALTIME, &now);
+
+  uint64_t now_ns = (now.tv_sec * kSecToNano + now.tv_nsec);
+  return now_ns;
+}
+
+// Describe the state of our handle in detail for debugging/logging.
+struct HandleDescription {
+  Handle handle_;
+  StateKind kind_{StateKind::kUncreated};  // Our state. required for correctness.
+  OnStateChangedCb callback_{nullptr};  // Required for Destroy callbacks.
+  void* callback_arg_{nullptr};
+
+  // For dumping to logs:
+  State state_{State::kSessionNotFound};  // perfetto state
+  std::optional<uint64_t> started_tracing_ns_; // when StartedTracing last called.
+  std::optional<uint64_t> read_trace_ns_;  // When ReadTrace last called.
+  std::uint64_t last_transition_ns_{0};
+  std::optional<uint64_t> trace_cookie_;  // atrace beginning at StartTracing.
+  bool trace_ended_{false};               // atrace ending at ReadTrace or Destroy.
+
+  HandleDescription(Handle handle): handle_(handle) {}
+};
+
+// pimpl idiom to hide the implementation details from header
+//
+// Track and verify that our perfetto usage is sane.
+struct PerfettoConsumerImpl::Impl {
+  Impl() : raw_{new PerfettoConsumerRawImpl{}},
+      message_handler_{new TraceMessageHandler{this}} {
+    std::thread watchdog_thread{ [this]() {
+      ::android::sp<::android::Looper> looper;
+      {
+        std::lock_guard<std::mutex> guard{looper_mutex_};
+        looper = ::android::Looper::prepare(/*opts*/0);
+        looper_ = looper;
+      }
+
+      static constexpr int kTimeoutMillis = std::numeric_limits<int>::max();
+
+      while (true) {
+        // Execute any pending callbacks, otherwise just block forever.
+        int result = looper->pollAll(kTimeoutMillis);
+
+        if (result == ::android::Looper::POLL_ERROR) {
+          LOG(ERROR) << "PerfettoConsumerImpl::Looper got a POLL_ERROR";
+        } else {
+          LOG(DEBUG) << "PerfettoConsumerImpl::Looper result was " << result;
+        }
+      }
+    }};
+
+    // Let thread run freely on its own.
+    watchdog_thread.detach();
+
+    // Block until looper_ is prepared.
+    while (true) {
+      std::lock_guard<std::mutex> guard{looper_mutex_};
+      if (looper_ != nullptr) {
+        break;
+      }
+    }
+  }
+
+ private:
+  std::unique_ptr<PerfettoConsumerRawImpl> raw_;
+  std::map<Handle, HandleDescription> states_;
+
+  // We need this to be a counter to avoid memory leaks.
+  Handle last_created_{0};
+  Handle last_destroyed_{0};
+  uint64_t trace_cookie_{0};
+
+  std::mutex mutex_;  // Guard above values.
+
+  ::android::sp<::android::Looper> looper_;
+  std::mutex looper_mutex_;  // Guard looper_.
+
+  struct TraceMessageHandler : public ::android::MessageHandler {
+    TraceMessageHandler(Impl* impl) : impl_{impl} {
+      CHECK(impl != nullptr);
+    }
+
+    Impl* impl_;
+
+    virtual void handleMessage(const ::android::Message& message) override {
+      impl_->OnTraceMessage(static_cast<Handle>(message.what));
+    }
+  };
+
+  ::android::sp<TraceMessageHandler> message_handler_;
+
+ public:
+  Handle Create(const void* config_proto,
+                size_t config_len,
+                OnStateChangedCb callback,
+                void* callback_arg) {
+    LOG(VERBOSE) << "PerfettoConsumer::Create("
+                 << "config_len=" << config_len << ")";
+    Handle handle = raw_->Create(config_proto, config_len, callback, callback_arg);
+
+    std::lock_guard<std::mutex> guard{mutex_};
+
+    // Assume every Handle starts at 0 and then increments by 1 every Create.
+    ++last_created_;
+    CHECK_EQ(last_created_, handle) << "perfetto handle had unexpected behavior.";
+    // Without this^ increment-by-1 behavior our detection of untracked state values is broken.
+    // If we have to, we can go with Untracked=Uncreated|Destroyed but it's better to distinguish
+    // the two if possible.
+
+    HandleDescription handle_desc{handle};
+    handle_desc.handle_ = handle;
+    handle_desc.callback_ = callback;
+    handle_desc.callback_arg_ = callback_arg;
+    UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kCreated);
+
+    // assume we never wrap around due to using int64
+    bool inserted = states_.insert({handle, handle_desc}).second;
+    CHECK(inserted) << "perfetto handle was re-used: " << handle;
+
+    return handle;
+  }
+
+  void StartTracing(Handle handle) {
+    LOG(DEBUG) << "PerfettoConsumer::StartTracing(handle=" << handle << ")";
+
+    uint64_t trace_cookie;
+    {
+      std::lock_guard<std::mutex> guard{mutex_};
+
+      auto it = states_.find(handle);
+      if (it == states_.end()) {
+        LOG(ERROR) << "Cannot StartTracing(" << handle << "), untracked handle";
+        return;
+      }
+      HandleDescription& handle_desc = it->second;
+
+      raw_->StartTracing(handle);
+      UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kStartedTracing);
+    }
+
+    // Use a looper here to add a timeout and immediately destroy the trace buffer.
+    CHECK_LE(static_cast<int64_t>(handle), static_cast<int64_t>(std::numeric_limits<int>::max()));
+    int message_code = static_cast<int>(handle);
+    ::android::Message message{message_code};
+
+    std::lock_guard<std::mutex> looper_guard{looper_mutex_};
+    looper_->sendMessageDelayed(static_cast<nsecs_t>(GetPropertyTraceTimeoutNs()),
+                                message_handler_,
+                                message);
+  }
+
+  TraceBuffer ReadTrace(Handle handle) {
+    LOG(DEBUG) << "PerfettoConsumer::ReadTrace(handle=" << handle << ")";
+
+    std::lock_guard<std::mutex> guard{mutex_};
+
+    auto it = states_.find(handle);
+    if (it == states_.end()) {
+      LOG(ERROR) << "Cannot ReadTrace(" << handle << "), untracked handle";
+      return TraceBuffer{};
+    }
+
+    HandleDescription& handle_desc = it->second;
+
+    TraceBuffer trace_buffer = raw_->ReadTrace(handle);
+    UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kReadTracing);
+
+    return trace_buffer;
+  }
+
+  void Destroy(Handle handle) {
+    HandleDescription handle_desc{handle};
+    TryDestroy(handle, /*do_destroy*/true, /*out*/&handle_desc);;
+  }
+
+  bool TryDestroy(Handle handle, bool do_destroy, /*out*/HandleDescription* handle_desc_out) {
+    CHECK(handle_desc_out != nullptr);
+
+    LOG(VERBOSE) << "PerfettoConsumer::Destroy(handle=" << handle << ")";
+
+    std::lock_guard<std::mutex> guard{mutex_};
+
+    auto it = states_.find(handle);
+    if (it == states_.end()) {
+      // Leniency for calling Destroy multiple times. It's not a mistake.
+      LOG(ERROR) << "Cannot Destroy(" << handle << "), untracked handle";
+      return false;
+    }
+
+    HandleDescription& handle_desc = it->second;
+
+    if (do_destroy) {
+      raw_->Destroy(handle);
+    }
+    UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kDestroyed);
+
+    *handle_desc_out = handle_desc;
+
+    // No longer track this handle to avoid memory leaks.
+    last_destroyed_ = handle;
+    states_.erase(it);
+
+    return true;
+  }
+
+  State PollState(Handle handle) {
+    // Just pass-through the call, we never use it directly anyway.
+    return raw_->PollState(handle);
+  }
+
+  // Either fetch or infer the current handle state from a handle.
+  // Meant for debugging/logging only.
+  HandleDescription GetOrInferHandleDescription(Handle handle) {
+    std::lock_guard<std::mutex> guard{mutex_};
+
+    auto it = states_.find(handle);
+    if (it == states_.end()) {
+      HandleDescription state{handle};
+      // If it's untracked it hasn't been created yet, or it was already destroyed.
+      if (IsDestroyed(handle)) {
+        UpdateHandleDescription(/*inout*/&state, StateKind::kDestroyed);
+      } else {
+        if (!IsUncreated(handle)) {
+          LOG(WARNING) << "bad state detection";
+        }
+        UpdateHandleDescription(/*inout*/&state, StateKind::kUncreated);
+      }
+      return state;
+    }
+    return it->second;
+  }
+
+  void OnTraceMessage(Handle handle) {
+    LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle) << ")";
+    HandleDescription handle_desc{handle};
+    {
+      std::lock_guard<std::mutex> guard{mutex_};
+
+      auto it = states_.find(handle);
+      if (it == states_.end()) {
+        // Handle values are never re-used, so we can simply ignore the message here
+        // instead of having to remove it from the message queue.
+        LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle)
+                     << ") no longer tracked handle";
+        return;
+      }
+      handle_desc = it->second;
+    }
+
+    // First check. Has this trace been active for too long?
+    uint64_t now_ns = GetTimeNanoseconds();
+    if (handle_desc.kind_ == StateKind::kStartedTracing) {
+      // Ignore other kinds of traces because they don't exhaust perfetto resources.
+      CHECK(handle_desc.started_tracing_ns_.has_value()) << static_cast<int64_t>(handle);
+
+      uint64_t started_tracing_ns = *handle_desc.started_tracing_ns_;
+
+      if ((now_ns - started_tracing_ns) > GetPropertyTraceTimeoutNs()) {
+        LOG(WARNING) << "Perfetto Handle timed out after " << (now_ns - started_tracing_ns) << "ns"
+                     << ", forcibly destroying";
+
+        // Let the callback handler call Destroy.
+        handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_);
+      }
+    }
+
+    // Second check. Are there too many traces now? Cull the old traces.
+    std::vector<HandleDescription> handle_list;
+    do {
+      std::lock_guard<std::mutex> guard{mutex_};
+
+      size_t max_trace_count = GetPropertyMaxTraceCount();
+      if (states_.size() > max_trace_count) {
+        size_t overflow_count = states_.size() - max_trace_count;
+        LOG(WARNING) << "Too many perfetto handles, overflowed by " << overflow_count
+                     << ", pruning down to " << max_trace_count;
+      } else {
+        break;
+      }
+
+      size_t prune_count = states_.size() - max_trace_count;
+      auto it = states_.begin();
+      for (size_t i = 0; i < prune_count; ++i) {
+        // Simply prune by handle 1,2,3,4...
+        // We could do better with a timestamp if we wanted to.
+        ++it;
+        handle_list.push_back(it->second);
+      }
+    } while (false);
+
+    for (HandleDescription& handle_desc : handle_list) {
+      LOG(DEBUG) << "Perfetto handle pruned: " << static_cast<int64_t>(handle);
+
+      // Let the callback handler call Destroy.
+      handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_);
+    }
+  }
+
+ private:
+  static uint64_t GetPropertyTraceTimeoutNs() {
+    static uint64_t value =                       // property is timeout in seconds
+        ::android::base::GetUintProperty<uint64_t>("iorapd.perfetto.timeout", /*default*/10);
+    return value * kSecToNano;
+  }
+
+  static size_t GetPropertyMaxTraceCount() {
+    static size_t value =
+        ::android::base::GetUintProperty<size_t>("iorapd.perfetto.max_traces", /*default*/5);
+    return value;
+  }
+
+  void UpdateHandleDescription(/*inout*/HandleDescription* handle_desc, StateKind kind) {
+    CHECK(handle_desc != nullptr);
+    handle_desc->kind_ = kind;
+    handle_desc->state_ = raw_->PollState(handle_desc->handle_);
+
+    handle_desc->last_transition_ns_ = GetTimeNanoseconds();
+    if (kind == StateKind::kStartedTracing) {
+      if (!handle_desc->started_tracing_ns_) {
+        handle_desc->started_tracing_ns_ = handle_desc->last_transition_ns_;
+
+        handle_desc->trace_cookie_ = ++trace_cookie_;
+
+        atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER,
+                           "Perfetto Scoped Trace",
+                           *handle_desc->trace_cookie_);
+        atrace_int(ATRACE_TAG_ACTIVITY_MANAGER,
+                   "Perfetto::Trace Handle",
+                   static_cast<int32_t>(handle_desc->handle_));
+      }
+    }
+
+    if (kind == StateKind::kReadTracing) {
+      if (!handle_desc->read_trace_ns_) {
+        handle_desc->read_trace_ns_ = handle_desc->last_transition_ns_;
+
+        if (handle_desc->trace_cookie_.has_value() && !handle_desc->trace_ended_) {
+          atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER,
+                           "Perfetto Scoped Trace",
+                           handle_desc->trace_cookie_.value());
+
+          handle_desc->trace_ended_ = true;
+        }
+      }
+    }
+
+    // If Destroy is called prior to ReadTrace, mark the atrace as finished.
+    if (kind == StateKind::kDestroyed && handle_desc->trace_cookie_ && !handle_desc->trace_ended_) {
+      atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER,
+                       "Perfetto Scoped Trace",
+                       *handle_desc->trace_cookie_);
+      handle_desc->trace_ended_ = true;
+    }
+  }
+
+  // The following state detection is for debugging only.
+  // We figure out if something is destroyed, uncreated, or live.
+
+  // Does not distinguish between kTimedOutDestroyed and kDestroyed.
+  bool IsDestroyed(Handle handle) const {
+    auto it = states_.find(handle);
+    if (it != states_.end()) {
+      // Tracked values are not destroyed yet.
+      return false;
+    }
+
+    if (handle == kInvalidHandle) {
+      return false;
+    }
+
+    // The following assumes handles are incrementally generated:
+    if (it == states_.end()) {
+      // value is in range of [0, last_destroyed]  => destroyed.
+      return handle <= last_destroyed_;
+    }
+
+    auto min_it = states_.begin();
+    if (handle < min_it->first) {
+      // value smaller than anything tracked: it was destroyed and we stopped tracking it.
+      return true;
+    }
+
+    auto max_it = states_.rbegin();
+    if (handle > max_it->first) {
+      // value too big: it's uncreated;
+      return false;
+    }
+
+    // else it was a value that was previously tracked within [min,max] but no longer
+    return true;
+  }
+
+  bool IsUncreated(Handle handle) const {
+    auto it = states_.find(handle);
+    if (it != states_.end()) {
+      // Tracked values are not uncreated.
+      return false;
+    }
+
+    if (handle == kInvalidHandle) {
+      // Strangely enough, an invalid handle can never be created.
+      return true;
+    }
+
+    // The following assumes handles are incrementally generated:
+    if (it == states_.end()) {
+      // value is in range of (last_destroyed, inf)  => uncreated.
+      return handle > last_destroyed_;
+    }
+
+    auto min_it = states_.begin();
+    if (handle < min_it->first) {
+      // value smaller than anything tracked: it was destroyed and we stopped tracking it.
+      return false;
+    }
+
+    auto max_it = states_.rbegin();
+    if (handle > max_it->first) {
+      // value too big: it's uncreated;
+      return true;
+    }
+
+    // else it was a value that was previously tracked within [min,max] but no longer
+    return false;
+  }
+
+ public:
+  void Dump(::android::Printer& printer) {
+    // Locking can fail if we dump during a deadlock, so just do a best-effort lock here.
+    bool is_it_locked = mutex_.try_lock();
+
+    printer.printFormatLine("Perfetto consumer state:");
+    if (!is_it_locked) {
+      printer.printLine("""""  (possible deadlock)");
+    }
+    printer.printFormatLine("  Last destroyed handle: %" PRId64, last_destroyed_);
+    printer.printFormatLine("  Last created handle: %" PRId64, last_created_);
+    printer.printFormatLine("");
+    printer.printFormatLine("  In-flight handles:");
+
+    for (auto it = states_.begin(); it != states_.end(); ++it) {
+      HandleDescription& handle_desc = it->second;
+      uint64_t started_tracing =
+          handle_desc.started_tracing_ns_ ? *handle_desc.started_tracing_ns_ : 0;
+      printer.printFormatLine("    Handle %" PRId64, handle_desc.handle_);
+      printer.printFormatLine("      Kind: %s", ToString(handle_desc.kind_).c_str());
+      printer.printFormatLine("      Perfetto State: %d", static_cast<int>(handle_desc.state_));
+      printer.printFormatLine("      Started tracing at: %" PRIu64, started_tracing);
+      printer.printFormatLine("      Last transition at: %" PRIu64,
+                              handle_desc.last_transition_ns_);
+    }
+    if (states_.empty()) {
+      printer.printFormatLine("    (None)");
+    }
+
+    printer.printFormatLine("");
+
+    if (is_it_locked) {  // u.b. if calling unlock on an unlocked mutex.
+      mutex_.unlock();
+    }
+  }
+
+  static PerfettoConsumerImpl::Impl* GetImplSingleton() {
+    static PerfettoConsumerImpl::Impl impl;
+    return &impl;
+  }
+};
+
+// Use a singleton because fruit instantiates a new PerfettoConsumer object for every
+// new rx chain in RxProducerFactory. However, we want to track all perfetto transitions globally
+// through 1 impl object.
+//
+// TODO: Avoiding a singleton would mean a more significant refactoring to remove the fruit/perfetto
+// usage.
+
+
+//
+// Forward all calls to PerfettoConsumerImpl::Impl
+//
+
+PerfettoConsumerImpl::~PerfettoConsumerImpl() {
+  // delete impl_;  // TODO: no singleton
+}
+
+void PerfettoConsumerImpl::Initialize() {
+  // impl_ = new PerfettoConsumerImpl::Impl();  // TODO: no singleton
+  impl_ = PerfettoConsumerImpl::Impl::GetImplSingleton();
+}
+
+void PerfettoConsumerImpl::Dump(::android::Printer& printer) {
+  PerfettoConsumerImpl::Impl::GetImplSingleton()->Dump(/*borrow*/printer);
+}
+
+PerfettoConsumer::Handle PerfettoConsumerImpl::Create(const void* config_proto,
+                                    size_t config_len,
+                                    PerfettoConsumer::OnStateChangedCb callback,
+                                    void* callback_arg) {
+  return impl_->Create(config_proto,
+                       config_len,
+                       callback,
+                       callback_arg);
+}
+
+void PerfettoConsumerImpl::StartTracing(PerfettoConsumer::Handle handle) {
+  impl_->StartTracing(handle);
+}
+
+PerfettoConsumer::TraceBuffer PerfettoConsumerImpl::ReadTrace(PerfettoConsumer::Handle handle) {
+  return impl_->ReadTrace(handle);
+}
+
+void PerfettoConsumerImpl::Destroy(PerfettoConsumer::Handle handle) {
+  impl_->Destroy(handle);
+}
+
+PerfettoConsumer::State PerfettoConsumerImpl::PollState(PerfettoConsumer::Handle handle) {
+  return impl_->PollState(handle);
+}
+
+}  // namespace iorap::perfetto
diff --git a/src/perfetto/perfetto_consumer.h b/src/perfetto/perfetto_consumer.h
index ce04ffd..2d828c0 100644
--- a/src/perfetto/perfetto_consumer.h
+++ b/src/perfetto/perfetto_consumer.h
@@ -18,6 +18,10 @@
 #include <fruit/fruit.h>
 #include <perfetto/public/consumer_api.h>  // libperfetto
 
+namespace android {
+class Printer;
+}  // namespace android
+
 namespace iorap::perfetto {
 
 // Abstract out the Perfetto C API behind a virtual interface:
@@ -48,9 +52,9 @@ struct PerfettoConsumer {
 };
 
 // "Live" implementation that calls down to libperfetto.
-struct PerfettoConsumerImpl : public PerfettoConsumer {
+struct PerfettoConsumerRawImpl : public PerfettoConsumer {
   // Marks this constructor as the one to use for injection.
-  INJECT(PerfettoConsumerImpl()) = default;
+  INJECT(PerfettoConsumerRawImpl()) = default;
 
   virtual Handle Create(const void* config_proto,
                         size_t config_len,
@@ -77,7 +81,32 @@ struct PerfettoConsumerImpl : public PerfettoConsumer {
     return ::perfetto::consumer::PollState(handle);
   }
 
-  virtual ~PerfettoConsumerImpl() {}
+  virtual ~PerfettoConsumerRawImpl() {}
+};
+
+// "Safe" implementation that has extra checking around it.
+class PerfettoConsumerImpl : public PerfettoConsumer {
+ public:
+  // Marks this constructor as the one to use for injection.
+  INJECT(PerfettoConsumerImpl()) { Initialize(); }
+
+  virtual Handle Create(const void* config_proto,
+                        size_t config_len,
+                        OnStateChangedCb callback,
+                        void* callback_arg) override;
+  virtual void StartTracing(Handle handle) override;
+  virtual TraceBuffer ReadTrace(Handle handle) override;
+  virtual void Destroy(Handle handle) override;
+  virtual State PollState(Handle handle) override;
+
+  virtual ~PerfettoConsumerImpl();
+
+  static void Dump(/*borrow*/::android::Printer& printer);
+
+ private:
+  void Initialize();
+  struct Impl;
+  PerfettoConsumerImpl::Impl* impl_;
 };
 
 }  // namespace iorap::perfetto
diff --git a/src/perfetto/rx_producer.cc b/src/perfetto/rx_producer.cc
index 14311a1..824600c 100644
--- a/src/perfetto/rx_producer.cc
+++ b/src/perfetto/rx_producer.cc
@@ -83,7 +83,7 @@ PerfettoDependencies::Component PerfettoDependencies::CreateComponent() {
     .bind<PerfettoConsumer, PerfettoConsumerImpl>()
     .registerProvider([]() /* -> TraceConfig */ {
         return CreateConfig(kTraceDurationMs,
-                            /*deferred_start*/true,
+                            /*deferred_start*/false,
                             kBufferSize);
     });
 }
@@ -199,6 +199,12 @@ class StateChangedSubject {
   using State = ::perfetto::consumer::State;
   using Handle = ::perfetto::consumer::Handle;
 
+  // Static members to solve use-after-free bug.
+  // The object is accessed from not only perfetto thread, but also iorap
+  // thread. Use this global map to manage it.
+  static std::mutex state_subject_mutex_;
+  static std::unordered_map<Handle, StateChangedSubject*> state_subject_map_;
+
   StateChangedSubject(const ::perfetto::protos::TraceConfig& trace_config,
                       rxcpp::subscriber<PerfettoStateChange> destination,
                       std::shared_ptr<PerfettoConsumer> perfetto_consumer)
@@ -391,7 +397,9 @@ class StateChangedSubject {
     bound_.store(true);  // seq_cst release.
   }
 
-  // Thread safety: Called by libperfetto background thread (same one every time).
+
+  // Called by libperfetto background thread (same one every time) and iorap
+  // thread.
   static void CallbackOnStateChanged(Handle handle, State state, void* callback_arg) {
     LOG(VERBOSE) << "CallbackOnStateChanged(handle=" << handle << ",state=" << state
                  << ",callback_arg=" << callback_arg << ")";
@@ -399,24 +407,30 @@ class StateChangedSubject {
     // Validate OnStateChanged callback invariants, guaranteed by libperfetto.
     DCHECK_NE(handle, ::perfetto::consumer::kInvalidHandle);
 
-    // Note: Perfetto guarantees this callback always occurs on the same thread,
-    // so we don't need to do any extra thread synchronization here since we are only mutating
-    // StateChangedSubject from within this function.
-
     // TODO: the memory ordering guarantees should be explicitly specified in consumer_api.h:
     // This isn't specific enough:
     // "The callback will be invoked on an internal thread and must not block."
     // However looking at the implementation it posts onto a single-thread task runner,
     // so this must be the case.
 
-    StateChangedSubject* state_subject = reinterpret_cast<StateChangedSubject*>(callback_arg);
     // This current thread owns 'StateChangedSubject', no other threads must access it.
     // Explicit synchronization is not necessary.
 
-    if (!state_subject->OnStateChanged(handle, state)) {
-      // Clean up the state tracker when we reach a terminal state.
-      // This means that no future callbacks will occur anymore.
-      delete state_subject;
+    {
+      std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_);
+      auto it = StateChangedSubject::state_subject_map_.find(handle);
+      // If the object is already deleted, do nothing.
+      if (it == StateChangedSubject::state_subject_map_.end()) {
+        return;
+      }
+
+      StateChangedSubject* state_subject = it->second;
+      if (!state_subject->OnStateChanged(handle, state)) {
+        // Clean up the state tracker when we reach a terminal state.
+        // This means that no future callbacks will occur anymore.
+        StateChangedSubject::state_subject_map_.erase(it);
+        delete state_subject;
+      }
     }
   }
 
@@ -447,6 +461,10 @@ class StateChangedSubject {
   // of just being subject-like?
 };
 
+std::mutex StateChangedSubject::state_subject_mutex_;
+std::unordered_map<::perfetto::consumer::Handle,
+    StateChangedSubject*> StateChangedSubject::state_subject_map_;
+
 // Note: The states will be emitted on a separate thread, so e.g. #as_blocking()
 // needs to be used to avoid dropping everything on the floor.
 //
@@ -484,6 +502,11 @@ static auto /*[observable<State>, shared_ptr<PerfettoConsumerHandle>]*/
         return;
       }
 
+      {
+        std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_);
+        StateChangedSubject::state_subject_map_[handle] = state_subject.get();
+      }
+
       std::shared_ptr<PerfettoConsumerHandle> safe_handle{
           new PerfettoConsumerHandle{perfetto_consumer, handle}};
 
@@ -567,11 +590,63 @@ bool BinaryWireProtobuf<T>::WriteStringToFd(int fd) const {
   return true;
 }
 
+template <typename T>
+std::optional<BinaryWireProtobuf<T>> BinaryWireProtobuf<T>::ReadFullyFromFile(
+    const std::string& path,
+    bool follow_symlinks) {
+  std::vector<std::byte> data;
+
+  int flags = O_RDONLY | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW);
+  android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags)));
+  if (fd == -1) {
+    return std::nullopt;
+  }
+
+  if (ReadFdToString(fd.get(), /*out*/&data)) {
+    return BinaryWireProtobuf<T>{std::move(data)};
+  } else {
+    return std::nullopt;
+  }
+}
+
+template <typename T>
+bool BinaryWireProtobuf<T>::operator==(const BinaryWireProtobuf<T>& other) const {
+  if (data_.size() != other.data_.size()) {
+    return false;
+  }
+  return std::equal(data_.begin(), data_.end(), other.data_.begin());
+}
+
+template <typename T>
+bool BinaryWireProtobuf<T>::ReadFdToString(int fd, /*out*/std::vector<std::byte>* content) {
+  DCHECK(content != nullptr);
+
+  content->clear();
+
+  struct stat sb;
+  if (fstat(fd, /*out*/&sb) != -1 && sb.st_size > 0) {
+    content->reserve(sb.st_size);
+  }
+
+  char buf[BUFSIZ];
+  auto it = content->begin();
+  ssize_t n;
+  while ((n = TEMP_FAILURE_RETRY(read(fd, &buf[0], sizeof(buf)))) > 0) {
+    content->insert(it,
+                    reinterpret_cast<std::byte*>(&buf[0]),
+                    reinterpret_cast<std::byte*>(&buf[n]));
+
+    std::advance(/*inout*/it, static_cast<size_t>(n));
+
+    static_assert(sizeof(char) == sizeof(std::byte), "sanity check for reinterpret cast");
+  }
+  return (n == 0) ? true : false;
+}
+
 // explicit template instantiation.
 template struct BinaryWireProtobuf<::google::protobuf::MessageLite>;
 // TODO: refactor this not to need the template instantiation.
 
-#if defined(__ANDROID__)
 // Copy of the 2.6.18 kernel header (linux/ioprio.h)
 
 #define IOPRIO_WHO_PROCESS (1)
@@ -584,7 +659,6 @@ template struct BinaryWireProtobuf<::google::protobuf::MessageLite>;
 #define IOPRIO_PRIO_CLASS(mask)	((mask) >> IOPRIO_CLASS_SHIFT)
 #define IOPRIO_PRIO_DATA(mask)	((mask) & IOPRIO_PRIO_MASK)
 #define IOPRIO_PRIO_VALUE(class, data)	(((class) << IOPRIO_CLASS_SHIFT) | data)
-#endif
 
 static int ioprio_get(int which, int who) {
   return syscall(SYS_ioprio_get, which, who);
diff --git a/src/perfetto/rx_producer.h b/src/perfetto/rx_producer.h
index f4c40b4..4e93f0d 100644
--- a/src/perfetto/rx_producer.h
+++ b/src/perfetto/rx_producer.h
@@ -51,10 +51,39 @@ struct PerfettoDependencies {
                                                       uint32_t buffer_size = 4096);
 };
 
+namespace detail {
+  template <typename T>
+  struct concept_message_lite_base {
+    static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>,
+                  "T must inherit from MessageLite");
+    using type = T;
+  };
+
+  template <typename T>
+  using concept_message_lite_base_t = typename concept_message_lite_base<T>::type;
+}  // namespace detail
+
+/*
+ * In Android's version of libprotobuf, move-constructors are not generated.
+ * This results in a legitimate (~10sec per TracePacket being compiled) slowdown,
+ * so we need to avoid it everywhere.
+ *
+ * 1) Don't copy the protos, move them instead.
+ * 2) Use 'shared_ptr' because rxcpp won't compile with unique_ptr.
+ */
+template <typename T>
+using ProtobufPtr = std::shared_ptr<detail::concept_message_lite_base_t<const T>>;
+
+template <typename T>
+using ProtobufMutablePtr = std::shared_ptr<detail::concept_message_lite_base_t<T>>;
+
 // This acts as a lightweight type marker so that we know what data has actually
 // encoded under the hood.
 template <typename T>
 struct BinaryWireProtobuf {
+  static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>,
+                "T should be a base class of MessageLite");
+
   std::vector<std::byte>& data() {
     return data_;
   }
@@ -78,13 +107,21 @@ struct BinaryWireProtobuf {
               data_.data());
   }
 
+  explicit BinaryWireProtobuf(std::vector<std::byte> data) : data_{std::move(data)} {
+  }
+
+  // You wouldn't want to accidentally copy a giant multi-megabyte chunk would you?
+  // BinaryWireProtobuf(const BinaryWireProtobuf& other) = delete;  // FIXME: rx likes to copy.
+  BinaryWireProtobuf(const BinaryWireProtobuf& other) = default;
+  BinaryWireProtobuf(BinaryWireProtobuf&& other) = default;
+
   // Important: Deserialization could fail, for example data is truncated or
   // some minor disc corruption occurred.
   template <typename U>
-  std::optional<U> MaybeUnserialize() {
-    U unencoded;
+  std::optional<ProtobufPtr<U>> MaybeUnserialize() {
+    ProtobufMutablePtr<U> unencoded{new U{}};
 
-    if (!unencoded.ParseFromArray(data_.data(), data_.size())) {
+    if (!unencoded->ParseFromArray(data_.data(), data_.size())) {
       return std::nullopt;
     }
 
@@ -94,10 +131,20 @@ struct BinaryWireProtobuf {
   bool WriteFullyToFile(const std::string& path,
                         bool follow_symlinks = false) const;
 
+  static std::optional<BinaryWireProtobuf<T>> ReadFullyFromFile(const std::string& path,
+                                                                bool follow_symlinks = false);
+
+  bool operator==(const BinaryWireProtobuf<T>& other) const;
+  bool operator!=(const BinaryWireProtobuf<T>& other) const {
+    return !(*this == other);
+  }
+
  private:
   static bool CleanUpAfterFailedWrite(const std::string& path);
   bool WriteStringToFd(int fd) const;
 
+  static bool ReadFdToString(int fd, /*out*/std::vector<std::byte>* data);
+
   std::vector<std::byte> data_;
 };
 
diff --git a/src/prefetcher/main.cc b/src/prefetcher/main.cc
new file mode 100644
index 0000000..94ba141
--- /dev/null
+++ b/src/prefetcher/main.cc
@@ -0,0 +1,190 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/debug.h"
+#include "common/loggers.h"
+#include "prefetcher/prefetcher_daemon.h"
+
+#include <android-base/parseint.h>
+#include <android-base/logging.h>
+
+#include <iostream>
+#include <optional>
+#include <string_view>
+#include <string>
+#include <vector>
+
+#include <signal.h>
+
+#if defined(IORAP_PREFETCHER_MAIN)
+
+namespace iorap::prefetcher {
+
+void Usage(char** argv) {
+  std::cerr << "Usage: " << argv[0] << " [--input-fd=#] [--output-fd=#]" << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Run the readahead daemon which can prefetch files given a command." << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Optional flags:" << std::endl;
+  std::cerr << "    --help,-h                  Print this Usage." << std::endl;
+  std::cerr << "    --input-fd,-if             Input FD (default stdin)." << std::endl;
+  std::cerr << "    --output-fd,-of            Output FD (default stdout)." << std::endl;
+  std::cerr << "    --use-sockets,-us          Use AF_UNIX sockets (default off)." << std::endl;
+  std::cerr << "    --command-format=[text|binary],-cf   (default text)." << std::endl;
+  std::cerr << "    --verbose,-v               Set verbosity (default off)." << std::endl;
+  std::cerr << "    --wait,-w                  Wait for key stroke before continuing (default off)." << std::endl;
+  exit(1);
+}
+
+int Main(int argc, char** argv) {
+  // Go to system logcat + stderr when running from command line.
+  android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM});
+
+  bool wait_for_keystroke = false;
+  bool enable_verbose = false;
+
+  bool command_format_text = false; // false = binary.
+
+  int arg_input_fd = -1;
+  int arg_output_fd = -1;
+
+  std::vector<std::string> arg_input_filenames;
+  bool arg_use_sockets = false;
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+    bool has_arg_next = (arg+1)<argc;
+    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+
+    if (argstr == "--help" || argstr == "-h") {
+      Usage(argv);
+    } else if (argstr == "--input-fd" || argstr == "-if") {
+      if (!has_arg_next) {
+        LOG(ERROR) << "--input-fd=<numeric-value>";
+        Usage(argv);
+      }
+      if (!::android::base::ParseInt(arg_next, /*out*/&arg_input_fd)) {
+        LOG(ERROR) << "--input-fd value must be numeric";
+        Usage(argv);
+      }
+    } else if (argstr == "--output-fd" || argstr == "-of") {
+      if (!has_arg_next) {
+        LOG(ERROR) << "--output-fd=<numeric-value>";
+        Usage(argv);
+      }
+      if (!::android::base::ParseInt(arg_next, /*out*/&arg_output_fd)) {
+        LOG(ERROR) << "--output-fd value must be numeric";
+        Usage(argv);
+      }
+    } else if (argstr == "--command-format=" || argstr == "-cf") {
+      if (!has_arg_next) {
+        LOG(ERROR) << "--command-format=text|binary";
+        Usage(argv);
+      }
+      if (arg_next == "text") {
+        command_format_text = true;
+      } else if (arg_next == "binary") {
+        command_format_text = false;
+      } else {
+        LOG(ERROR) << "--command-format must be one of {text,binary}";
+        Usage(argv);
+      }
+    } else if (argstr == "--use-sockets" || argstr == "-us") {
+      arg_use_sockets = true;
+    } else if (argstr == "--verbose" || argstr == "-v") {
+      enable_verbose = true;
+    } else if (argstr == "--wait" || argstr == "-w") {
+      wait_for_keystroke = true;
+    } else {
+      arg_input_filenames.push_back(argstr);
+    }
+  }
+
+  if (enable_verbose) {
+    android::base::SetMinimumLogSeverity(android::base::VERBOSE);
+
+    LOG(VERBOSE) << "Verbose check";
+    LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild;
+  } else {
+    android::base::SetMinimumLogSeverity(android::base::DEBUG);
+  }
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+  }
+
+  // Useful to attach a debugger...
+  // 1) $> iorap.cmd.readahead -w <args>
+  // 2) $> gdbclient <pid>
+  if (wait_for_keystroke) {
+    LOG(INFO) << "Self pid: " << getpid();
+
+    raise(SIGSTOP);
+    // LOG(INFO) << "Press any key to continue...";
+    // std::cin >> wait_for_keystroke;
+  }
+
+  // auto system_call = std::make_unique<SystemCallImpl>();
+  // TODO: mock readahead calls?
+  //
+  // Uncomment this if we want to leave the process around to inspect it from adb shell.
+  // sleep(100000);
+
+  int return_code = 0;
+
+  LOG(VERBOSE) << "Hello world";
+
+  if (arg_input_fd == -1) {
+    arg_input_fd = STDIN_FILENO;
+  }
+  if (arg_output_fd == -1) {
+    arg_output_fd = STDOUT_FILENO;
+  }
+
+  PrefetcherForkParameters params{};
+  params.input_fd = arg_input_fd;
+  params.output_fd = arg_output_fd;
+  params.format_text = command_format_text;
+  params.use_sockets = arg_use_sockets;
+
+  LOG(VERBOSE) << "main: Starting PrefetcherDaemon: "
+               << "input_fd=" << params.input_fd
+               << ",output_fd=" << params.output_fd;
+  {
+    PrefetcherDaemon daemon;
+    // Blocks until receiving an exit command.
+    daemon.Main(std::move(params));
+  }
+  LOG(VERBOSE) << "main: Terminating";
+
+  // 0 -> successfully executed all commands.
+  // 1 -> failed along the way (#on_error and also see the error logs).
+  return return_code;
+}
+
+}  // namespace iorap::prefetcher
+
+int main(int argc, char** argv) {
+  return ::iorap::prefetcher::Main(argc, argv);
+}
+
+#endif  // IORAP_PREFETCHER_MAIN
diff --git a/src/prefetcher/main_client.cc b/src/prefetcher/main_client.cc
new file mode 100644
index 0000000..ff7ee6d
--- /dev/null
+++ b/src/prefetcher/main_client.cc
@@ -0,0 +1,160 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "common/debug.h"
+#include "prefetcher/read_ahead.h"
+#include "prefetcher/task_id.h"
+
+#include <android-base/parseint.h>
+#include <android-base/logging.h>
+
+#include <iostream>
+#include <optional>
+#include <string_view>
+#include <string>
+#include <vector>
+
+#include <signal.h>
+#include <unistd.h>
+
+
+namespace iorap::prefetcher {
+
+static void UsageClient(char** argv) {
+  std::cerr << "UsageClient: " << argv[0] << " <path-to-compiled-trace.pb> [... pathN]" << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Run the readahead daemon which can prefetch files given a command." << std::endl;
+  std::cerr << "" << std::endl;
+  std::cerr << "  Optional flags:" << std::endl;
+  std::cerr << "    --help,-h                  Print this UsageClient." << std::endl;
+  std::cerr << "    --verbose,-v               Set verbosity (default off)." << std::endl;
+  std::cerr << "    --task-duration-ms,-tdm    Set task duration (default: 0ms)." << std::endl;
+  std::cerr << "    --use-sockets,-us          Use AF_UNIX sockets (default: off)" << std::endl;
+  std::cerr << "    --wait,-w                  Wait for key stroke before continuing (default off)." << std::endl;
+  exit(1);
+}
+
+int MainClient(int argc, char** argv) {
+  android::base::InitLogging(argv);
+  android::base::SetLogger(android::base::StderrLogger);
+
+  bool wait_for_keystroke = false;
+  bool enable_verbose = false;
+
+  bool command_format_text = false; // false = binary.
+
+  unsigned int arg_task_duration_ms = 10000;
+  std::vector<std::string> arg_input_filenames;
+  bool arg_use_sockets = false;
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+    bool has_arg_next = (arg+1)<argc;
+    std::string arg_next = has_arg_next ? argv[arg+1] : "";
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+
+    if (argstr == "--help" || argstr == "-h") {
+      UsageClient(argv);
+    } else if (argstr == "--use-sockets" || argstr == "-us") {
+      arg_use_sockets = true;
+    } else if (argstr == "--verbose" || argstr == "-v") {
+      enable_verbose = true;
+    } else if (argstr == "--wait" || argstr == "-w") {
+      wait_for_keystroke = true;
+    } else if (argstr == "--task-duration-ms" || argstr == "-tdm") {
+      if (!has_arg_next) {
+        LOG(ERROR) << "--task-duration-ms: requires uint parameter";
+        UsageClient(argv);
+      } else if (!::android::base::ParseUint(arg_next, &arg_task_duration_ms)) {
+        LOG(ERROR) << "--task-duration-ms: requires non-negative parameter";
+        UsageClient(argv);
+      }
+    } else {
+      arg_input_filenames.push_back(argstr);
+    }
+  }
+
+  if (enable_verbose) {
+    android::base::SetMinimumLogSeverity(android::base::VERBOSE);
+
+    LOG(VERBOSE) << "Verbose check";
+    LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild;
+  } else {
+    android::base::SetMinimumLogSeverity(android::base::DEBUG);
+  }
+
+  LOG(VERBOSE) << "argparse: argc=" << argc;
+
+  for (int arg = 1; arg < argc; ++arg) {
+    std::string argstr = argv[arg];
+
+    LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr;
+  }
+
+  // Useful to attach a debugger...
+  // 1) $> iorap.cmd.readahead -w <args>
+  // 2) $> gdbclient <pid>
+  if (wait_for_keystroke) {
+    LOG(INFO) << "Self pid: " << getpid();
+
+    raise(SIGSTOP);
+    // LOG(INFO) << "Press any key to continue...";
+    // std::cin >> wait_for_keystroke;
+  }
+
+  // auto system_call = std::make_unique<SystemCallImpl>();
+  // TODO: mock readahead calls?
+  //
+  // Uncomment this if we want to leave the process around to inspect it from adb shell.
+  // sleep(100000);
+
+  int return_code = 0;
+
+  LOG(VERBOSE) << "Hello world";
+
+  ReadAhead read_ahead{arg_use_sockets};  // Don't count the time it takes to fork+exec.
+
+  size_t task_id_counter = 0;
+  for (const std::string& compiled_trace_path : arg_input_filenames) {
+    TaskId task_id{task_id_counter++, compiled_trace_path};
+
+    LOG(DEBUG) << "main: ReadAhead BeginTask: "
+                 << "task_duration_ms=" << arg_task_duration_ms << ","
+                 << task_id;
+
+    read_ahead.BeginTask(task_id);
+    usleep(arg_task_duration_ms*1000);
+
+    LOG(DEBUG) << "main: ReadAhead FinishTask: " << task_id;
+
+    read_ahead.FinishTask(task_id);
+  }
+  LOG(VERBOSE) << "main: Terminating";
+
+  // 0 -> successfully executed all commands.
+  // 1 -> failed along the way (#on_error and also see the error logs).
+  return return_code;
+}
+
+}  // namespace iorap::prefetcher
+
+#if defined(IORAP_PREFETCHER_MAIN_CLIENT)
+int main(int argc, char** argv) {
+  return ::iorap::prefetcher::MainClient(argc, argv);
+}
+
+#endif  // IORAP_PREFETCHER_MAIN_CLIENT
diff --git a/src/prefetcher/minijail.cc b/src/prefetcher/minijail.cc
new file mode 100644
index 0000000..df4b11b
--- /dev/null
+++ b/src/prefetcher/minijail.cc
@@ -0,0 +1,49 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "prefetcher/minijail.h"
+
+#include <android-base/logging.h>
+#include <libminijail.h>
+
+namespace iorap::prefetcher {
+
+static const char kSeccompFilePath[] = "/system/etc/seccomp_policy/iorap.prefetcherd.policy";
+
+bool MiniJail() {
+  /* no seccomp policy for this architecture */
+  if (access(kSeccompFilePath, R_OK) == -1) {
+      LOG(WARNING) << "No seccomp filter defined for this architecture.";
+      return true;
+  }
+
+  struct minijail* jail = minijail_new();
+  if (jail == NULL) {
+      LOG(WARNING) << "Failed to create minijail.";
+      return false;
+  }
+
+  minijail_no_new_privs(jail);
+  minijail_log_seccomp_filter_failures(jail);
+  minijail_use_seccomp_filter(jail);
+  minijail_parse_seccomp_filters(jail, kSeccompFilePath);
+  minijail_enter(jail);
+  minijail_destroy(jail);
+
+  LOG(DEBUG) << "minijail installed.";
+
+  return true;
+}
+
+}
diff --git a/src/prefetcher/minijail.h b/src/prefetcher/minijail.h
new file mode 100644
index 0000000..9caefe6
--- /dev/null
+++ b/src/prefetcher/minijail.h
@@ -0,0 +1,23 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SRC_PREFETCHER_MINIJAIL_H
+#define SRC_PREFETCHER_MINIJAIL_H
+
+namespace iorap::prefetcher {
+// Install a minijail using seccomp_policy/prefetcherd.{ARCH}.policy file.
+bool MiniJail();
+}
+
+#endif  // SRC_PREFETCHER_MINIJAIL_H
diff --git a/src/prefetcher/prefetcher_daemon.cc b/src/prefetcher/prefetcher_daemon.cc
new file mode 100644
index 0000000..f4b9087
--- /dev/null
+++ b/src/prefetcher/prefetcher_daemon.cc
@@ -0,0 +1,1367 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "prefetcher/minijail.h"
+#include "common/cmd_utils.h"
+#include "prefetcher/prefetcher_daemon.h"
+#include "prefetcher/session_manager.h"
+#include "prefetcher/session.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+
+#include <deque>
+#include <iomanip>
+#include <string>
+#include <sstream>
+#include <vector>
+
+#include <fcntl.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/un.h>
+#include <unistd.h>
+
+namespace iorap::prefetcher {
+
+// Gate super-spammy IPC logging behind a property.
+// This is beyond merely annoying, enabling this logging causes prefetching to be about 1000x slower.
+static bool LogVerboseIpc() {
+  static bool initialized = false;
+  static bool verbose_ipc;
+
+  if (initialized == false) {
+    initialized = true;
+
+    verbose_ipc =
+        ::android::base::GetBoolProperty("iorapd.readahead.verbose_ipc", /*default*/false);
+  }
+
+  return verbose_ipc;
+}
+
+static const bool kInstallMiniJail =
+    ::android::base::GetBoolProperty("iorapd.readahead.minijail", /*default*/true);
+
+static constexpr const char kCommandFileName[] = "/system/bin/iorap.prefetcherd";
+
+static constexpr size_t kPipeBufferSize = 1024 * 1024;  // matches /proc/sys/fs/pipe-max-size
+
+using ArgString = const char*;
+
+std::ostream& operator<<(std::ostream& os, ReadAheadKind ps) {
+  switch (ps) {
+    case ReadAheadKind::kFadvise:
+      os << "fadvise";
+      break;
+    case ReadAheadKind::kMmapLocked:
+      os << "mmap";
+      break;
+    case ReadAheadKind::kMlock:
+      os << "mlock";
+      break;
+    default:
+      os << "<invalid>";
+  }
+  return os;
+}
+
+std::ostream& operator<<(std::ostream& os, CommandChoice choice) {
+  switch (choice) {
+    case CommandChoice::kRegisterFilePath:
+      os << "kRegisterFilePath";
+      break;
+    case CommandChoice::kUnregisterFilePath:
+      os << "kUnregisterFilePath";
+      break;
+    case CommandChoice::kReadAhead:
+      os << "kReadAhead";
+      break;
+    case CommandChoice::kExit:
+      os << "kExit";
+      break;
+    case CommandChoice::kCreateSession:
+      os << "kCreateSession";
+      break;
+    case CommandChoice::kDestroySession:
+      os << "kDestroySession";
+      break;
+    case CommandChoice::kDumpSession:
+      os << "kDumpSession";
+      break;
+    case CommandChoice::kDumpEverything:
+      os << "kDumpEverything";
+      break;
+    case CommandChoice::kCreateFdSession:
+      os << "kCreateFdSession";
+      break;
+    default:
+      CHECK(false) << "forgot to handle this choice";
+      break;
+  }
+  return os;
+}
+
+std::ostream& operator<<(std::ostream& os, const Command& command) {
+  os << "Command{";
+  os << "choice=" << command.choice << ",";
+
+  bool has_session_id = true;
+  bool has_id = true;
+  switch (command.choice) {
+    case CommandChoice::kDumpEverything:
+    case CommandChoice::kExit:
+      has_session_id = false;
+      FALLTHROUGH_INTENDED;
+    case CommandChoice::kCreateFdSession:
+    case CommandChoice::kCreateSession:
+    case CommandChoice::kDestroySession:
+    case CommandChoice::kDumpSession:
+      has_id = false;
+      break;
+    default:
+      break;
+  }
+
+  if (has_session_id) {
+    os << "sid=" << command.session_id << ",";
+  }
+
+  if (has_id) {
+    os << "id=" << command.id << ",";
+  }
+
+  switch (command.choice) {
+    case CommandChoice::kRegisterFilePath:
+      os << "file_path=";
+
+      if (command.file_path) {
+        os << *(command.file_path);
+      } else {
+        os << "(nullopt)";
+      }
+      break;
+    case CommandChoice::kUnregisterFilePath:
+      break;
+    case CommandChoice::kReadAhead:
+      os << "read_ahead_kind=" << command.read_ahead_kind << ",";
+      os << "length=" << command.length << ",";
+      os << "offset=" << command.offset << ",";
+      break;
+    case CommandChoice::kExit:
+      break;
+    case CommandChoice::kCreateFdSession:
+      os << "fd=";
+      if (command.fd.has_value()) {
+        os << command.fd.value();
+      } else {
+        os << "(nullopt)";
+      }
+      os << ",";
+    FALLTHROUGH_INTENDED;
+    case CommandChoice::kCreateSession:
+      os << "description=";
+      if (command.file_path) {
+        os << "'" << *(command.file_path) << "'";
+      } else {
+        os << "(nullopt)";
+      }
+      break;
+    case CommandChoice::kDestroySession:
+      break;
+    case CommandChoice::kDumpSession:
+      break;
+    case CommandChoice::kDumpEverything:
+      break;
+    default:
+      CHECK(false) << "forgot to handle this choice";
+      break;
+  }
+
+  os << "}";
+
+  return os;
+}
+
+template <typename T>
+struct ParseResult {
+  T value;
+  char* next_token;
+  size_t stream_size;
+
+  ParseResult() : value{}, next_token{nullptr}, stream_size{} {
+  }
+
+  constexpr operator bool() const {
+    return next_token != nullptr;
+  }
+};
+
+// Very spammy: Keep it off by default. Set to true if changing this code.
+static constexpr bool kDebugParsingRead = false;
+
+#define DEBUG_PREAD if (kDebugParsingRead) LOG(VERBOSE) << "ParsingRead "
+
+
+
+// Parse a strong type T from a buffer stream.
+// If there's insufficient space left to parse the value, an empty ParseResult is returned.
+template <typename T>
+ParseResult<T> ParsingRead(char* stream, size_t stream_size) {
+  if (stream == nullptr) {
+    DEBUG_PREAD << "stream was null";
+    return {};
+  }
+
+  if constexpr (std::is_same_v<T, std::string>) {
+    ParseResult<uint32_t> length = ParsingRead<uint32_t>(stream, stream_size);
+
+    if (!length) {
+      DEBUG_PREAD << "could not find length";
+      // Not enough bytes left?
+      return {};
+    }
+
+    ParseResult<std::string> string_result;
+    string_result.value.reserve(length);
+
+    stream = length.next_token;
+    stream_size = length.stream_size;
+
+    for (size_t i = 0; i < length.value; ++i) {
+      ParseResult<char> char_result = ParsingRead<char>(stream, stream_size);
+
+      stream = char_result.next_token;
+      stream_size = char_result.stream_size;
+
+      if (!char_result) {
+        DEBUG_PREAD << "too few chars in stream, expected length: " << length.value;
+        // Not enough bytes left?
+        return {};
+      }
+
+      string_result.value += char_result.value;
+
+      DEBUG_PREAD << "string preliminary is : " << string_result.value;
+    }
+
+    DEBUG_PREAD << "parsed string to: " << string_result.value;
+    string_result.next_token = stream;
+    return string_result;
+  } else {
+    if (sizeof(T) > stream_size) {
+      return {};
+    }
+
+    ParseResult<T> result;
+    result.next_token = stream + sizeof(T);
+    result.stream_size = stream_size - sizeof(T);
+
+    memcpy(&result.value, stream, sizeof(T));
+
+    return result;
+  }
+}
+
+// Convenience overload to chain multiple ParsingRead together.
+template <typename T, typename U>
+ParseResult<T> ParsingRead(ParseResult<U> result) {
+  return ParsingRead<T>(result.next_token, result.stream_size);
+}
+
+class CommandParser {
+ public:
+  CommandParser(PrefetcherForkParameters params) {
+    params_ = params;
+  }
+
+  std::vector<Command> ParseSocketCommands(bool& eof) {
+    eof = false;
+
+    std::vector<Command> commands_vec;
+
+    std::vector<char> buf_vector;
+    buf_vector.resize(1024*1024);  // 1MB.
+    char* buf = &buf_vector[0];
+
+    // Binary only parsing. The higher level code can parse text
+    // with ifstream if it really wants to.
+    char* stream = &buf[0];
+    size_t stream_size = buf_vector.size();
+
+    while (true) {
+      if (stream_size == 0) {
+        // TODO: reply with an overflow command.
+        LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands.";
+        stream = &buf[0];
+        stream_size = buf_vector.size();
+        memset(&buf[0], /*c*/0, buf_vector.size());
+      }
+
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon block recvmsg for commands (fd=" << params_.input_fd << ")";
+      }
+
+      ssize_t count;
+      struct msghdr hdr;
+      memset(&hdr, 0, sizeof(hdr));
+
+      {
+        union {
+          struct cmsghdr cmh;
+          char   control[CMSG_SPACE(sizeof(int))];
+        } control_un;
+        memset(&control_un, 0, sizeof(control_un));
+
+        /* Set 'control_un' to describe ancillary data that we want to receive */
+        control_un.cmh.cmsg_len = CMSG_LEN(sizeof(int));  /* fd is sizeof(int) */
+        control_un.cmh.cmsg_level = SOL_SOCKET;
+        control_un.cmh.cmsg_type = SCM_CREDENTIALS;
+
+        // the regular message data will be read into stream
+        struct iovec iov;
+        memset(&iov, 0, sizeof(iov));
+        iov.iov_base = stream;
+        iov.iov_len = stream_size;
+
+        /* Set hdr fields to describe 'control_un' */
+        hdr.msg_control = control_un.control;
+        hdr.msg_controllen = sizeof(control_un.control);
+        hdr.msg_iov = &iov;
+        hdr.msg_iovlen = 1;
+        hdr.msg_name = nullptr; /* no peer address */
+        hdr.msg_namelen = 0;
+
+        count = TEMP_FAILURE_RETRY(recvmsg(params_.input_fd, &hdr, /*flags*/0));
+      }
+
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon recvmsg " << count << " for stream size:" << stream_size;
+      }
+
+      if (count < 0) {
+        PLOG(ERROR) << "failed to recvmsg from input fd";
+        break;
+        // TODO: let the daemon be restarted by higher level code?
+      } else if (count == 0) {
+        LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating";
+        eof = true;
+        break;
+        // TODO: let the daemon be restarted by higher level code?
+      }
+
+      {
+        /* Extract fd from ancillary data if present */
+        struct cmsghdr* hp;
+        hp = CMSG_FIRSTHDR(&hdr);
+        if (hp                                              &&
+            // FIXME: hp->cmsg_len returns an absurdly large value. is it overflowing?
+            // (hp->cmsg_len == CMSG_LEN(sizeof(int)))          &&
+            (hp->cmsg_level == SOL_SOCKET)                   &&
+            (hp->cmsg_type == SCM_RIGHTS)) {
+
+          int passed_fd = *(int*) CMSG_DATA(hp);
+          if (LogVerboseIpc()) {
+            LOG(VERBOSE) << "PrefetcherDaemon received FD " << passed_fd;
+          }
+
+          // tack the FD into our dequeue.
+          // we assume the FDs are sent in-order same as the regular iov are sent in-order.
+          longbuf_fds_.insert(longbuf_fds_.end(), passed_fd);
+        } else if (hp != nullptr) {
+          if (LogVerboseIpc()) {
+            LOG(VERBOSE) << "PrefetcherDaemon::read got CMSG but it wasn't matching SCM_RIGHTS,"
+                         << "cmsg_len=" << hp->cmsg_len << ","
+                         << "cmsg_level=" << hp->cmsg_level << ","
+                         << "cmsg_type=" << hp->cmsg_type;
+          }
+        }
+      }
+
+      longbuf_.insert(longbuf_.end(), stream, stream + count);
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size();
+      }
+
+      // reconstruct a stream of [iov_Command chdr_fd?]* back into [Command]*
+      {
+        if (longbuf_.size() == 0) {
+          break;
+        }
+
+        std::vector<char> v(longbuf_.begin(),
+                            longbuf_.end());
+
+        std::vector<int> v_fds{longbuf_fds_.begin(), longbuf_fds_.end()};
+
+        if (LogVerboseIpc()) {
+          LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size();
+          if (WOULD_LOG(VERBOSE)) {
+            std::stringstream dump;
+            dump << std::hex << std::setfill('0');
+            for (size_t i = 0; i < v.size(); ++i) {
+              dump << std::setw(2) << static_cast<unsigned>(v[i]);
+            }
+
+            LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str();
+          }
+          LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ size: " << v_fds.size();
+          if (WOULD_LOG(VERBOSE)) {
+            std::stringstream dump;
+            for (size_t i = 0; i < v_fds.size(); ++i) {
+              dump << v_fds[i] << ", ";
+            }
+
+            LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ dump: " << dump.str();
+          }
+
+        }
+
+        size_t v_fds_off = 0;
+        size_t consumed_fds_total = 0;
+
+        size_t v_off = 0;
+        size_t consumed_bytes = std::numeric_limits<size_t>::max();
+        size_t consumed_total = 0;
+
+        while (true) {
+          std::optional<Command> maybe_command;
+          maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes);
+          consumed_total += consumed_bytes;
+          // Normal every time we get to the end of a buffer.
+          if (!maybe_command) {
+              if (LogVerboseIpc()) {
+                LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size();
+              }
+              break;
+          }
+
+          if (maybe_command->RequiresFd()) {
+            if (v_fds_off < v_fds.size()) {
+              maybe_command->fd = v_fds[v_fds_off++];
+              consumed_fds_total++;
+              if (LogVerboseIpc()) {
+                LOG(VERBOSE) << "Append the FD to " << *maybe_command;
+              }
+            } else {
+              LOG(WARNING) << "Failed to acquire FD for " << *maybe_command;
+            }
+          }
+
+          // in the next pass ignore what we already consumed.
+          v_off += consumed_bytes;
+
+          // true as long we don't hit the 'break' above.
+          DCHECK_EQ(v_off, consumed_total);
+          if (LogVerboseIpc()) {
+            LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size()
+                         << "," << *maybe_command;
+
+            // Pretty-print a single command for debugging/testing.
+            LOG(VERBOSE) << *maybe_command;
+          }
+
+          // add to the commands we parsed.
+          commands_vec.push_back(*maybe_command);
+        }
+
+        // erase however many were consumed
+        longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total);
+
+        // erase however many FDs were consumed.
+        longbuf_fds_.erase(longbuf_fds_.begin(), longbuf_fds_.begin() + consumed_fds_total);
+      }
+      break;
+    }
+
+    return commands_vec;
+  }
+
+  std::vector<Command> ParseCommands(bool& eof) {
+    eof = false;
+
+    std::vector<Command> commands_vec;
+
+    std::vector<char> buf_vector;
+    buf_vector.resize(kPipeBufferSize);
+    char* buf = &buf_vector[0];
+
+    // Binary only parsing. The higher level code can parse text
+    // with ifstream if it really wants to.
+    char* stream = &buf[0];
+    size_t stream_size = buf_vector.size();
+
+    while (true) {
+      if (stream_size == 0) {
+        // TODO: reply with an overflow command.
+        LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands.";
+        stream = &buf[0];
+        stream_size = buf_vector.size();
+        memset(&buf[0], /*c*/0, buf_vector.size());
+      }
+
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon block read for commands (fd=" << params_.input_fd << ")";
+      }
+      ssize_t count = TEMP_FAILURE_RETRY(read(params_.input_fd, stream, stream_size));
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon::read " << count << " for stream size:" << stream_size;
+      }
+
+      if (count < 0) {
+        PLOG(ERROR) << "failed to read from input fd";
+        break;
+        // TODO: let the daemon be restarted by higher level code?
+      } else if (count == 0) {
+        LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating";
+        eof = true;
+        break;
+        // TODO: let the daemon be restarted by higher level code?
+      }
+
+      longbuf_.insert(longbuf_.end(), stream, stream + count);
+      if (LogVerboseIpc()) {
+        LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size();
+      }
+
+      std::optional<Command> maybe_command;
+      {
+        if (longbuf_.size() == 0) {
+          break;
+        }
+
+        std::vector<char> v(longbuf_.begin(),
+                            longbuf_.end());
+
+        if (LogVerboseIpc()) {
+          LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size();
+          if (WOULD_LOG(VERBOSE)) {
+            std::stringstream dump;
+            dump << std::hex << std::setfill('0');
+            for (size_t i = 0; i < v.size(); ++i) {
+              dump << std::setw(2) << static_cast<unsigned>(v[i]);
+            }
+
+            LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str();
+          }
+        }
+
+        size_t v_off = 0;
+        size_t consumed_bytes = std::numeric_limits<size_t>::max();
+        size_t consumed_total = 0;
+
+        while (true) {
+          maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes);
+          consumed_total += consumed_bytes;
+          // Normal every time we get to the end of a buffer.
+          if (!maybe_command) {
+              if (LogVerboseIpc()) {
+                LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size();
+              }
+              break;
+          }
+
+          // in the next pass ignore what we already consumed.
+          v_off += consumed_bytes;
+
+          // true as long we don't hit the 'break' above.
+          DCHECK_EQ(v_off, consumed_total);
+          if (LogVerboseIpc()) {
+            LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size()
+                         << "," << *maybe_command;
+
+            // Pretty-print a single command for debugging/testing.
+            LOG(VERBOSE) << *maybe_command;
+          }
+
+          // add to the commands we parsed.
+          commands_vec.push_back(*maybe_command);
+        }
+
+        // erase however many were consumed
+        longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total);
+      }
+      break;
+    }
+
+    return commands_vec;
+  }
+
+ private:
+  bool IsTextMode() const {
+    return params_.format_text;
+  }
+
+  PrefetcherForkParameters params_;
+
+  // A buffer long enough to contain a lot of buffers.
+  // This handles reads that only contain a partial command.
+  std::deque<char> longbuf_;
+
+  // File descriptor buffers.
+  std::deque<int> longbuf_fds_;
+};
+
+static constexpr bool kDebugCommandRead = true;
+
+#define DEBUG_READ if (kDebugCommandRead) LOG(VERBOSE) << "Command::Read "
+
+std::optional<Command> Command::Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes) {
+  *consumed_bytes = 0;
+  if (buf == nullptr) {
+    return std::nullopt;
+  }
+
+  Command cmd{};  // zero-initialize any unused fields
+  ParseResult<CommandChoice> parsed_choice = ParsingRead<CommandChoice>(buf, buf_size);
+  cmd.choice = parsed_choice.value;
+
+  if (!parsed_choice) {
+    DEBUG_READ << "no choice";
+    return std::nullopt;
+  }
+
+  switch (parsed_choice.value) {
+    case CommandChoice::kRegisterFilePath: {
+      ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice);
+      if (!parsed_session_id) {
+        DEBUG_READ << "no parsed session id";
+        return std::nullopt;
+      }
+
+      ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id);
+      if (!parsed_id) {
+        DEBUG_READ << "no parsed id";
+        return std::nullopt;
+      }
+
+      ParseResult<std::string> parsed_file_path = ParsingRead<std::string>(parsed_id);
+
+      if (!parsed_file_path) {
+        DEBUG_READ << "no file path";
+        return std::nullopt;
+      }
+      *consumed_bytes = parsed_file_path.next_token - buf;
+
+      cmd.session_id = parsed_session_id.value;
+      cmd.id = parsed_id.value;
+      cmd.file_path = parsed_file_path.value;
+
+      break;
+    }
+    case CommandChoice::kUnregisterFilePath: {
+      ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice);
+      if (!parsed_session_id) {
+        DEBUG_READ << "no parsed session id";
+        return std::nullopt;
+      }
+
+      ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id);
+      if (!parsed_id) {
+        DEBUG_READ << "no parsed id";
+        return std::nullopt;
+      }
+      *consumed_bytes = parsed_id.next_token - buf;
+
+      cmd.session_id = parsed_session_id.value;
+      cmd.id = parsed_id.value;
+
+      break;
+    }
+    case CommandChoice::kReadAhead: {
+      ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice);
+      if (!parsed_session_id) {
+        DEBUG_READ << "no parsed session id";
+        return std::nullopt;
+      }
+
+      ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id);
+      if (!parsed_id) {
+        DEBUG_READ << "no parsed id";
+        return std::nullopt;
+      }
+
+      ParseResult<ReadAheadKind> parsed_kind = ParsingRead<ReadAheadKind>(parsed_id);
+      if (!parsed_kind) {
+        DEBUG_READ << "no parsed kind";
+        return std::nullopt;
+      }
+      ParseResult<uint64_t> parsed_length = ParsingRead<uint64_t>(parsed_kind);
+      if (!parsed_length) {
+        DEBUG_READ << "no parsed length";
+        return std::nullopt;
+      }
+      ParseResult<uint64_t> parsed_offset = ParsingRead<uint64_t>(parsed_length);
+      if (!parsed_offset) {
+        DEBUG_READ << "no parsed offset";
+        return std::nullopt;
+      }
+      *consumed_bytes = parsed_offset.next_token - buf;
+
+      cmd.session_id = parsed_session_id.value;
+      cmd.id = parsed_id.value;
+      cmd.read_ahead_kind = parsed_kind.value;
+      cmd.length = parsed_length.value;
+      cmd.offset = parsed_offset.value;
+
+      break;
+    }
+    case CommandChoice::kCreateSession:
+    case CommandChoice::kCreateFdSession: {
+      ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice);
+      if (!parsed_session_id) {
+        DEBUG_READ << "no parsed session id";
+        return std::nullopt;
+      }
+
+      ParseResult<std::string> parsed_description = ParsingRead<std::string>(parsed_session_id);
+
+      if (!parsed_description) {
+        DEBUG_READ << "no description";
+        return std::nullopt;
+      }
+      *consumed_bytes = parsed_description.next_token - buf;
+
+      cmd.session_id = parsed_session_id.value;
+      cmd.file_path = parsed_description.value;
+
+      break;
+    }
+    case CommandChoice::kDestroySession:
+    case CommandChoice::kDumpSession: {
+      ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice);
+      if (!parsed_session_id) {
+        DEBUG_READ << "no parsed session id";
+        return std::nullopt;
+      }
+
+      *consumed_bytes = parsed_session_id.next_token - buf;
+
+      cmd.session_id = parsed_session_id.value;
+
+      break;
+    }
+    case CommandChoice::kExit:
+    case CommandChoice::kDumpEverything:
+      *consumed_bytes = parsed_choice.next_token - buf;
+      // Only need to parse the choice.
+      break;
+    default:
+      LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(parsed_choice.value);
+      break;
+  }
+
+  return cmd;
+}
+
+bool Command::Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const {
+  *produced_bytes = 0;
+  if (buf == nullptr) {
+    LOG(WARNING) << "null buf, is this expected?";
+    return false;
+  }
+
+  bool has_enough_space = false;
+  size_t space_requirement = std::numeric_limits<size_t>::max();
+
+  space_requirement = sizeof(choice);
+
+  switch (choice) {
+    case CommandChoice::kRegisterFilePath:
+      space_requirement += sizeof(session_id);
+      space_requirement += sizeof(id);
+      space_requirement += sizeof(uint32_t);    // string length
+
+      if (!file_path) {
+        LOG(WARNING) << "Missing file path for kRegisterFilePath";
+        return false;
+      }
+
+      space_requirement += file_path->size(); // string contents
+      break;
+    case CommandChoice::kUnregisterFilePath:
+      space_requirement += sizeof(session_id);
+      space_requirement += sizeof(id);
+      break;
+    case CommandChoice::kReadAhead:
+      space_requirement += sizeof(session_id);
+      space_requirement += sizeof(id);
+      space_requirement += sizeof(read_ahead_kind);
+      space_requirement += sizeof(length);
+      space_requirement += sizeof(offset);
+      break;
+    case CommandChoice::kCreateSession:
+    case CommandChoice::kCreateFdSession:
+      space_requirement += sizeof(session_id);
+      space_requirement += sizeof(uint32_t);    // string length
+
+      if (!file_path) {
+        LOG(WARNING) << "Missing file path for kCreateSession";
+        return false;
+      }
+
+      space_requirement += file_path->size(); // string contents
+      break;
+    case CommandChoice::kDestroySession:
+    case CommandChoice::kDumpSession:
+      space_requirement += sizeof(session_id);
+      break;
+    case CommandChoice::kExit:
+    case CommandChoice::kDumpEverything:
+      // Only need space for the choice.
+      break;
+    default:
+      LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(choice);
+      break;
+  }
+
+  if (buf_size < space_requirement) {
+    return false;
+  }
+
+  *produced_bytes = space_requirement;
+
+  // Always write out the choice.
+  size_t buf_offset = 0;
+
+  memcpy(&buf[buf_offset], &choice, sizeof(choice));
+  buf_offset += sizeof(choice);
+
+  switch (choice) {
+    case CommandChoice::kRegisterFilePath:
+      memcpy(&buf[buf_offset], &session_id, sizeof(session_id));
+      buf_offset += sizeof(session_id);
+      memcpy(&buf[buf_offset], &id, sizeof(id));
+      buf_offset += sizeof(id);
+
+      {
+        uint32_t string_length = static_cast<uint32_t>(file_path->size());
+        memcpy(&buf[buf_offset], &string_length, sizeof(string_length));
+        buf_offset += sizeof(string_length);
+      }
+
+      DCHECK(file_path.has_value());
+
+      memcpy(&buf[buf_offset], file_path->c_str(), file_path->size());
+      buf_offset += file_path->size();
+      break;
+    case CommandChoice::kUnregisterFilePath:
+      memcpy(&buf[buf_offset], &session_id, sizeof(session_id));
+      buf_offset += sizeof(session_id);
+      memcpy(&buf[buf_offset], &id, sizeof(id));
+      buf_offset += sizeof(id);
+      break;
+    case CommandChoice::kReadAhead:
+      memcpy(&buf[buf_offset], &session_id, sizeof(session_id));
+      buf_offset += sizeof(session_id);
+      memcpy(&buf[buf_offset], &id, sizeof(id));
+      buf_offset += sizeof(id);
+      memcpy(&buf[buf_offset], &read_ahead_kind, sizeof(read_ahead_kind));
+      buf_offset += sizeof(read_ahead_kind);
+      memcpy(&buf[buf_offset], &length, sizeof(length));
+      buf_offset += sizeof(length);
+      memcpy(&buf[buf_offset], &offset, sizeof(offset));
+      buf_offset += sizeof(offset);
+      break;
+    case CommandChoice::kCreateSession:
+    case CommandChoice::kCreateFdSession:
+      memcpy(&buf[buf_offset], &session_id, sizeof(session_id));
+      buf_offset += sizeof(session_id);
+
+      {
+        uint32_t string_length = static_cast<uint32_t>(file_path->size());
+        memcpy(&buf[buf_offset], &string_length, sizeof(string_length));
+        buf_offset += sizeof(string_length);
+      }
+
+      DCHECK(file_path.has_value());
+
+      memcpy(&buf[buf_offset], file_path->c_str(), file_path->size());
+      buf_offset += file_path->size();
+
+      DCHECK_EQ(buf_offset, space_requirement) << *this << ",file_path_size:" << file_path->size();
+      DCHECK_EQ(buf_offset, *produced_bytes) << *this;
+
+      break;
+    case CommandChoice::kDestroySession:
+    case CommandChoice::kDumpSession:
+      memcpy(&buf[buf_offset], &session_id, sizeof(session_id));
+      buf_offset += sizeof(session_id);
+      break;
+    case CommandChoice::kExit:
+    case CommandChoice::kDumpEverything:
+      // Only need to write out the choice.
+      break;
+    default:
+      LOG(FATAL) << "should have fallen out in the above switch"
+                 << static_cast<uint32_t>(choice);
+      break;
+  }
+
+  DCHECK_EQ(buf_offset, space_requirement) << *this;
+  DCHECK_EQ(buf_offset, *produced_bytes) << *this;
+
+  return true;
+}
+
+class PrefetcherDaemon::Impl {
+ public:
+  std::optional<PrefetcherForkParameters> StartPipesViaFork() {
+    int pipefds[2];
+    if (pipe(&pipefds[0]) != 0) {
+      PLOG(FATAL) << "Failed to create read/write pipes";
+    }
+
+    if (WOULD_LOG(VERBOSE)) {
+      long pipe_size = static_cast<long>(fcntl(pipefds[0], F_GETPIPE_SZ));
+      if (pipe_size < 0) {
+        PLOG(ERROR) << "Failed to F_GETPIPE_SZ:";
+      }
+      LOG(VERBOSE) << "StartPipesViaFork: default pipe size: " << pipe_size;
+    }
+
+    for (int i = 0; i < 2; ++i) {
+      // Default pipe size is usually 64KB.
+      // Increase to 1MB so that iorapd has to rarely run during prefetching.
+      if (fcntl(pipefds[i], F_SETPIPE_SZ, kPipeBufferSize) < 0) {
+        PLOG(FATAL) << "Failed to increase pipe size to max";
+      }
+    }
+
+    pipefd_read_ = pipefds[0];
+    pipefd_write_ = pipefds[1];
+
+    PrefetcherForkParameters params;
+    params.input_fd = pipefd_read_;
+    params.output_fd = pipefd_write_;
+    params.format_text = false;
+    params.use_sockets = false;
+
+    bool res = StartViaFork(params);
+    if (res) {
+      return params;
+    } else {
+      return std::nullopt;
+    }
+  }
+
+std::optional<PrefetcherForkParameters> StartSocketViaFork() {
+    int socket_fds[2];
+    if (socketpair(AF_UNIX, SOCK_STREAM, /*protocol*/0, &socket_fds[0]) != 0) {
+      PLOG(FATAL) << "Failed to create read/write socketpair";
+    }
+
+    pipefd_read_ = socket_fds[0];  // iorapd writer, iorap.prefetcherd reader
+    pipefd_write_ = socket_fds[1]; // iorapd reader, iorap.prefetcherd writer
+
+    PrefetcherForkParameters params;
+    params.input_fd = pipefd_read_;
+    params.output_fd = pipefd_write_;
+    params.format_text = false;
+    params.use_sockets = true;
+
+    bool res = StartViaFork(params);
+    if (res) {
+      return params;
+    } else {
+      return std::nullopt;
+    }
+  }
+
+  bool StartViaFork(PrefetcherForkParameters params) {
+    params_ = params;
+
+    forked_ = true;
+    child_ = fork();
+
+    if (child_ == -1) {
+      LOG(FATAL) << "Failed to fork PrefetcherDaemon";
+    } else if (child_ > 0) {  // we are the caller of this function
+      LOG(DEBUG) << "forked into iorap.prefetcherd, pid = " << child_;
+
+      return true;
+    } else {
+      // we are the child that was forked.
+      std::stringstream argv;  // for logging
+      std::vector<std::string> argv_vec;
+
+      {
+        std::stringstream s;
+        s << "--input-fd";
+        argv_vec.push_back(s.str());
+
+        std::stringstream s2;
+        s2 << params.input_fd;
+        argv_vec.push_back(s2.str());
+
+        argv << " --input-fd" << " " << params.input_fd;
+      }
+
+      {
+        std::stringstream s;
+        s << "--output-fd";
+        argv_vec.push_back(s.str());
+
+        std::stringstream s2;
+        s2 << params.output_fd;
+        argv_vec.push_back(s2.str());
+
+        argv << " --output-fd" << " " << params.output_fd;
+      }
+
+
+      if (params.use_sockets) {
+        std::stringstream s;
+        s << "--use-sockets";
+        argv_vec.push_back(s.str());
+
+        argv << " --use-sockets";
+      }
+
+      if (WOULD_LOG(VERBOSE)) {
+        std::stringstream s;
+        s << "--verbose";
+        argv_vec.push_back(s.str());
+
+        argv << " --verbose";
+      }
+
+      std::unique_ptr<ArgString[]> argv_ptr = common::VecToArgv(kCommandFileName, argv_vec);
+
+      LOG(DEBUG) << "fork+exec: " << kCommandFileName << " "
+                 << argv.str();
+      execve(kCommandFileName, (char **)argv_ptr.get(), /*envp*/nullptr);
+      // This should never return.
+      _exit(EXIT_FAILURE);
+    }
+
+    DCHECK(false);
+    return false;
+  }
+
+  // TODO: Not very useful since this can never return 'true'
+  // -> in the child we would've already execd which loses all this code.
+  bool IsDaemon() {
+    // In the child the pid is always 0.
+    return child_ > 0;
+  }
+
+  bool Main(PrefetcherForkParameters params) {
+    LOG(VERBOSE) << "PrefetcherDaemon::Main " << params;
+
+    CommandParser command_parser{params};
+
+    Command next_command{};
+
+    std::vector<Command> many_commands;
+
+    // Ensure alogd is pre-initialized before installing minijail.
+    LOG(DEBUG) << "Installing minijail";
+
+    // Install seccomp filter using libminijail.
+    if (kInstallMiniJail) {
+      MiniJail();
+    }
+
+    while (true) {
+      bool eof = false;
+
+      if (params.use_sockets) {
+        // use recvmsg(2). supports receiving FDs.
+        many_commands = command_parser.ParseSocketCommands(/*out*/eof);
+      } else {
+        // use read(2). does not support receiving FDs.
+        many_commands = command_parser.ParseCommands(/*out*/eof);
+      }
+
+      if (eof) {
+        LOG(WARNING) << "PrefetcherDaemon got EOF, terminating";
+        return true;
+      }
+
+      for (auto& command : many_commands) {
+        if (LogVerboseIpc()) {
+          LOG(VERBOSE) << "PrefetcherDaemon got command: " << command;
+        }
+
+        if (command.choice == CommandChoice::kExit) {
+          LOG(DEBUG) << "PrefetcherDaemon got kExit command, terminating";
+          return true;
+        }
+
+        if (!ReceiveCommand(command)) {
+          // LOG(WARNING) << "PrefetcherDaemon command processing failure: " << command;
+        }
+
+        // ReceiveCommand should dup to keep the FD. Avoid leaks.
+        if (command.fd.has_value()) {
+          close(*command.fd);
+        }
+      }
+    }
+
+    LOG(VERBOSE) << "PrefetcherDaemon::Main got exit, terminating";
+
+    return true;
+    // Terminate.
+  }
+
+  Impl(PrefetcherDaemon* daemon) {
+    session_manager_ = SessionManager::CreateManager(SessionKind::kInProcessDirect);
+    DCHECK(session_manager_ != nullptr);
+  };
+
+  ~Impl() {
+    // Don't do anything if we never called 'StartViaFork'
+    if (forked_) {
+      if (!IsDaemon()) {
+        int status;
+        waitpid(child_, /*out*/&status, /*options*/0);
+      } else {
+        LOG(WARNING) << "execve should have avoided this path";
+        // DCHECK(false) << "not possible because the execve would avoid this path";
+      }
+    }
+  }
+
+  bool SendCommand(const Command& command) {
+    // Only parent is the sender.
+    DCHECK(forked_);
+    //DCHECK(!IsDaemon());
+
+    char buf[1024];
+    size_t stream_size;
+    if (!command.Write(buf, sizeof(buf), /*out*/&stream_size)) {
+      PLOG(ERROR) << "Failed to serialize command: " << command;
+      return false;
+    }
+
+    if (LogVerboseIpc()) {
+      LOG(VERBOSE) << "pre-write(fd=" << pipefd_write_ << ", buf=" << buf
+                   << ", size=" << stream_size<< ")";
+    }
+
+    if (params_.use_sockets) {
+      /* iov contains the normal message (Command) */
+      struct iovec iov;
+      memset(&iov, 0, sizeof(iov));
+      iov.iov_base = &buf[0];
+      iov.iov_len = stream_size;
+
+      struct msghdr msg;
+      memset(&msg, 0, sizeof(msg));
+
+      /* point to iov to transmit */
+      msg.msg_iov = &iov;
+      msg.msg_iovlen = 1;
+
+      /* no dest address; socket is connected */
+      msg.msg_name = nullptr;
+      msg.msg_namelen = 0;
+
+      // append a CMSG with SCM_RIGHTS if we have an FD.
+      if (command.fd.has_value()) {
+          union {
+            struct cmsghdr cmh;
+            char   control[CMSG_SPACE(sizeof(int))]; /* sized to hold an fd (int) */
+          } control_un;
+          memset(&control_un, 0, sizeof(control_un));
+
+          msg.msg_control = &control_un.control[0];
+          msg.msg_controllen = sizeof(control_un.control);
+
+          struct cmsghdr *hp;
+          hp = CMSG_FIRSTHDR(&msg);
+          hp->cmsg_len = CMSG_LEN(sizeof(int));
+          hp->cmsg_level = SOL_SOCKET;
+          hp->cmsg_type = SCM_RIGHTS;
+          *((int *) CMSG_DATA(hp)) = *(command.fd);
+
+          DCHECK(command.RequiresFd()) << command;
+
+          if (LogVerboseIpc()) {
+            LOG(VERBOSE) << "append FD to sendmsg: " << *(command.fd);
+          }
+      }
+
+      // TODO: add CMSG for the FD passage.
+
+      if (TEMP_FAILURE_RETRY(sendmsg(pipefd_write_, &msg, /*flags*/0)) < 0) {
+        PLOG(ERROR) << "Failed to sendmsg command: " << command;
+        return false;
+      }
+    } else {
+      if (TEMP_FAILURE_RETRY(write(pipefd_write_, buf, stream_size)) < 0) {
+        PLOG(ERROR) << "Failed to write command: " << command;
+        return false;
+      }
+    }
+
+    if (LogVerboseIpc()) {
+      LOG(VERBOSE) << "write(fd=" << pipefd_write_ << ", buf=" << buf
+                   << ", size=" << stream_size<< ")";
+    }
+
+    // TODO: also read the reply?
+    return true;
+  }
+
+  bool ReceiveCommand(const Command& command) {
+    // Only child is the command receiver.
+    // DCHECK(IsDaemon());
+
+    switch (command.choice) {
+      case CommandChoice::kRegisterFilePath: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+
+        if (!session) {
+          LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command;
+          return false;
+        }
+
+        CHECK(command.file_path.has_value()) << command;
+        return session->RegisterFilePath(command.id, *command.file_path);
+      }
+      case CommandChoice::kUnregisterFilePath: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+
+        if (!session) {
+          LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command;
+          return false;
+        }
+
+        return session->UnregisterFilePath(command.id);
+      }
+      case CommandChoice::kReadAhead: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+
+        if (!session) {
+          LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command;
+          return false;
+        }
+
+        return session->ReadAhead(command.id, command.read_ahead_kind, command.length, command.offset);
+      }
+      // TODO: unreadahead
+      case CommandChoice::kExit: {
+        LOG(WARNING) << "kExit should be handled earlier.";
+        return true;
+      }
+      case CommandChoice::kCreateSession: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+        if (session != nullptr) {
+          LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command;
+          return false;
+        }
+        CHECK(command.file_path.has_value()) << command;
+        if (session_manager_->CreateSession(command.session_id, /*description*/*command.file_path)
+                == nullptr) {
+          LOG(ERROR) << "ReceiveCommand: Failure to kCreateSession: " << command;
+          return false;
+        }
+        return true;
+      }
+      case CommandChoice::kDestroySession: {
+        if (!session_manager_->DestroySession(command.session_id)) {
+          LOG(ERROR) << "ReceiveCommand: Failure to kDestroySession: " << command;
+          return false;
+        }
+        return true;
+      }
+      case CommandChoice::kDumpSession: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+
+        if (!session) {
+          LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command;
+          return false;
+        }
+
+        // TODO: Consider doing dumpsys support somehow?
+        session->Dump(LOG_STREAM(DEBUG), /*multiline*/true);
+        return true;
+      }
+      case CommandChoice::kDumpEverything: {
+        session_manager_->Dump(LOG_STREAM(DEBUG), /*multiline*/true);
+        break;
+      }
+      case CommandChoice::kCreateFdSession: {
+        std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id);
+        if (session != nullptr) {
+          LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command;
+          return false;
+        }
+        CHECK(command.file_path.has_value()) << command;
+        CHECK(command.fd.has_value()) << command;
+
+        LOG(VERBOSE) << "ReceiveCommand: kCreateFdSession fd=" << *(command.fd);
+
+        // TODO: Maybe use CreateFdSession instead?
+        session =
+            session_manager_->CreateSession(command.session_id,
+                                            /*description*/*command.file_path,
+                                            command.fd.value());
+        if (session == nullptr) {
+          LOG(ERROR) << "ReceiveCommand: Failure to kCreateFdSession: " << command;
+          return false;
+        }
+
+        return session->ProcessFd(*command.fd);
+      }
+    }
+
+    return true;
+  }
+
+  pid_t child_;
+  bool forked_;
+  int pipefd_read_;
+  int pipefd_write_;
+  PrefetcherForkParameters params_;
+  // do not ever use an indirect session manager here, as it would cause a lifetime cycle.
+  std::unique_ptr<SessionManager> session_manager_; // direct only.
+};
+
+PrefetcherDaemon::PrefetcherDaemon()
+  : impl_{new Impl{this}} {
+  LOG(VERBOSE) << "PrefetcherDaemon() constructor";
+}
+
+bool PrefetcherDaemon::StartViaFork(PrefetcherForkParameters params) {
+  return impl_->StartViaFork(std::move(params));
+}
+
+
+std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartPipesViaFork() {
+  return impl_->StartPipesViaFork();
+}
+
+std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartSocketViaFork() {
+  return impl_->StartSocketViaFork();
+}
+
+bool PrefetcherDaemon::Main(PrefetcherForkParameters params) {
+  return impl_->Main(params);
+}
+
+bool PrefetcherDaemon::SendCommand(const Command& command) {
+  return impl_->SendCommand(command);
+}
+
+PrefetcherDaemon::~PrefetcherDaemon() {
+  // required for unique_ptr for incomplete types.
+}
+
+}  // namespace iorap::prefetcher
diff --git a/src/prefetcher/prefetcher_daemon.h b/src/prefetcher/prefetcher_daemon.h
new file mode 100644
index 0000000..693f871
--- /dev/null
+++ b/src/prefetcher/prefetcher_daemon.h
@@ -0,0 +1,130 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PREFETCHER_DAEMON_H_
+#define PREFETCHER_DAEMON_H_
+
+#include "prefetcher/session_manager.h"
+
+#include <memory>
+#include <optional>
+#include <ostream>
+
+namespace iorap {
+namespace prefetcher {
+
+struct PrefetcherForkParameters {
+  int input_fd;
+  int output_fd;
+  bool use_sockets;  // use the socket path instead of simpler read/write path.
+  bool format_text;  // true=>text, false=>binary
+};
+
+inline std::ostream& operator<<(std::ostream& os, const PrefetcherForkParameters& p) {
+  os << "PrefetcherForkParameters{";
+  os << "input_fd=" << p.input_fd << ",";
+  os << "output_fd=" << p.output_fd << ",";
+  os << "format_text=" << p.format_text << ",";
+  os << "use_sockets=" << p.use_sockets << ",";
+  os << "}";
+  return os;
+}
+
+
+#ifndef READ_AHEAD_KIND
+enum class ReadAheadKind : uint32_t {
+  kFadvise = 0,
+  kMmapLocked = 1,
+  kMlock = 2,
+};
+#define READ_AHEAD_KIND 1
+#endif
+
+std::ostream& operator<<(std::ostream& os, ReadAheadKind k);
+
+enum class CommandChoice : uint32_t {
+  kRegisterFilePath,   // kRegisterFilePath <sid:uint32> <id:uint32> <path:c-string>
+  kUnregisterFilePath, // kUnregisterFilePath <sid:uint32> <id:uint32>
+  kReadAhead,          // kReadAhead <sid:uint32> <id:uint32> <kind:uint32_t> <length:uint64> <offset:uint64>
+  kExit,               // kExit
+  kCreateSession,      // kCreateSession <sid:uint32> <description:c-string>
+  kDestroySession,     // kDestroySession <sid:uint32>
+  kDumpSession,        // kDumpSession <sid:uint32>
+  kDumpEverything,     // kDumpEverything
+  kCreateFdSession,    // kCreateFdSession $CMSG{<fd:int>} <sid:uint32> <description:c-string>
+};
+
+struct Command {
+  CommandChoice choice;
+  uint32_t session_id;
+  uint32_t id;  // file_path_id
+  std::optional<std::string> file_path;  // required for choice=kRegisterFilePath.
+  // also serves as the description for choice=kCreateSession
+
+  // choice=kReadAhead
+  ReadAheadKind read_ahead_kind;
+  uint64_t length;
+  uint64_t offset;
+
+  std::optional<int> fd;  // only valid in kCreateFdSession.
+
+  // Deserialize from a char buffer.
+  // This can only fail if buf_size is too small.
+  static std::optional<Command> Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes);
+  // Serialize to a char buffer.
+  // This can only fail if the buf_size is too small.
+  bool Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const;
+
+  bool RequiresFd() const {
+    return choice == CommandChoice::kCreateFdSession;
+  }
+};
+
+std::ostream& operator<<(std::ostream& os, const Command& command);
+
+class PrefetcherDaemon {
+ public:
+  PrefetcherDaemon();
+  ~PrefetcherDaemon();
+
+  // Asynchronously launch a new fork.
+  //
+  // The destructor will waitpid automatically on the child process.
+  bool StartViaFork(PrefetcherForkParameters params);
+
+  // Launch a new fork , returning the pipes as input/output fds.
+  std::optional<PrefetcherForkParameters> StartPipesViaFork();
+
+  // Launch a new fork , returning the socket pair as input/output fds.
+  std::optional<PrefetcherForkParameters> StartSocketViaFork();
+
+  // Execute the main code in-process.
+  //
+  // Intended as the execve target.
+  bool Main(PrefetcherForkParameters params);
+
+  // Send a command via IPC.
+  // The caller must be the parent process after using StartViaFork.
+  bool SendCommand(const Command& command);
+
+ private:
+  class Impl;
+  std::unique_ptr<PrefetcherDaemon::Impl> impl_;
+};
+
+}  // namespace prefetcher
+}  // namespace iorap
+
+#endif
+
diff --git a/src/prefetcher/read_ahead.cc b/src/prefetcher/read_ahead.cc
new file mode 100644
index 0000000..164f2ff
--- /dev/null
+++ b/src/prefetcher/read_ahead.cc
@@ -0,0 +1,435 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "read_ahead.h"
+
+#include "common/trace.h"
+#include "prefetcher/session_manager.h"
+#include "prefetcher/session.h"
+#include "prefetcher/task_id.h"
+#include "serialize/arena_ptr.h"
+#include "serialize/protobuf_io.h"
+
+#include <android-base/chrono_utils.h>
+#include <android-base/logging.h>
+#include <android-base/scopeguard.h>
+#include <android-base/properties.h>
+#include <android-base/unique_fd.h>
+#include <cutils/trace.h>
+#include <deque>
+#include <fcntl.h>
+#include <functional>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unordered_map>
+#include <utils/Printer.h>
+
+namespace iorap {
+namespace prefetcher {
+
+enum class PrefetchStrategy {
+  kFadvise = 0,
+  kMmapLocked = 1,
+  kMlock = 2,
+};
+
+std::ostream& operator<<(std::ostream& os, PrefetchStrategy ps) {
+  switch (ps) {
+    case PrefetchStrategy::kFadvise:
+      os << "fadvise";
+      break;
+    case PrefetchStrategy::kMmapLocked:
+      os << "mmap";
+      break;
+    case PrefetchStrategy::kMlock:
+      os << "mlock";
+      break;
+    default:
+      os << "<invalid>";
+  }
+  return os;
+}
+
+static constexpr PrefetchStrategy kPrefetchStrategy = PrefetchStrategy::kFadvise;
+
+static PrefetchStrategy GetPrefetchStrategy() {
+  PrefetchStrategy strat = PrefetchStrategy::kFadvise;
+
+  std::string prefetch_env =
+      ::android::base::GetProperty("iorapd.readahead.strategy", /*default*/"");
+
+  if (prefetch_env == "") {
+    LOG(VERBOSE)
+        << "ReadAhead strategy defaulted. Did you want to set iorapd.readahead.strategy ?";
+  } else if (prefetch_env == "mmap") {
+    strat = PrefetchStrategy::kMmapLocked;
+    LOG(VERBOSE) << "ReadAhead strategy: kMmapLocked";
+  } else if (prefetch_env == "mlock") {
+    strat = PrefetchStrategy::kMlock;
+    LOG(VERBOSE) << "ReadAhead strategy: kMlock";
+  } else if (prefetch_env == "fadvise") {
+    strat = PrefetchStrategy::kFadvise;
+    LOG(VERBOSE) << "ReadAhead strategy: kFadvise";
+  } else {
+    LOG(WARNING) << "Unknown iorapd.readahead.strategy: " << prefetch_env << ", ignoring";
+  }
+
+  return strat;
+}
+
+struct TaskData {
+  TaskId task_id;   // also the session ID.
+
+  size_t SessionId() const {
+    if (session != nullptr) {
+      DCHECK_EQ(session->SessionId(), task_id.id);
+    }
+
+    // good enough to be used as the session ID. Task IDs are always monotonically increasing.
+    return task_id.id;
+  }
+
+  std::shared_ptr<Session> session;
+  int32_t trace_cookie;  // async trace cookie in BeginTask/FinishTask.
+};
+
+// Remember the last 5 files being prefetched.
+static constexpr size_t kRecentDataCount = 5;
+
+struct RecentData {
+  TaskId task_id;
+  size_t file_lengths_sum;
+};
+
+struct RecentDataKeeper {
+  std::deque<RecentData> recents_;
+  std::mutex mutex_;
+
+  void RecordRecent(TaskId task_id, size_t file_lengths_sum) {
+    std::lock_guard<std::mutex> guard{mutex_};
+
+    while (recents_.size() > kRecentDataCount) {
+      recents_.pop_front();
+    }
+    recents_.push_back(RecentData{std::move(task_id), file_lengths_sum});
+  }
+
+  void Dump(/*borrow*/::android::Printer& printer) {
+    bool locked = mutex_.try_lock();
+
+    printer.printFormatLine("Recent prefetches:");
+    if (!locked) {
+      printer.printLine("""""  (possible deadlock)");
+    }
+
+    for (const RecentData& data : recents_) {
+      printer.printFormatLine("  %s", data.task_id.path.c_str());
+      printer.printFormatLine("    Task ID: %zu", data.task_id.id);
+      printer.printFormatLine("    Bytes count: %zu", data.file_lengths_sum);
+    }
+
+    if (recents_.empty()) {
+      printer.printFormatLine("  (None)");
+    }
+
+    printer.printLine("");
+
+    if (locked) {
+      mutex_.unlock();
+    }
+  }
+};
+
+struct ReadAhead::Impl {
+  Impl(bool use_sockets) {
+    // Flip this property to test in-process vs out-of-process for the prefetcher code.
+    bool out_of_process =
+        ::android::base::GetBoolProperty("iorapd.readahead.out_of_process", /*default*/true);
+
+    SessionKind session_kind =
+        out_of_process ? SessionKind::kOutOfProcessIpc : SessionKind::kInProcessDirect;
+
+    if (use_sockets) {
+      session_kind = SessionKind::kOutOfProcessSocket;
+    }
+
+    session_manager_ = SessionManager::CreateManager(session_kind);
+    session_kind_ = session_kind;
+  }
+
+  std::unique_ptr<SessionManager> session_manager_;
+  SessionKind session_kind_;
+  std::unordered_map<size_t /*task index*/, TaskData> read_ahead_file_map_;
+  static RecentDataKeeper recent_data_keeper_;
+  int32_t trace_cookie_{0};
+
+  bool UseSockets() const {
+    return session_kind_ == SessionKind::kOutOfProcessSocket;
+  }
+};
+
+RecentDataKeeper ReadAhead::Impl::recent_data_keeper_{};
+
+ReadAhead::ReadAhead() : ReadAhead(/*use_sockets*/false) {
+}
+
+ReadAhead::ReadAhead(bool use_sockets) : impl_(new Impl(use_sockets)) {
+}
+
+ReadAhead::~ReadAhead() {}
+
+static bool PerformReadAhead(std::shared_ptr<Session> session, size_t path_id, ReadAheadKind kind, size_t length, size_t offset) {
+  return session->ReadAhead(path_id, kind, length, offset);
+}
+
+void ReadAhead::FinishTask(const TaskId& id) {
+  auto it = impl_->read_ahead_file_map_.find(id.id);
+  if (it == impl_->read_ahead_file_map_.end()) {
+    LOG(DEBUG) << "Could not find any TaskData for " << id;
+    return;
+  }
+
+  TaskData& task_data = it->second;
+  atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER,
+                   "ReadAhead Task Scope (for File Descriptors)",
+                   task_data.trace_cookie);
+
+  auto deleter = [&]() {
+    impl_->read_ahead_file_map_.erase(it);
+  };
+  auto scope_guard = android::base::make_scope_guard(deleter);
+  (void)scope_guard;
+
+  LOG(VERBOSE) << "ReadAhead (Finish)";
+
+  if (!impl_->session_manager_->DestroySession(task_data.SessionId())) {
+    LOG(WARNING) << "ReadAhead: Failed to destroy Session " << task_data.SessionId();
+  }
+}
+
+void ReadAhead::BeginTaskForSockets(const TaskId& id, int32_t trace_cookie) {
+  LOG(VERBOSE) << "BeginTaskForSockets: " << id;
+
+  // TODO: atrace.
+  android::base::Timer timer{};
+  android::base::Timer open_timer{};
+
+  int trace_file_fd_raw =
+      TEMP_FAILURE_RETRY(open(id.path.c_str(), /*flags*/O_RDONLY));
+
+  android::base::unique_fd trace_file_fd{trace_file_fd_raw};
+
+  if (!trace_file_fd.ok()) {
+    PLOG(ERROR) << "ReadAhead failed to open trace file: " << id.path;
+    return;
+  }
+
+  TaskData task_data;
+  task_data.task_id = id;
+  task_data.trace_cookie = trace_cookie;
+
+  std::shared_ptr<Session> session =
+      impl_->session_manager_->CreateSession(task_data.SessionId(),
+                                             /*description*/id.path,
+                                             trace_file_fd.get());
+  task_data.session = session;
+  CHECK(session != nullptr);
+
+  task_data.trace_cookie = ++trace_cookie;
+
+  // TODO: maybe getprop and a single line by default?
+  session->Dump(LOG_STREAM(INFO), /*multiline*/true);
+
+  impl_->read_ahead_file_map_[id.id] = std::move(task_data);
+  // FinishTask is identical, as it just destroys the session.
+}
+
+void ReadAhead::BeginTask(const TaskId& id) {
+  {
+    struct timeval now;
+    gettimeofday(&now, nullptr);
+
+    uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec);
+    LOG(DEBUG) << "BeginTask: beginning usec: " << now_usec;
+  }
+
+  int32_t trace_cookie = ++impl_->trace_cookie_;
+  atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER,
+                     "ReadAhead Task Scope (for File Descriptors)",
+                     trace_cookie);
+
+  if (impl_->UseSockets()) {
+    BeginTaskForSockets(id, trace_cookie);
+    return;
+  }
+
+  LOG(VERBOSE) << "BeginTask: " << id;
+
+  // TODO: atrace.
+  android::base::Timer timer{};
+
+  // TODO: refactor this code with SessionDirect::ProcessFd ?
+  TaskData task_data;
+  task_data.task_id = id;
+  task_data.trace_cookie = trace_cookie;
+
+  ScopedFormatTrace atrace_begin_task(ATRACE_TAG_ACTIVITY_MANAGER,
+                                      "ReadAhead::BeginTask %s",
+                                      id.path.c_str());
+
+  // Include CreateSession above the Protobuf deserialization so that we can include
+  // the 'total_duration' as part of the Session dump (relevant when we use IPC mode only).
+  std::shared_ptr<Session> session =
+      impl_->session_manager_->CreateSession(task_data.SessionId(),
+                                             /*description*/id.path);
+
+  android::base::Timer open_timer{};
+
+  // XX: Should we rename all the 'Create' to 'Make', or rename the 'Make' to 'Create' ?
+  // Unfortunately make_unique, make_shared, etc is the standard C++ terminology.
+  serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr =
+      serialize::ProtobufIO::Open(id.path);
+
+  if (trace_file_ptr == nullptr) {
+    // TODO: distinguish between missing trace (this is OK, most apps wont have one)
+    // and a bad error.
+    LOG(DEBUG) << "ReadAhead could not start, missing trace file? " << id.path;
+    return;
+  }
+
+  task_data.session = session;
+  CHECK(session != nullptr);
+
+  ReadAheadKind kind = static_cast<ReadAheadKind>(GetPrefetchStrategy());
+
+  // TODO: The "Task[Id]" should probably be the one owning the trace file.
+  // When the task is fully complete, the task can be deleted and the
+  // associated arenas can go with them.
+
+  // TODO: we should probably have the file entries all be relative
+  // to the package path?
+
+  // Open every file in the trace index.
+  const serialize::proto::TraceFileIndex& index = trace_file_ptr->index();
+
+  size_t count_entries = 0;
+  {
+    ScopedFormatTrace atrace_register_file_paths(ATRACE_TAG_ACTIVITY_MANAGER,
+                                                 "ReadAhead::RegisterFilePaths %s",
+                                                 id.path.c_str());
+    for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) {
+      LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name();
+
+      if (index_entry.id() < 0) {
+        LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: "
+                     << index_entry.id();
+        continue;
+      }
+
+      size_t path_id = index_entry.id();
+      const auto& path_file_name = index_entry.file_name();
+
+      if (!session->RegisterFilePath(path_id, path_file_name)) {
+        LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name;
+      } else {
+        ++count_entries;
+      }
+    }
+  }
+  LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths";
+  std::chrono::milliseconds open_duration_ms = open_timer.duration();
+
+  LOG(DEBUG) << "ReadAhead: Opened file&headers in " << open_duration_ms.count() << "ms";
+
+  size_t length_sum = 0;
+  size_t entry_offset = 0;
+  {
+    ScopedFormatTrace atrace_perform_read_ahead(ATRACE_TAG_ACTIVITY_MANAGER,
+                                                "ReadAhead::PerformReadAhead entries=%zu, path=%s",
+                                                count_entries,
+                                                id.path.c_str());
+
+    // Go through every trace entry and readahead every (file,offset,len) tuple.
+    const serialize::proto::TraceFileList& file_list = trace_file_ptr->list();
+    for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) {
+      ++entry_offset;
+
+      if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) {
+        LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: "
+                     << "index_id=" << file_entry.index_id() << ", skipping";
+        continue;
+      }
+
+      // Attempt to perform readahead. This can generate more warnings dynamically.
+      if (!PerformReadAhead(session, file_entry.index_id(), kind, file_entry.file_length(), file_entry.file_offset())) {
+        // TODO: Do we need below at all? The always-on Dump already prints a % of failed entries.
+        // LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ " << (entry_offset - 1);
+      }
+
+      length_sum += static_cast<size_t>(file_entry.file_length());
+    }
+  }
+
+  {
+    ScopedFormatTrace atrace_session_dump(ATRACE_TAG_ACTIVITY_MANAGER,
+                                          "ReadAhead Session Dump entries=%zu",
+                                          entry_offset);
+    // TODO: maybe getprop and a single line by default?
+    session->Dump(LOG_STREAM(INFO), /*multiline*/true);
+  }
+
+  atrace_int(ATRACE_TAG_ACTIVITY_MANAGER,
+             "ReadAhead Bytes Length",
+             static_cast<int32_t>(length_sum));
+
+  impl_->read_ahead_file_map_[id.id] = std::move(task_data);
+
+  ReadAhead::Impl::recent_data_keeper_.RecordRecent(id, length_sum);
+}
+
+void ReadAhead::Dump(::android::Printer& printer) {
+  ReadAhead::Impl::recent_data_keeper_.Dump(printer);
+}
+
+std::optional<size_t> ReadAhead::PrefetchSizeInBytes(const std::string& file_path) {
+  serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr =
+      serialize::ProtobufIO::Open(file_path);
+
+  if (trace_file_ptr == nullptr) {
+    LOG(WARNING) << "PrefetchSizeInBytes: bad file at " << file_path;
+    return std::nullopt;
+  }
+
+  size_t length_sum = 0;
+  const serialize::proto::TraceFileList& file_list = trace_file_ptr->list();
+  for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) {
+
+    if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) {
+      LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: "
+                   << "index_id=" << file_entry.index_id() << ", skipping";
+      continue;
+    }
+
+    length_sum += static_cast<size_t>(file_entry.file_length());
+  }
+
+  return length_sum;
+}
+
+}  // namespace prefetcher
+}  // namespace iorap
+
diff --git a/src/prefetcher/read_ahead.h b/src/prefetcher/read_ahead.h
new file mode 100644
index 0000000..afc6ec8
--- /dev/null
+++ b/src/prefetcher/read_ahead.h
@@ -0,0 +1,63 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PREFETCHER_READAHEAD_H_
+#define PREFETCHER_READAHEAD_H_
+
+#include <memory>
+#include <optional>
+
+namespace android {
+class Printer;
+}  // namespace android
+
+namespace iorap {
+namespace prefetcher {
+
+struct TaskId;
+struct ReadAheadFileEntry;
+
+// Manage I/O readahead for a task.
+class ReadAhead {
+  struct Impl;
+ public:
+  // Process a task *now*. Currently will block until all readaheads have been
+  // issued for all entries in that task.
+  //
+  // Any memory mapped or file descriptors opened as a side effect must be
+  // cleaned up with #FinishTask.
+  void BeginTask(const TaskId& id);
+  // Complete a task, releasing any memory/file descriptors associated with it.
+  void FinishTask(const TaskId& id);
+
+  static void Dump(/*borrow*/::android::Printer& printer);
+
+  // Calculate the sum of file_lengths. Returns nullopt if the file path does not
+  // point to a valid compiled TraceFile.
+  static std::optional<size_t> PrefetchSizeInBytes(const std::string& file_path);
+
+  ReadAhead(bool use_sockets);
+
+  ReadAhead();
+  ~ReadAhead();
+ private:
+  void BeginTaskForSockets(const TaskId& id, int32_t trace_cookie);
+  std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace prefetcher
+}  // namespace iorap
+
+#endif
+
diff --git a/src/prefetcher/session.cc b/src/prefetcher/session.cc
new file mode 100644
index 0000000..e998580
--- /dev/null
+++ b/src/prefetcher/session.cc
@@ -0,0 +1,724 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "session.h"
+
+#include "prefetcher/prefetcher_daemon.h"
+#include "prefetcher/task_id.h"
+#include "serialize/arena_ptr.h"
+#include "serialize/protobuf_io.h"
+
+#include <android-base/logging.h>
+#include <android-base/properties.h>
+#include <android-base/unique_fd.h>
+#include <fcntl.h>
+#include <functional>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unordered_map>
+
+namespace iorap {
+namespace prefetcher {
+
+// Print per-entry details even if successful. Default-off, too spammy.
+static constexpr bool kLogVerboseReadAhead = false;
+
+std::ostream& operator<<(std::ostream& os, const Session& session) {
+  session.Dump(os, /*multiline*/false);
+  return os;
+}
+
+Session::Session() {
+}
+
+void SessionBase::Dump(std::ostream& os, bool multiline) const {
+  if (!multiline) {
+    os << "Session{";
+    os << "session_id=" << SessionId();
+    os << "}";
+    return;
+  } else {
+    os << "Session (id=" << SessionId() << ")" << std::endl;
+    return;
+  }
+}
+
+SessionBase::SessionBase(size_t session_id, std::string description)
+  : session_id_{session_id}, description_{description} {
+}
+
+std::optional<std::string_view> SessionBase::GetFilePath(size_t path_id) const {
+  auto it = path_map_.find(path_id);
+  if (it != path_map_.end()) {
+    return {it->second};
+  } else {
+    return std::nullopt;
+  }
+}
+
+bool SessionBase::RemoveFilePath(size_t path_id) {
+  auto it = path_map_.find(path_id);
+  if (it != path_map_.end()) {
+    path_map_.erase(it);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+bool SessionBase::InsertFilePath(size_t path_id, std::string file_path) {
+  path_map_.insert({path_id, std::move(file_path)});
+  return true;
+}
+
+bool SessionBase::ProcessFd(int fd) {
+  // Only SessionDirect has an implementation of this.
+  // TODO: Maybe add a CommandChoice::kProcessFd ? instead of kCreateFdSession?
+  LOG(FATAL) << "SessionBase::ProcessFd is not implemented";
+
+  return false;
+}
+
+//
+// Direct
+//
+
+std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry) {
+  os << "Entry{";
+  os << "path_id=" << entry.path_id << ",";
+  os << "kind=" << static_cast<int>(entry.kind) << ",";
+  os << "length=" << entry.length << ",";
+  os << "offset=" << entry.offset << ",";
+  os << "}";
+
+  return os;
+}
+
+// Just in case the failures are slowing down performance, turn them off.
+// static constexpr bool kLogFailures = true;
+static constexpr bool kLogFailures = false;
+
+bool SessionDirect::RegisterFilePath(size_t path_id, std::string_view file_path) {
+  std::string file_path_str{file_path};  // no c_str for string_view.
+
+  auto fd = TEMP_FAILURE_RETRY(open(file_path_str.c_str(), O_RDONLY));
+  if (fd < 0) {
+    if (kLogFailures) {
+      PLOG(ERROR) << "Failed to register file path: " << file_path << ", id=" << path_id
+                  << ", open(2) failed: ";
+    }
+    fd = android::base::unique_fd{};  // mark as 'bad' descriptor.
+  }
+
+  LOG(VERBOSE) << "RegisterFilePath path_id=" << path_id << ", file_path=" << file_path_str;
+
+  if (!InsertFilePath(path_id, std::move(file_path_str))) {
+    return false;
+  }
+
+  path_fd_map_.insert(std::make_pair(path_id, std::move(fd)));
+  DCHECK(entry_list_map_[path_id].empty());
+
+  return true;
+}
+
+
+bool SessionDirect::UnregisterFilePath(size_t path_id) {
+  if (!RemoveFilePath(path_id)) {
+    return false;
+  }
+
+  {  // Scoped FD reference lifetime.
+    auto maybe_fd = GetFdForPath(path_id);
+
+    DCHECK(*maybe_fd != nullptr);
+    const android::base::unique_fd& entry_fd = **maybe_fd;
+
+    auto list = entry_list_map_[path_id];
+
+    for (const EntryMapping& entry_mapping : list) {
+      ReadAheadKind kind = entry_mapping.entry.kind;
+
+      switch (kind) {
+        case ReadAheadKind::kFadvise:
+          // Nothing to do.
+          break;
+        case ReadAheadKind::kMmapLocked:
+        FALLTHROUGH_INTENDED;
+        case ReadAheadKind::kMlock:
+          // Don't do any erases in the unregister file path to avoid paying O(n^2) erase cost.
+          UnmapWithoutErase(entry_mapping);
+          break;
+      }
+    }
+  }
+
+  auto it = entry_list_map_.find(path_id);
+  auto end = entry_list_map_.end();
+  DCHECK(it != end);
+  entry_list_map_.erase(it);
+
+  // Close the FD for this file path.
+  auto fd_it = path_fd_map_.find(path_id);
+  DCHECK(fd_it != path_fd_map_.end());
+  path_fd_map_.erase(fd_it);
+
+  return true;
+}
+
+// Note: return a pointer because optional doesn't hold references directly.
+std::optional<android::base::unique_fd*> SessionDirect::GetFdForPath(size_t path_id) {
+  auto it = path_fd_map_.find(path_id);
+  if (it == path_fd_map_.end()) {
+    return std::nullopt;
+  } else {
+    return &it->second;
+  }
+}
+
+bool SessionDirect::ReadAhead(size_t path_id,
+                              ReadAheadKind kind,
+                              size_t length,
+                              size_t offset) {
+  // Take by-reference so we can mutate list at the end.
+  auto& list = entry_list_map_[path_id];
+
+  Entry entry{path_id, kind, length, offset};
+  EntryMapping entry_mapping{entry, /*address*/nullptr, /*success*/false};
+
+  bool success = true;
+
+  auto maybe_fd = GetFdForPath(path_id);
+  if (!maybe_fd) {
+    LOG(ERROR) << "SessionDirect: Failed to find FD for path_id=" << path_id;
+    return false;
+  }
+
+  DCHECK(*maybe_fd != nullptr);
+  const android::base::unique_fd& entry_fd = **maybe_fd;
+
+  std::optional<std::string_view> file_name_opt = GetFilePath(path_id);
+  DCHECK(file_name_opt.has_value());  // if one map has it, all maps have it.
+  std::string_view file_name = *file_name_opt;
+
+  if (!entry_fd.ok()) {
+    LOG(VERBOSE) << "SessionDirect: No file descriptor for (path_id=" << path_id << ") "
+                 << "path '" << file_name << "', failed to readahead entry.";
+    // Even failures get kept with success=false.
+    list.push_back(entry_mapping);
+    return false;
+  }
+
+  switch (kind) {
+    case ReadAheadKind::kFadvise:
+      if (posix_fadvise(entry_fd, offset, length, POSIX_FADV_WILLNEED) < 0) {
+        PLOG(ERROR) << "SessionDirect: Failed to fadvise entry " << file_name
+                    << ", offset=" << offset << ", length=" << length;
+        success = false;
+      }
+      break;
+    case ReadAheadKind::kMmapLocked:
+    FALLTHROUGH_INTENDED;
+    case ReadAheadKind::kMlock: {
+      const bool need_mlock = kind == ReadAheadKind::kMlock;
+
+      int flags = MAP_SHARED;
+      if (!need_mlock) {
+        // MAP_LOCKED is a best-effort to lock the page. it could still be
+        // paged in later at a fault.
+        flags |= MAP_LOCKED;
+      }
+
+      entry_mapping.address =
+        mmap(/*addr*/nullptr, length, PROT_READ, flags, entry_fd, offset);
+
+      if (entry_mapping.address == nullptr) {
+        PLOG(ERROR) << "SessionDirect: Failed to mmap entry " << file_name
+                    << ", offset=" << offset << ", length=" << length;
+        success = false;
+        break;
+      }
+
+      // Strong guarantee that page will be locked if mlock returns successfully.
+      if (need_mlock && mlock(entry_mapping.address, length) < 0) {
+        PLOG(ERROR) << "SessionDirect: Failed to mlock entry " << file_name
+                    << ", offset=" << offset << ", length=" << length;
+        // We already have a mapping address, so we should add it to the list.
+        // However this didn't succeed 100% because the lock failed, so return false later.
+        success = false;
+      }
+    }
+  }
+
+  // Keep track of success so we know in Dump() what the number of failed entry mappings were.
+  entry_mapping.success = success;
+
+  // Keep track of this so that we can clean it up later in UnreadAhead.
+  list.push_back(entry_mapping);
+
+  if (entry_mapping.success) {
+    if (kLogVerboseReadAhead) {
+      LOG(VERBOSE) << "SessionDirect: ReadAhead for " << entry_mapping.entry;
+    }
+  }  // else one of the errors above already did print.
+
+  return success;
+}
+
+bool SessionDirect::UnreadAhead(size_t path_id,
+                                ReadAheadKind kind,
+                                size_t length,
+                                size_t offset) {
+  Entry entry{path_id, kind, length, offset};
+
+  auto list = entry_list_map_[path_id];
+  if (list.empty()) {
+    return false;
+  }
+
+  std::optional<EntryMapping> entry_mapping;
+  size_t idx = 0;
+
+  for (size_t i = 0; i < list.size(); ++i) {
+    if (entry == list[i].entry) {
+      entry_mapping = list[i];
+      idx = 0;
+      break;
+    }
+  }
+
+  if (!entry_mapping) {
+    return false;
+  }
+
+  switch (kind) {
+    case ReadAheadKind::kFadvise:
+      // Nothing to do.
+      // TODO: maybe fadvise(RANDOM)?
+      return true;
+    case ReadAheadKind::kMmapLocked:
+    FALLTHROUGH_INTENDED;
+    case ReadAheadKind::kMlock:
+      UnmapWithoutErase(*entry_mapping);
+      return true;
+  }
+
+  list.erase(list.begin() + idx);
+
+  // FDs close only with UnregisterFilePath.
+  return true;
+}
+
+void SessionDirect::UnmapWithoutErase(const EntryMapping& entry_mapping) {
+  void* address = entry_mapping.address;
+  size_t length = entry_mapping.entry.length;
+
+  // munmap also unlocks. Do not need explicit munlock.
+  if (munmap(address, length) < 0) {
+    PLOG(WARNING) << "ReadAhead (Finish): Failed to munmap address: "
+                  << address << ", length: " << length;
+  }
+
+}
+
+bool SessionDirect::ProcessFd(int fd) {
+  // TODO: the path is advisory, but it would still be cleaner to pass it separately
+  const char* fd_path = SessionDescription().c_str();
+
+  android::base::Timer open_timer{};
+  android::base::Timer total_timer{};
+
+  serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr =
+      serialize::ProtobufIO::Open(fd, fd_path);
+
+  if (trace_file_ptr == nullptr) {
+    LOG(ERROR) << "SessionDirect::ProcessFd failed, corrupted protobuf format? " << fd_path;
+    return false;
+  }
+
+  // TODO: maybe make it part of a kProcessFd type of command?
+  ReadAheadKind kind = ReadAheadKind::kFadvise;
+
+  // TODO: The "Task[Id]" should probably be the one owning the trace file.
+  // When the task is fully complete, the task can be deleted and the
+  // associated arenas can go with them.
+
+  // TODO: we should probably have the file entries all be relative
+  // to the package path?
+
+  // Open every file in the trace index.
+  const serialize::proto::TraceFileIndex& index = trace_file_ptr->index();
+
+  size_t count_entries = 0;
+  for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) {
+    LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name();
+
+    if (index_entry.id() < 0) {
+      LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: "
+                   << index_entry.id();
+      continue;
+    }
+
+    size_t path_id = index_entry.id();
+    const auto& path_file_name = index_entry.file_name();
+
+    if (!this->RegisterFilePath(path_id, path_file_name)) {
+      LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name;
+      ++count_entries;
+    }
+  }
+  LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths";
+  std::chrono::milliseconds open_duration_ms = open_timer.duration();
+
+  LOG(DEBUG) << "ProcessFd: open+parsed headers in " << open_duration_ms.count() << "ms";
+
+  // Go through every trace entry and readahead every (file,offset,len) tuple.
+  size_t entry_offset = 0;
+  const serialize::proto::TraceFileList& file_list = trace_file_ptr->list();
+  for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) {
+    ++entry_offset;
+
+    if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) {
+      LOG(WARNING) << "ProcessFd entry negative file length or offset, illegal: "
+                   << "index_id=" << file_entry.index_id() << ", skipping";
+      continue;
+    }
+
+    // Attempt to perform readahead. This can generate more warnings dynamically.
+    if (!this->ReadAhead(file_entry.index_id(),
+                         kind,
+                         file_entry.file_length(),
+                         file_entry.file_offset())) {
+      if (kLogFailures) {
+        LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ "
+                     << (entry_offset - 1);
+      }
+    }
+  }
+
+  std::chrono::milliseconds total_duration_ms = total_timer.duration();
+  LOG(DEBUG) << "ProcessFd: total duration " << total_duration_ms.count() << "ms";
+
+  {
+    struct timeval now;
+    gettimeofday(&now, nullptr);
+
+    uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec);
+    LOG(DEBUG) << "ProcessFd: finishing usec: " << now_usec;
+  }
+
+  return true;
+}
+
+
+static bool IsDumpEveryEntry() {
+  // Set to 'true' to dump every single entry for debugging (multiline).
+  // Otherwise it only prints per-file-path summaries.
+  return ::android::base::GetBoolProperty("iorapd.readahead.dump_all", /*default*/false);
+}
+
+static bool IsDumpEveryPath() {
+  // Dump per-file-path (entry) stats. Defaults to on if the above property is on.
+  return ::android::base::GetBoolProperty("iorapd.readahead.dump_paths", /*default*/false);
+}
+
+void SessionDirect::Dump(std::ostream& os, bool multiline) const {
+  {
+    struct timeval now;
+    gettimeofday(&now, nullptr);
+
+    uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec);
+    LOG(DEBUG) << "SessionDirect::Dump: beginning usec: " << now_usec;
+  }
+
+  size_t path_count = entry_list_map_.size();
+
+  size_t read_ahead_entries = 0;
+  size_t read_ahead_bytes = 0;
+
+  size_t overall_entry_count = 0;
+  size_t overall_byte_count = 0;
+  for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) {
+    const auto& entry_mapping_list = it->second;
+
+    for (size_t j = 0; j < entry_mapping_list.size(); ++j) {
+      const EntryMapping& entry_mapping = entry_mapping_list[j];
+      const Entry& entry = entry_mapping.entry;
+
+      ++overall_entry_count;
+      overall_byte_count += entry.length;
+
+      if (entry_mapping.success) {
+        ++read_ahead_entries;
+        read_ahead_bytes += entry.length;
+      }
+    }
+  }
+
+  double overall_success_entry_rate =
+      read_ahead_entries * 100.0 / overall_entry_count;
+  double overall_success_byte_rate =
+      read_ahead_bytes * 100.0 / overall_byte_count;
+
+  size_t fd_count = path_fd_map_.size();
+  size_t good_fd_count = 0;
+  for (auto it = path_fd_map_.begin(); it != path_fd_map_.end(); ++it) {
+    if (it->second.ok()) {
+      ++good_fd_count;
+    }
+  }
+  double good_fd_rate = good_fd_count * 100.0 / fd_count;
+  // double bad_fd_rate = (fd_count - good_fd_count) * 1.0 / fd_count;
+
+  if (!multiline) {
+    os << "SessionDirect{";
+    os << "session_id=" << SessionId() << ",";
+
+    os << "file_paths=" << path_count << " (good: " << good_fd_rate << "),";
+    os << "read_ahead_entries=" << read_ahead_entries;
+    os << "(" << overall_success_entry_rate << "%),";
+    os << "read_ahead_bytes=" << read_ahead_bytes << "";
+    os << "(" << overall_success_byte_rate << "%),";
+    os << "timer=" << timer_.duration().count() << ",";
+
+    os << "}";
+    return;
+  } else {
+    // Always try to pay attention to these stats below.
+    // They can be signs of potential performance problems.
+    os << "Session Direct (id=" << SessionId() << ")" << std::endl;
+
+    os << "  Summary: " << std::endl;
+    os << "    Description = " << SessionDescription() << std::endl;
+    os << "    Duration = " << timer_.duration().count() << "ms" << std::endl;
+    os << "    Total File Paths=" << path_count << " (good: " << good_fd_rate << "%)" << std::endl;
+    os << "    Total Entries=" << overall_entry_count;
+    os << " (good: " << overall_success_entry_rate << "%)" << std::endl;
+    os << "    Total Bytes=" << overall_byte_count << "";
+    os << " (good: " << overall_success_byte_rate << "%)" << std::endl;
+    os << std::endl;
+
+    // Probably too spammy, but they could narrow down the issue for a problem in above stats.
+    if (!IsDumpEveryPath() && !IsDumpEveryEntry()) {
+      return;
+    }
+
+    for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) {
+      size_t path_id = it->first;
+      const auto& entry_mapping_list = it->second;
+
+      std::optional<std::string_view> file_path = GetFilePath(path_id);
+      os << "  File Path (id=" << path_id << "): ";
+      if (file_path.has_value()) {
+        os << "'" << *file_path << "'";
+      } else {
+        os << "(nullopt)";
+      }
+
+      auto fd_it = path_fd_map_.find(path_id);
+      os << ", FD=";
+      if (fd_it != path_fd_map_.end()) {
+        const android::base::unique_fd& fd = fd_it->second;
+        os << fd.get();  // -1 for failed fd.
+      } else {
+        os << "(none)";
+      }
+      os << std::endl;
+
+      size_t total_entries = entry_mapping_list.size();
+      size_t total_bytes = 0;
+
+      size_t local_read_ahead_entries = 0;
+      size_t local_read_ahead_bytes = 0;
+      for (size_t j = 0; j < entry_mapping_list.size(); ++j) {
+        const EntryMapping& entry_mapping = entry_mapping_list[j];
+        const Entry& entry = entry_mapping.entry;
+
+        total_bytes += entry.length;
+
+        // Sidenote: Bad FDs will have 100% failed mappings.
+        // Good FDs may sometimes have failed mappings.
+        if (entry_mapping.success) {
+          ++local_read_ahead_entries;
+          local_read_ahead_bytes += entry.length;
+        }
+
+        if (IsDumpEveryEntry()) {
+          os << "    Entry " << j << " details:" << std::endl;
+          os << "      " << entry << std::endl;
+          os << "      Mapping " << (entry_mapping.success ? "Succeeded" :  "Failed")
+             << ", Address " << entry_mapping.address << std::endl;
+        }
+      }
+
+      double entry_success_rate = local_read_ahead_entries * 100.0 / total_entries;
+      double bytes_success_rate = local_read_ahead_bytes * 100.0 / total_bytes;
+
+      double entry_failure_rate = (total_entries - local_read_ahead_entries) * 100.0 / total_entries;
+      double bytes_failure_rate = (total_bytes - local_read_ahead_bytes) * 100.0 / total_bytes;
+
+      os << "    Successful: Entries=" << local_read_ahead_entries
+         << " (" << entry_success_rate << "%)"
+         << ", Bytes=" << local_read_ahead_bytes
+         << " (" << bytes_success_rate << "%)"
+         << std::endl;
+      os << "    Failed: Entries=" << (total_entries - local_read_ahead_entries)
+         << " (" << entry_failure_rate << "%)"
+         << ", Bytes=" << (total_bytes - local_read_ahead_bytes)
+         << " (" << bytes_failure_rate << "%)"
+         << std::endl;
+      os << "    Total: Entries=" << total_entries
+         << ", Bytes=" << total_bytes
+         << std::endl;
+    }
+
+    return;
+  }
+}
+
+SessionDirect::~SessionDirect() {
+  for (auto it = entry_list_map_.begin(); it != entry_list_map_.end();) {
+    size_t path_id = it->first;
+
+    ++it; // the iterator is removed in the following Unregister method.
+    UnregisterFilePath(path_id);
+  }
+}
+
+//
+// Indirect
+//
+
+SessionIndirect::SessionIndirect(size_t session_id,
+                                 std::string description,
+                                 std::shared_ptr<PrefetcherDaemon> daemon,
+                                 bool send_command)
+    : SessionBase{session_id, description},
+      daemon_{daemon} {
+  // Don't do anything in e.g. subclasses.
+  if (!send_command) {
+    return;
+  }
+
+  Command cmd{};
+  cmd.choice = CommandChoice::kCreateSession;
+  cmd.session_id = session_id;
+  cmd.file_path = description;
+
+  LOG(VERBOSE) << "SessionIndirect: " << cmd;
+
+  if (!daemon_->SendCommand(cmd)) {
+    LOG(FATAL) << "SessionIndirect: Failure to create session " << session_id
+               << ", description: " << description;
+  }
+}
+
+SessionIndirect::~SessionIndirect() {
+  Command cmd{};
+  cmd.choice = CommandChoice::kDestroySession;
+  cmd.session_id = SessionId();
+
+  if (!daemon_->SendCommand(cmd)) {
+    LOG(WARNING) << "SessionIndirect: Failure to destroy session " << SessionId()
+                 << ", description: " << SessionDescription();
+  }
+}
+
+void SessionIndirect::Dump(std::ostream& os, bool multiline) const {
+  // SessionBase::Dump(os, multiline);
+  // TODO: does having the local dump do anything for us?
+
+  Command cmd{};
+  cmd.choice = CommandChoice::kDumpSession;
+  cmd.session_id = SessionId();
+
+  daemon_->SendCommand(cmd);
+}
+
+bool SessionIndirect::RegisterFilePath(size_t path_id, std::string_view file_path) {
+  Command cmd{};
+  cmd.choice = CommandChoice::kRegisterFilePath;
+  cmd.session_id = SessionId();
+  cmd.id = path_id;
+  cmd.file_path = file_path;
+
+  return daemon_->SendCommand(cmd);
+}
+
+bool SessionIndirect::UnregisterFilePath(size_t path_id) {
+  Command cmd{};
+  cmd.choice = CommandChoice::kUnregisterFilePath;
+  cmd.session_id = SessionId();
+  cmd.id = path_id;
+
+  return daemon_->SendCommand(cmd);
+}
+bool SessionIndirect::ReadAhead(size_t path_id,
+                                ReadAheadKind kind,
+                                size_t length,
+                                size_t offset) {
+  Command cmd{};
+  cmd.choice = CommandChoice::kReadAhead;
+  cmd.session_id = SessionId();
+  cmd.id = path_id;
+  cmd.read_ahead_kind = kind;
+  cmd.length = length;
+  cmd.offset = offset;
+
+  return daemon_->SendCommand(cmd);
+}
+
+bool SessionIndirect::UnreadAhead(size_t path_id,
+                                  ReadAheadKind kind,
+                                  size_t length,
+                                  size_t offset) {
+  LOG(WARNING) << "UnreadAhead: command not implemented yet";
+  return true;
+}
+
+//
+// IndirectSocket
+//
+
+SessionIndirectSocket::SessionIndirectSocket(size_t session_id,
+                                             int fd,
+                                             std::string description,
+                                             std::shared_ptr<PrefetcherDaemon> daemon)
+    : SessionIndirect{session_id, description, daemon, /*send_command*/false} {
+  // TODO: all of the WriteCommand etc in the daemon.
+  Command cmd{};
+  cmd.choice = CommandChoice::kCreateFdSession;
+  cmd.fd = fd;
+  cmd.session_id = session_id;
+  cmd.file_path = description;
+
+  LOG(VERBOSE) << "SessionIndirectSocket: " << cmd;
+
+  if (!daemon_->SendCommand(cmd)) {
+    LOG(FATAL) << "SessionIndirectSocket: Failure to create session " << session_id
+               << ", description: " << description;
+  }
+
+  // This goes into the SessionDirect ctor + SessionDirect::ProcessFd
+  // as implemented in PrefetcherDaemon::ReceiveCommand
+}
+
+SessionIndirectSocket::~SessionIndirectSocket() {
+}
+
+}  // namespace prefetcher
+}  // namespace iorap
diff --git a/src/prefetcher/session.h b/src/prefetcher/session.h
new file mode 100644
index 0000000..a4a9e6b
--- /dev/null
+++ b/src/prefetcher/session.h
@@ -0,0 +1,236 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PREFETCHER_SESSION_H_
+#define PREFETCHER_SESSION_H_
+
+#include <android-base/chrono_utils.h>
+#include <android-base/unique_fd.h>
+
+#include <optional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+namespace iorap {
+namespace prefetcher {
+
+#ifndef READ_AHEAD_KIND
+#define READ_AHEAD_KIND 1
+enum class ReadAheadKind : uint32_t {
+  kFadvise = 0,
+  kMmapLocked = 1,
+  kMlock = 2,
+};
+#endif
+
+class Session {
+ public:
+  virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) = 0;
+  virtual bool UnregisterFilePath(size_t path_id) = 0;
+
+  // Immediately perform a readahead now.
+  // Fadvise: the readahead will have been queued by the kernel.
+  // MmapLocked/Mlock: the memory is pinned by the requested process.
+  virtual bool ReadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0;
+
+  // Cancels a readahead previously done.
+  // The length/offset should match the call of ReadAhead.
+  virtual bool UnreadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0;
+
+  // Multi-line detailed dump, e.g. for dumpsys.
+  // Single-line summary dump, e.g. for logcat.
+  virtual void Dump(std::ostream& os, bool multiline) const = 0;
+
+  // Process the FD for kCreateFdSession.
+  // Assumes there's a compiled_trace.pb at the fd, calling this function
+  // will immediately process it and execute any read-aheads.
+  //
+  // FD is borrowed only for the duration of the function call.
+  virtual bool ProcessFd(int fd) = 0;
+
+  // Get the session ID associated with this session.
+  // Session IDs are distinct, they are not used for new sessions.
+  virtual size_t SessionId() const = 0;
+
+  // Get this session's description.
+  // Only useful for logging/dumping.
+  virtual const std::string& SessionDescription() const = 0;
+
+  // Implicitly unregister any remaining file paths.
+  // All read-aheads are also cancelled.
+  virtual ~Session() {}
+
+ protected:
+  Session();
+};
+
+// Single-line summary dump of Session.
+std::ostream& operator<<(std::ostream&os, const Session& session);
+
+class SessionBase : public Session {
+ public:
+  virtual void Dump(std::ostream& os, bool multiline) const override;
+  virtual ~SessionBase() {}
+
+  virtual size_t SessionId() const override {
+    return session_id_;
+  }
+
+  virtual const std::string& SessionDescription() const override {
+    return description_;
+  }
+
+  virtual bool ProcessFd(int fd) override;
+
+ protected:
+  SessionBase(size_t session_id, std::string description);
+  std::optional<std::string_view> GetFilePath(size_t path_id) const;
+  bool RemoveFilePath(size_t path_id);
+  bool InsertFilePath(size_t path_id, std::string file_path);
+
+  android::base::Timer timer_{};
+ private:
+  // Note: Store filename for easier debugging and for dumping.
+  std::unordered_map</*path_id*/size_t, std::string> path_map_;
+  size_t session_id_;
+  std::string description_;
+};
+
+// In-process session.
+class SessionDirect : public SessionBase {
+ public:
+  virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override;
+
+  virtual bool UnregisterFilePath(size_t path_id) override;
+  virtual bool ReadAhead(size_t path_id,
+                         ReadAheadKind kind,
+                         size_t length,
+                         size_t offset);
+
+  virtual bool UnreadAhead(size_t path_id,
+                           ReadAheadKind kind,
+                           size_t length,
+                           size_t offset) override;
+
+  virtual bool ProcessFd(int fd) override;
+
+  virtual void Dump(std::ostream& os, bool multiline) const override;
+
+  virtual ~SessionDirect();
+
+  SessionDirect(size_t session_id, std::string description)
+    : SessionBase{session_id, std::move(description)} {
+  }
+ protected:
+  struct Entry {
+    size_t path_id;
+    ReadAheadKind kind;
+    size_t length;
+    size_t offset;
+
+    constexpr bool operator==(const Entry& other) const {
+      return path_id == other.path_id &&
+        kind == other.kind &&
+        length == other.length &&
+        offset == other.offset;
+    }
+    constexpr bool operator!=(const Entry& other) const {
+      return !(*this == other);
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const Entry& entry);
+  };
+
+  struct EntryMapping {
+    Entry entry;
+    void* address;
+    bool success;
+  };
+
+  // bool EntryReadAhead(const Entry& entry) const;
+  // bool EntryUnReadAhead(const Entry& entry) const;
+
+  void UnmapWithoutErase(const EntryMapping& entry_mapping);
+  std::optional<android::base::unique_fd*> GetFdForPath(size_t path_id);
+
+ private:
+  std::unordered_map</*path_id*/size_t, std::vector<EntryMapping>> entry_list_map_;
+  std::unordered_map</*path_id*/size_t, android::base::unique_fd> path_fd_map_;
+
+ public:
+  friend std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry);
+};
+
+std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry);
+
+class PrefetcherDaemon;
+
+// Out-of-process session. Requires prefetcher daemon.
+class SessionIndirect : public SessionBase {
+ public:
+  virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override;
+
+  virtual bool UnregisterFilePath(size_t path_id) override;
+  virtual bool ReadAhead(size_t path_id,
+                         ReadAheadKind kind,
+                         size_t length,
+                         size_t offset) override;
+
+  virtual bool UnreadAhead(size_t path_id,
+                           ReadAheadKind kind,
+                           size_t length,
+                           size_t offset) override;
+
+  virtual void Dump(std::ostream& os, bool multiline) const override;
+
+  // Creates a new session indirectly.
+  // Writes to daemon the new session command.
+  SessionIndirect(size_t session_id,
+                  std::string description,
+                  std::shared_ptr<PrefetcherDaemon> daemon,
+                  bool send_command = true);
+
+  // Destroys the current session.
+  // Writes to daemon that the session is to be destroyed.
+  virtual ~SessionIndirect();
+
+ protected:
+  std::shared_ptr<PrefetcherDaemon> daemon_;
+};
+
+// Out-of-process session. Requires prefetcher daemon.
+class SessionIndirectSocket : public SessionIndirect {
+ public:
+  // Creates a new session indirectly.
+  // Writes to daemon the new session command.
+  SessionIndirectSocket(size_t session_id,
+                        int fd,
+                        std::string description,
+                        std::shared_ptr<PrefetcherDaemon> daemon);
+  // Destroys the current session.
+  // Writes to daemon that the session is to be destroyed.
+  virtual ~SessionIndirectSocket();
+
+ private:
+};
+
+
+}  // namespace prefetcher
+}  // namespace iorap
+
+#endif
+
diff --git a/src/prefetcher/session_manager.cc b/src/prefetcher/session_manager.cc
new file mode 100644
index 0000000..d6fab1f
--- /dev/null
+++ b/src/prefetcher/session_manager.cc
@@ -0,0 +1,281 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "session_manager.h"
+
+#include "prefetcher/prefetcher_daemon.h"
+#include "prefetcher/session.h"
+#include "prefetcher/task_id.h"
+#include "serialize/arena_ptr.h"
+#include "serialize/protobuf_io.h"
+
+#include <android-base/logging.h>
+#include <android-base/chrono_utils.h>
+#include <android-base/unique_fd.h>
+#include <fcntl.h>
+#include <functional>
+#include <stdint.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unordered_map>
+
+namespace iorap {
+namespace prefetcher {
+
+std::ostream& operator<<(std::ostream& os, const SessionManager& manager) {
+  manager.Dump(os, /*multiline*/false);
+  return os;
+}
+
+SessionManager::SessionManager() {
+}
+
+class SessionManagerBase  : public SessionManager {
+ public:
+  virtual void Dump(std::ostream& os, bool multiline) const {
+    if (!multiline) {
+      os << "SessionManager{";
+
+      os << "sessions=[";
+      for (auto it = sessions_map_.begin();
+           it != sessions_map_.end();
+           ++it) {
+        os << "(" << it->second.description << ") ";
+        it->second.session->Dump(os, /*multiline*/false);
+      }
+      os << "]";
+      return;
+    }
+
+    os << "SessionManager (session count = " << sessions_map_.size() << "):" << std::endl;
+    os << std::endl;
+
+    for (auto it = sessions_map_.begin();
+         it != sessions_map_.end();
+         ++it) {
+      os << "Description: " << it->second.description << std::endl;
+      it->second.session->Dump(os, /*multiline*/true);
+    }
+
+    // TODO: indentations? Use this pseudo line break for the time being.
+    os << "--------------------------------" << std::endl;
+  }
+
+  virtual ~SessionManagerBase() {}
+
+  virtual std::shared_ptr<Session> FindSession(size_t session_id) const override {
+    auto it = sessions_map_.find(session_id);
+    if (it != sessions_map_.end()) {
+      DCHECK_EQ(session_id, it->second.SessionId());
+      return it->second.session;
+    } else {
+      return nullptr;
+    }
+  }
+
+  virtual bool DestroySession(size_t session_id) override {
+    auto it = sessions_map_.find(session_id);
+    if (it != sessions_map_.end()) {
+      sessions_map_.erase(it);
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+ protected:
+  void InsertNewSession(std::shared_ptr<Session> session, std::string description) {
+    DCHECK(!FindSession(session->SessionId())) << "session cannot already exist";
+
+    size_t session_id = session->SessionId();
+
+    SessionData data;
+    data.session = std::move(session);
+    data.description = std::move(description);
+
+    sessions_map_.insert({session_id, std::move(data)});
+  }
+
+ private:
+  struct SessionData {
+    std::shared_ptr<Session> session;
+    std::string description;
+
+    size_t SessionId() const {
+      return session->SessionId();
+    }
+  };
+
+  std::unordered_map</*session_id*/size_t, SessionData> sessions_map_;
+};
+
+class SessionManagerDirect : public SessionManagerBase {
+ public:
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description) override {
+    LOG(VERBOSE) << "CreateSessionDirect id=" << session_id << ", description=" << description;
+
+    std::shared_ptr<Session> session =
+      std::static_pointer_cast<Session>(std::make_shared<SessionDirect>(session_id,
+                                                                        description));
+    DCHECK(FindSession(session_id) == nullptr);
+    InsertNewSession(session, std::move(description));
+    return session;
+  }
+
+  SessionManagerDirect() {
+    // Intentionally left empty.
+  }
+
+ private:
+};
+
+
+class SessionManagerIndirect : public SessionManagerBase {
+ public:
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description) override {
+    LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description;
+
+    std::shared_ptr<Session> session =
+        std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id,
+                                                                            description,
+                                                                            daemon_));
+    InsertNewSession(session, description);
+    return session;
+  }
+
+  SessionManagerIndirect() : daemon_{std::make_shared<PrefetcherDaemon>()} {
+    //StartViaFork etc.
+    // TODO: also expose a 'MainLoop(...) -> daemon::Main(..)' somehow in the base interface.
+    auto params = daemon_->StartPipesViaFork();
+    if (!params) {
+      LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd";
+    }
+  }
+
+  virtual ~SessionManagerIndirect() {
+    Command cmd{};
+    cmd.choice = CommandChoice::kExit;
+
+    if (!daemon_->SendCommand(cmd)) {
+      LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd";
+    }
+  }
+
+  virtual void Dump(std::ostream& os, bool multiline) const override {
+    Command cmd{};
+    cmd.choice = CommandChoice::kDumpEverything;
+
+    if (!daemon_->SendCommand(cmd)) {
+      LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd";
+    }
+  }
+
+
+ private:
+  // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it.
+  std::shared_ptr<PrefetcherDaemon> daemon_;
+};
+
+class SessionManagerIndirectSocket : public SessionManagerBase {
+ public:
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description) override {
+    DCHECK(false) << "not supposed to create a regular session for Socket";
+
+    LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description;
+
+    std::shared_ptr<Session> session =
+        std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id,
+                                                                            description,
+                                                                            daemon_));
+    InsertNewSession(session, description);
+    return session;
+  }
+
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description,
+                                                 std::optional<int> fd) override {
+    CHECK(fd.has_value());
+    LOG(VERBOSE) << "CreateSessionIndirectSocket id=" << session_id
+                 << ", description=" << description
+                 << ", fd=" << *fd;
+
+    std::shared_ptr<Session> session =
+        std::static_pointer_cast<Session>(std::make_shared<SessionIndirectSocket>(session_id,
+                                                                                  *fd,
+                                                                                  description,
+                                                                                  daemon_));
+    InsertNewSession(session, description);
+    return session;
+  }
+
+  SessionManagerIndirectSocket() : daemon_{std::make_shared<PrefetcherDaemon>()} {
+    auto params = daemon_->StartSocketViaFork();
+    if (!params) {
+      LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd";
+    }
+  }
+
+  virtual ~SessionManagerIndirectSocket() {
+    Command cmd{};
+    cmd.choice = CommandChoice::kExit;
+
+    if (!daemon_->SendCommand(cmd)) {
+      LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd";
+    }
+  }
+
+  virtual void Dump(std::ostream& os, bool multiline) const override {
+    Command cmd{};
+    cmd.choice = CommandChoice::kDumpEverything;
+
+    if (!daemon_->SendCommand(cmd)) {
+      LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd";
+    }
+  }
+
+
+ private:
+  // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it.
+  std::shared_ptr<PrefetcherDaemon> daemon_;
+};
+
+std::unique_ptr<SessionManager> SessionManager::CreateManager(SessionKind kind) {
+  LOG(VERBOSE) << "SessionManager::CreateManager kind=" << kind;
+
+  switch (kind) {
+    case SessionKind::kInProcessDirect: {
+      SessionManager* ptr = new SessionManagerDirect();
+      return std::unique_ptr<SessionManager>{ptr};
+    }
+    case SessionKind::kOutOfProcessIpc: {
+      SessionManager* ptr = new SessionManagerIndirect();
+      return std::unique_ptr<SessionManager>{ptr};
+    }
+    case SessionKind::kOutOfProcessSocket: {
+      SessionManager* ptr = new SessionManagerIndirectSocket();
+      return std::unique_ptr<SessionManager>{ptr};
+    }
+    default: {
+      LOG(FATAL) << "Invalid session kind: " << static_cast<int>(kind);
+      break;
+    }
+  }
+}
+
+}  // namespace prefetcher
+}  // namespace iorap
diff --git a/src/prefetcher/session_manager.h b/src/prefetcher/session_manager.h
new file mode 100644
index 0000000..45120d5
--- /dev/null
+++ b/src/prefetcher/session_manager.h
@@ -0,0 +1,91 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PREFETCHER_SESSION_MANAGER_H_
+#define PREFETCHER_SESSION_MANAGER_H_
+
+#include <optional>
+#include <ostream>
+#include <memory>
+
+namespace iorap {
+namespace prefetcher {
+
+class Session;
+
+enum class SessionKind : uint32_t {
+  kInProcessDirect,
+  kOutOfProcessIpc,
+  kOutOfProcessSocket,
+};
+
+inline std::ostream& operator<<(std::ostream& os, SessionKind kind) {
+  if (kind == SessionKind::kInProcessDirect) {
+    os << "kInProcessDirect";
+  } else if (kind == SessionKind::kOutOfProcessIpc) {
+    os << "kOutOfProcessIpc";
+  } else if (kind == SessionKind::kOutOfProcessSocket) {
+    os << "kOutOfProcessSocket";
+  } else {
+    os << "(invalid)";
+  }
+  return os;
+}
+
+class SessionManager {
+ public:
+  static std::unique_ptr<SessionManager> CreateManager(SessionKind kind);
+
+  // Create a new session. The description is used by Dump.
+  // Manager maintains a strong ref to this session, so DestroySession must also
+  // be called prior to all refs dropping to 0.
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description) = 0;
+
+  // Create a new session. The description is used by Dump.
+  // Manager maintains a strong ref to this session, so DestroySession must also
+  // be called prior to all refs dropping to 0.
+  virtual std::shared_ptr<Session> CreateSession(size_t session_id,
+                                                 std::string description,
+                                                 std::optional<int> fd) {
+    return CreateSession(session_id, description);
+  }
+
+  // Look up an existing session that was already created.
+  // Returns null if there is no such session.
+  virtual std::shared_ptr<Session> FindSession(size_t session_id) const = 0;
+
+  // Drop all manager references to an existing session.
+  // Returns false if the session does not exist already.
+  virtual bool DestroySession(size_t session_id) = 0;
+
+  // Multi-line detailed dump, e.g. for dumpsys.
+  // Single-line summary dump, e.g. for logcat.
+  virtual void Dump(std::ostream& os, bool multiline) const = 0;
+
+  // Note: session lifetime is tied to manager. The manager has strong pointers to sessions.
+  virtual ~SessionManager() {}
+
+ protected:
+  SessionManager();
+};
+
+// Single-line summary dump of Session.
+std::ostream& operator<<(std::ostream&os, const SessionManager& session);
+
+}  // namespace prefetcher
+}  // namespace iorap
+
+#endif
+
diff --git a/src/prefetcher/task_id.h b/src/prefetcher/task_id.h
new file mode 100644
index 0000000..dc26954
--- /dev/null
+++ b/src/prefetcher/task_id.h
@@ -0,0 +1,39 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PREFETCHER_TASK_ID_H_
+#define PREFETCHER_TASK_ID_H_
+
+#include <ostream>
+#include <string>
+
+namespace iorap {
+namespace prefetcher {
+
+struct TaskId {
+  size_t id;  // Unique monotonically increasing ID.
+  std::string path;  // File path to the trace file.
+
+  friend std::ostream& operator<<(std::ostream& os, const TaskId& task_id) {
+    os << "TaskId { id: " << task_id.id << ", path: " << task_id.path << "}";
+    return os;
+  }
+
+};
+
+}  // namespace prefetcher
+}  // namespace iorap
+
+#endif
+
diff --git a/src/serialize/TraceFile.proto b/src/serialize/TraceFile.proto
new file mode 100644
index 0000000..fc72d0d
--- /dev/null
+++ b/src/serialize/TraceFile.proto
@@ -0,0 +1,47 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";  // use required fields, which aren't in proto3.
+
+package iorap.serialize.proto;  // C++ namespace iorap::serialize::proto
+option java_package = "com.google.android.iorap";
+option optimize_for = LITE_RUNTIME;
+
+// TODO: should these fields be 'packed' for "smaller encoding" ?
+
+message TraceFile {
+  required TraceFileIndex index = 1;
+  required TraceFileList list = 2;
+}
+
+message TraceFileIndex {
+  repeated TraceFileIndexEntry entries = 1;
+}
+
+message TraceFileIndexEntry {
+  required int64 id = 1;
+  required string file_name = 2;
+}
+
+message TraceFileList {
+  repeated TraceFileEntry entries = 1;
+}
+
+message TraceFileEntry {
+  required int64 index_id = 1;
+  required int64 file_offset = 2;
+  required int64 file_length = 3;
+}
+
+// XX: use nested messages?
\ No newline at end of file
diff --git a/src/serialize/arena_ptr.h b/src/serialize/arena_ptr.h
new file mode 100644
index 0000000..d8bea26
--- /dev/null
+++ b/src/serialize/arena_ptr.h
@@ -0,0 +1,76 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SERIALIZE_ARENA_PTR_H_
+#define SERIALIZE_ARENA_PTR_H_
+
+#include <google/protobuf/arena.h>
+#include <memory>
+
+namespace iorap {
+namespace serialize {
+
+/**
+ * @file
+ *
+ * Helpers for protobuf arena allocators. We use smart pointers
+ * with an arena embedded inside of them to avoid caring about the
+ * arena in other parts of libiorap.
+ */
+
+// Arena-managed objects must not be deleted manually.
+// When the Arena goes out of scope, it cleans everything up itself.
+template <typename T>
+void DoNotDelete(T*) {}
+
+template <typename T, typename Base = std::unique_ptr<T, decltype(&DoNotDelete<T>)>>
+struct ArenaPtr : public Base {
+  template <typename... Args>
+  static ArenaPtr<T> Make(Args&& ... args) {
+    ArenaPtr<T> arena_ptr(nullptr);
+    arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(),
+                                                       std::forward<Args>(args)...));
+    return arena_ptr;
+  }
+
+  ArenaPtr(std::nullptr_t) : Base(nullptr, &DoNotDelete<T>) {}  // NOLINT explicit.
+
+ private:
+  // Use a unique_ptr because Arena doesn't support move semantics.
+  std::unique_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}};
+};
+
+template <typename T, typename Base = std::shared_ptr<T>>
+struct ArenaSharedPtr : public Base {
+  template <typename... Args>
+  static ArenaSharedPtr<T> Make(Args&& ... args) {
+    ArenaSharedPtr<T> arena_ptr(nullptr, &DoNotDelete<T>);
+    arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(),
+                                                       std::forward<Args>(args)...));
+    return arena_ptr;
+  }
+
+  ArenaSharedPtr() = default;
+  template <typename Deleter>
+  ArenaSharedPtr(std::nullptr_t, Deleter d) : Base(nullptr, d) {}  // NOLINT explicit.
+
+ private:
+  std::shared_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}};
+};
+
+}  // namespace serialize
+}  // namespace iorap
+
+#endif
+
diff --git a/src/serialize/protobuf_io.cc b/src/serialize/protobuf_io.cc
new file mode 100644
index 0000000..1b6420f
--- /dev/null
+++ b/src/serialize/protobuf_io.cc
@@ -0,0 +1,173 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "protobuf_io.h"
+
+#include "common/trace.h"
+#include "serialize/arena_ptr.h"
+
+#include <android-base/chrono_utils.h>
+#include <android-base/logging.h>
+#include <android-base/unique_fd.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <utils/Trace.h>
+
+#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
+#include "system/iorap/src/serialize/TraceFile.pb.h"
+
+namespace iorap {
+namespace serialize {
+
+ArenaPtr<proto::TraceFile> ProtobufIO::Open(std::string file_path) {
+  // TODO: file a bug about this.
+  // Note: can't use {} here, clang think it's narrowing from long->int.
+  android::base::unique_fd fd(TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_RDONLY)));
+  if (fd.get() < 0) {
+    PLOG(DEBUG) << "ProtobufIO: open failed: " << file_path;
+    return nullptr;
+  }
+
+  return Open(fd.get(), file_path.c_str());
+}
+
+ArenaPtr<proto::TraceFile> ProtobufIO::Open(int fd, const char* file_path) {
+
+  ScopedFormatTrace atrace_protobuf_io_open(ATRACE_TAG_ACTIVITY_MANAGER,
+                                            "ProtobufIO::Open %s",
+                                            file_path);
+  android::base::Timer timer{};
+
+  struct stat buf;
+  if (fstat(fd, /*out*/&buf) < 0) {
+    PLOG(ERROR) << "ProtobufIO: open error, fstat failed: " << file_path;
+    return nullptr;
+  }
+  // XX: off64_t for stat::st_size ?
+
+  // Using the mmap appears to be the only way to do zero-copy with protobuf lite.
+  void* data = mmap(/*addr*/nullptr,
+                    buf.st_size,
+                    PROT_READ, MAP_SHARED | MAP_POPULATE,
+                    fd,
+                    /*offset*/0);
+  if (data == nullptr) {
+    PLOG(ERROR) << "ProtobufIO: open error, mmap failed: " << file_path;
+    return nullptr;
+  }
+
+  ArenaPtr<proto::TraceFile> protobuf_trace_file = ArenaPtr<proto::TraceFile>::Make();
+  if (protobuf_trace_file == nullptr) {
+    LOG(ERROR) << "ProtobufIO: open error, failed to create arena: " << file_path;
+    return nullptr;
+  }
+
+  google::protobuf::io::ArrayInputStream protobuf_input_stream{data, static_cast<int>(buf.st_size)};
+  if (!protobuf_trace_file->ParseFromZeroCopyStream(/*in*/&protobuf_input_stream)) {
+    // XX: Does protobuf on android already have the right LogHandler ?
+    LOG(ERROR) << "ProtobufIO: open error, protobuf parsing failed: " << file_path;
+    return nullptr;
+  }
+
+  if (munmap(data, buf.st_size) < 0) {
+    PLOG(WARNING) << "ProtobufIO: open problem, munmap failed, possibly memory leak? "
+                  << file_path;
+  }
+
+  LOG(VERBOSE) << "ProtobufIO: open succeeded: " << file_path << ", duration: " << timer;
+  return protobuf_trace_file;
+}
+
+iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully(
+    const ::google::protobuf::MessageLite& message,
+    std::string_view file_path) {
+
+  std::string str{file_path};
+  android::base::unique_fd fd(TEMP_FAILURE_RETRY(
+      ::open(str.c_str(),
+             O_CREAT | O_TRUNC | O_RDWR,
+             S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP)));  // ugo: rw-rw----
+  if (fd.get() < 0) {
+    int err = errno;
+    PLOG(ERROR) << "ProtobufIO: open failed: " << file_path;
+    return unexpected{err};
+  }
+
+  return WriteFully(message, fd.get(), file_path);
+}
+
+iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully(
+    const ::google::protobuf::MessageLite& message,
+    int fd,
+    std::string_view file_path) {
+
+  int byte_size = message.ByteSize();
+  if (byte_size < 0) {
+    DCHECK(false) << "Invalid protobuf size: " << byte_size;
+    LOG(ERROR) << "ProtobufIO: Invalid protobuf size: " << byte_size;
+    return unexpected{EDOM};
+  }
+  size_t serialized_size = static_cast<size_t>(byte_size);
+
+  // Change the file to be exactly the length of the protobuf.
+  if (ftruncate(fd, static_cast<off_t>(serialized_size)) < 0) {
+    int err = errno;
+    PLOG(ERROR) << "ProtobufIO: ftruncate (size=" << serialized_size << ") failed";
+    return unexpected{err};
+  }
+
+  // Using the mmap appears to be the only way to do zero-copy with protobuf lite.
+  void* data = mmap(/*addr*/nullptr,
+                    serialized_size,
+                    PROT_WRITE,
+                    MAP_SHARED,
+                    fd,
+                    /*offset*/0);
+  if (data == nullptr) {
+    int err = errno;
+    PLOG(ERROR) << "ProtobufIO: mmap failed: " << file_path;
+    return unexpected{err};
+  }
+
+  // Zero-copy write from protobuf to file via memory-map.
+  ::google::protobuf::io::ArrayOutputStream output_stream{data, byte_size};
+  if (!message.SerializeToZeroCopyStream(/*inout*/&output_stream)) {
+    // This should never happen since we pre-allocated the file and memory map to be large
+    // enough to store the full protobuf.
+    DCHECK(false) << "ProtobufIO:: SerializeToZeroCopyStream failed despite precalculating size";
+    LOG(ERROR) << "ProtobufIO: SerializeToZeroCopyStream failed";
+    return unexpected{EXFULL};
+  }
+
+  // Guarantee that changes are written back prior to munmap.
+  if (msync(data, static_cast<size_t>(serialized_size), MS_SYNC) < 0) {
+    int err = errno;
+    PLOG(ERROR) << "ProtobufIO: msync failed";
+    return unexpected{err};
+  }
+
+  if (munmap(data, serialized_size) < 0) {
+    PLOG(WARNING) << "ProtobufIO: munmap failed, possibly memory leak? "
+                  << file_path;
+  }
+
+  return serialized_size;
+}
+
+}  // namespace serialize
+}  // namespace iorap
+
diff --git a/src/serialize/protobuf_io.h b/src/serialize/protobuf_io.h
new file mode 100644
index 0000000..1092fd7
--- /dev/null
+++ b/src/serialize/protobuf_io.h
@@ -0,0 +1,64 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef SERIALIZE_PROTOBUF_IO_H_
+#define SERIALIZE_PROTOBUF_IO_H_
+
+#include "common/expected.h"
+#include "serialize/arena_ptr.h"
+#include "system/iorap/src/serialize/TraceFile.pb.h"
+
+#include <string>
+#include <string_view>
+
+namespace iorap {
+namespace serialize {
+
+// XX: either the namespace should be called pb|proto[buf]
+// or we should hide the protobuf-ness from the names and call this class "IO" , "Reader", etc?
+// although an obvious name might be the "OpenFactory" or "ProtobufFacade" that reads too much
+// like a bad joke.
+
+// Helpers to read a TraceFile protobuf from a file [descriptor].
+class ProtobufIO {
+ public:
+  // XX: proto::TraceFile seems annoying, maybe just serialize::TraceFile ?
+
+  // Open the protobuf associated at the filepath. Returns null on failure.
+  static ArenaPtr<proto::TraceFile> Open(std::string file_path);
+  // Open the protobuf from the file descriptor. Returns null on failure.
+  static ArenaPtr<proto::TraceFile> Open(int fd, const char* file_path = "<unknown>");
+
+  // Save the protobuf by overwriting the file at file_path.
+  // The file state is indeterminate at failure.
+  // Returns # of bytes written out on success, otherwise the errno value.
+  static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully(
+      const ::google::protobuf::MessageLite& message,
+      std::string_view file_path);
+  // Save the protobuf by truncating the file already open at 'fd'.
+  // The file state is indeterminate at failure.
+  // Returns # of bytes written out on success, otherwise the errno value.
+  static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully(
+      const ::google::protobuf::MessageLite& message,
+      int fd,
+      std::string_view file_path = "<unknown>");
+
+  ProtobufIO() = delete;
+};
+
+}  // namespace serialize
+}  // namespace iorap
+
+#endif
+