def-tool: Add APEX module support

This commit adds APEX module support to VNDK definition tool.

Test: ./tests/run.sh
Test: Run vndk_definition_tool.py on latest walleye
Test: Run vndk_definition_tool.py on latest aosp_arm64

Bug: 126126189
Change-Id: Ib92e7b058a3fee3520b612e053751d380cf448cd

8 files changed, 370 insertions, 50 deletions

diff --git a/vndk/tools/definition-tool/tests/test_elf_linker.py b/vndk/tools/definition-tool/tests/test_elf_linker.py
index ba2c9a85c..12cd71a11 100644
--- a/vndk/tools/definition-tool/tests/test_elf_linker.py
+++ b/vndk/tools/definition-tool/tests/test_elf_linker.py
@@ -4,7 +4,7 @@ import re
 import tempfile
 
 from vndk_definition_tool import (
-    ELF, GenericRefs, PT_SYSTEM, PT_VENDOR, VNDKLibDir)
+    ELF, ELFLinker, GenericRefs, PT_SYSTEM, PT_VENDOR, VNDKLibDir)
 
 from .compat import StringIO, TestCase, patch
 from .utils import GraphBuilder
@@ -460,6 +460,51 @@ class ELFLinkerTest(TestCase):
         self.assertIn(libvndk_sp_d_64, libvndk_sp_c_64.deps_all)
 
 
+    def test_rewrite_apex_modules(self):
+        graph = ELFLinker()
+
+        libfoo = graph.add_lib(PT_SYSTEM, '/system/apex/foo/lib/libfoo.so',
+                               ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
+        libbar = graph.add_lib(PT_SYSTEM, '/system/apex/bar/lib/libbar.so',
+                               ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
+
+        graph.rewrite_apex_modules()
+
+        self.assertEqual(libfoo.path, '/apex/foo/lib/libfoo.so')
+        self.assertEqual(libbar.path, '/apex/bar/lib/libbar.so')
+
+
+    def test_link_apex_modules(self):
+        graph = ELFLinker()
+
+        libfoo = graph.add_lib(PT_SYSTEM, '/system/apex/foo/lib/libfoo.so',
+                               ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
+        libbar = graph.add_lib(PT_SYSTEM, '/system/lib/libbar.so',
+                               ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB,
+                                   dt_needed=['libfoo.so']))
+
+        graph.rewrite_apex_modules()
+        graph.resolve_deps()
+
+        self.assertIn(libfoo, libbar.deps_all)
+
+
+    def test_link_apex_bionic(self):
+        graph = ELFLinker()
+
+        libc = graph.add_lib(
+            PT_SYSTEM, '/system/apex/com.android.runtime/lib/bionic/libc.so',
+            ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB))
+        libbar = graph.add_lib(
+            PT_SYSTEM, '/system/lib/libbar.so',
+            ELF(ELF.ELFCLASS32, ELF.ELFDATA2LSB, dt_needed=['libc.so']))
+
+        graph.rewrite_apex_modules()
+        graph.resolve_deps()
+
+        self.assertIn(libc, libbar.deps_all)
+
+
 class ELFLinkerDlopenDepsTest(TestCase):
     def test_add_dlopen_deps(self):
         gb = GraphBuilder()
diff --git a/vndk/tools/definition-tool/tests/test_scandir.py b/vndk/tools/definition-tool/tests/test_scandir.py
new file mode 100644
index 000000000..cc402c041
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/test_scandir.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+
+from vndk_definition_tool import scandir
+
+
+class ScanDirTest(unittest.TestCase):
+    def test_scandir(self):
+        testdata_dir = os.path.join(
+            os.path.dirname(__file__), 'testdata', 'test_scandir')
+
+        num_entries = 0
+        for ent in scandir(testdata_dir):
+            num_entries += 1
+
+            self.assertEqual(ent.path, os.path.join(testdata_dir, ent.name))
+
+            if ent.name == 'test_file':
+                self.assertTrue(ent.is_file())
+                self.assertFalse(ent.is_dir())
+                self.assertFalse(ent.is_symlink())
+            elif ent.name == 'test_dir':
+                self.assertFalse(ent.is_file())
+                self.assertTrue(ent.is_dir())
+                self.assertFalse(ent.is_symlink())
+            elif ent.name == 'link_test_file':
+                self.assertTrue(ent.is_file())
+                self.assertFalse(ent.is_file(follow_symlinks=False))
+                self.assertFalse(ent.is_dir())
+                self.assertFalse(ent.is_dir(follow_symlinks=False))
+                self.assertTrue(ent.is_symlink())
+            elif ent.name == 'link_test_dir':
+                self.assertFalse(ent.is_file())
+                self.assertFalse(ent.is_file(follow_symlinks=False))
+                self.assertTrue(ent.is_dir())
+                self.assertFalse(ent.is_dir(follow_symlinks=False))
+                self.assertTrue(ent.is_symlink())
+            elif ent.name == 'link_does_not_exist':
+                self.assertFalse(ent.is_file())
+                self.assertFalse(ent.is_file(follow_symlinks=False))
+                self.assertFalse(ent.is_dir())
+                self.assertFalse(ent.is_dir(follow_symlinks=False))
+                self.assertTrue(ent.is_symlink())
+            else:
+                self.fail('unexpected filename: ' + ent.name)
+
+        self.assertEqual(num_entries, 5)
diff --git a/vndk/tools/definition-tool/tests/testdata/test_scandir/link_does_not_exist b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_does_not_exist
new file mode 120000
index 000000000..ee1f6cb7f
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_does_not_exist
@@ -0,0 +1 @@
+does_not_exist
\ No newline at end of file
diff --git a/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_dir b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_dir
new file mode 120000
index 000000000..431786635
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_dir
@@ -0,0 +1 @@
+test_dir
\ No newline at end of file
diff --git a/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_file b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_file
new file mode 120000
index 000000000..e9b09f33e
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/testdata/test_scandir/link_test_file
@@ -0,0 +1 @@
+test_file
\ No newline at end of file
diff --git a/vndk/tools/definition-tool/tests/testdata/test_scandir/test_dir/test_file b/vndk/tools/definition-tool/tests/testdata/test_scandir/test_dir/test_file
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/testdata/test_scandir/test_dir/test_file
diff --git a/vndk/tools/definition-tool/tests/testdata/test_scandir/test_file b/vndk/tools/definition-tool/tests/testdata/test_scandir/test_file
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/vndk/tools/definition-tool/tests/testdata/test_scandir/test_file
diff --git a/vndk/tools/definition-tool/vndk_definition_tool.py b/vndk/tools/definition-tool/vndk_definition_tool.py
index ea3ee662a..982976e19 100755
--- a/vndk/tools/definition-tool/vndk_definition_tool.py
+++ b/vndk/tools/definition-tool/vndk_definition_tool.py
@@ -16,6 +16,7 @@ import re
 import shutil
 import stat
 import struct
+import subprocess
 import sys
 import zipfile
 
@@ -77,6 +78,98 @@ try:
 except ImportError:
     pass
 
+try:
+    from tempfile import TemporaryDirectory
+except ImportError:
+    import shutil
+    import tempfile
+
+    class TemporaryDirectory(object):
+        def __init__(self, suffix='', prefix='tmp', dir=None):
+            # pylint: disable=redefined-builtin
+            self.name = tempfile.mkdtemp(suffix, prefix, dir)
+
+        def __del__(self):
+            self.cleanup()
+
+        def __enter__(self):
+            return self.name
+
+        def __exit__(self, exc, value, tb):
+            self.cleanup()
+
+        def cleanup(self):
+            if self.name:
+                shutil.rmtree(self.name)
+                self.name = None
+
+try:
+    from os import scandir
+except ImportError:
+    import stat
+    import os
+
+    class DirEntry(object):
+        def __init__(self, name, path):
+            self.name = name
+            self.path = path
+            self._stat = None
+            self._lstat = None
+
+        @staticmethod
+        def _stat_impl(path, follow_symlinks):
+            return os.stat(path) if follow_symlinks else os.lstat(path)
+
+        def stat(self, follow_symlinks=True):
+            attr = '_stat' if follow_symlinks else '_lstat'
+            stat_res = getattr(self, attr)
+            if stat_res is None:
+                stat_res = self._stat_impl(self.path, follow_symlinks)
+                setattr(self, attr, stat_res)
+            return stat_res
+
+        def is_dir(self, follow_symlinks=True):
+            try:
+                return stat.S_ISDIR(self.stat(follow_symlinks).st_mode)
+            except EnvironmentError:
+                return False
+
+        def is_file(self, follow_symlinks=True):
+            try:
+                return stat.S_ISREG(self.stat(follow_symlinks).st_mode)
+            except EnvironmentError:
+                return False
+
+        def is_symlink(self):
+            return stat.S_ISLNK(self.stat(follow_symlinks=False).st_mode)
+
+    def scandir(path):
+        for name in os.listdir(path):
+            yield DirEntry(name, os.path.join(path, name))
+
+
+#------------------------------------------------------------------------------
+# Print Function
+#------------------------------------------------------------------------------
+
+def print_sb(*args, **kwargs):
+    """A print function that supports both str and bytes."""
+    sep = kwargs.get('sep', ' ')
+    end = kwargs.get('end', '\n')
+    out_file = kwargs.get('file', sys.stdout)
+    for i, arg in enumerate(args):
+        if i > 0:
+            out_file.write(sep)
+        if isinstance(arg, str):
+            out_file.write(arg)
+        elif isinstance(arg, bytes):
+            out_file.flush()
+            out_file.buffer.write(arg)
+            out_file.flush()
+        else:
+            out_file.write(str(arg))
+    out_file.write(end)
+
 
 #------------------------------------------------------------------------------
 # Modified UTF-8 Encoder and Decoder
@@ -1413,22 +1506,6 @@ class LibProperties(object):
 # ELF Linker
 #------------------------------------------------------------------------------
 
-def is_accessible(path):
-    try:
-        mode = os.stat(path).st_mode
-        return (mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) != 0
-    except FileNotFoundError:
-        return False
-
-
-def scan_accessible_files(root):
-    for base, _, files in os.walk(root):
-        for filename in files:
-            path = os.path.join(base, filename)
-            if is_accessible(path):
-                yield path
-
-
 def is_zipfile(path):
     # zipfile.is_zipfile() tries to find the zip header in the file.  But we
     # only want to scan the zip file that starts with the magic word.  Thus,
@@ -1452,24 +1529,134 @@ def scan_zip_file(zip_file_path):
                    zip_file.open(name, 'r').read())
 
 
-def scan_elf_files(root, unzip_files=True):
-    """Scan all ELF files under a directory."""
-    for path in scan_accessible_files(root):
-        # If this is a zip file and unzip_file is true, scan the ELF files in
-        # the zip file.
-        if unzip_files and is_zipfile(path):
-            for path, content in scan_zip_file(path):
-                try:
-                    yield (path, ELF.loads(content))
-                except ELFError:
-                    pass
+def dump_ext4_img(img_file_path, out_dir):
+    if ' ' in out_dir:
+        raise ValueError('out_dir must not have space character')
+
+    cmd = ['debugfs', img_file_path, '-R', 'rdump / ' + out_dir]
+
+    # Run the debugfs command.
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout, stderr = proc.communicate()
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(proc.returncode, cmd, stderr)
+
+    # Print error messages if they are not the ones that should be ignored.
+    for line in stderr.splitlines():
+        if line.startswith(b'debugfs '):
+            continue
+        if b'Operation not permitted while changing ownership of' in line:
             continue
+        print_sb('error: debugfs:', line, file=sys.stderr)
 
-        # Load ELF from the path.
-        try:
-            yield (path, ELF.load(path))
-        except ELFError:
-            pass
+
+def is_accessible(path):
+    try:
+        mode = os.stat(path).st_mode
+        return (mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) != 0
+    except FileNotFoundError:
+        return False
+
+
+def scan_ext4_image(img_file_path, mount_point, unzip_files):
+    """Scan all ELF files in the ext4 image."""
+    with TemporaryDirectory() as tmp_dir:
+        dump_ext4_img(img_file_path, tmp_dir)
+        for path, elf in scan_elf_files(tmp_dir, mount_point, unzip_files):
+            yield path, elf
+
+
+def scan_apex_dir(apex_collection_root, apex_dir, unzip_files):
+    # Read the manifest file.
+    manifest_file_path = os.path.join(apex_dir, 'apex_manifest.json')
+    try:
+        with open(manifest_file_path, 'r') as manifest_file:
+            manifest = json.load(manifest_file)
+    except FileNotFoundError:
+        print('error: Failed to find apex manifest: {}'
+              .format(manifest_file_path), file=sys.stderr)
+        return
+
+    # Read the module name.
+    try:
+        apex_name = manifest['name']
+    except KeyError:
+        print('error: Failed to read apex name from manifest: {}'
+              .format(manifest_file_path), file=sys.stderr)
+        return
+
+    # Scan the payload (or the flatten payload).
+    mount_point = os.path.join(apex_collection_root, apex_name)
+    img_file_path = os.path.join(apex_dir, 'apex_payload.img')
+    if os.path.exists(img_file_path):
+        for path, elf in scan_ext4_image(img_file_path, mount_point,
+                                         unzip_files):
+            yield path, elf
+    else:
+        for path, elf in scan_elf_files(apex_dir, mount_point, unzip_files):
+            yield path, elf
+
+
+def scan_apex_file(apex_collection_root, apex_zip_file, unzip_files):
+    with TemporaryDirectory() as tmp_dir:
+        with zipfile.ZipFile(apex_zip_file) as zip_file:
+            zip_file.extractall(tmp_dir)
+        for path, elf in scan_apex_dir(apex_collection_root, tmp_dir,
+                                       unzip_files):
+            yield path, elf
+
+
+def scan_apex_files(apex_collection_root, unzip_files):
+    for ent in scandir(apex_collection_root):
+        if ent.is_dir():
+            for path, elf in scan_apex_dir(apex_collection_root, ent.path,
+                                           unzip_files):
+                yield path, elf
+        elif ent.is_file() and ent.name.endswith('.apex'):
+            for path, elf in scan_apex_file(apex_collection_root, ent.path,
+                                            unzip_files):
+                yield path, elf
+
+
+def scan_elf_files(root, mount_point=None, unzip_files=True):
+    """Scan all ELF files under a directory."""
+
+    if mount_point:
+        root_prefix_len = len(root) + 1
+        def norm_path(path):
+            return os.path.join(mount_point, path[root_prefix_len:])
+    else:
+        def norm_path(path):
+            return path
+
+    for base, dirnames, filenames in os.walk(root):
+        if base == root and 'apex' in dirnames:
+            dirnames.remove('apex')
+            for path, elf in scan_apex_files(os.path.join(root, 'apex'),
+                                             unzip_files):
+                yield (path, elf)
+
+        for filename in filenames:
+            path = os.path.join(base, filename)
+            if not is_accessible(path):
+                continue
+
+
+            # If this is a zip file and unzip_file is true, scan the ELF files
+            # in the zip file.
+            if unzip_files and is_zipfile(path):
+                for path, content in scan_zip_file(path):
+                    try:
+                        yield (norm_path(path), ELF.loads(content))
+                    except ELFError:
+                        pass
+                continue
+
+            # Load ELF from the path.
+            try:
+                yield (norm_path(path), ELF.load(path))
+            except ELFError:
+                pass
 
 
 PT_SYSTEM = 0
@@ -1999,6 +2186,8 @@ class ELFLinker(object):
 
         self.ro_vndk_version = ro_vndk_version
 
+        self.apex_module_names = set()
+
 
     def _add_lib_to_lookup_dict(self, lib):
         self.lib_pt[lib.partition].add(lib.path, lib)
@@ -2008,6 +2197,12 @@ class ELFLinker(object):
         self.lib_pt[lib.partition].remove(lib)
 
 
+    def _rename_lib(self, lib, new_path):
+        self._remove_lib_from_lookup_dict(lib)
+        lib.path = new_path
+        self._add_lib_to_lookup_dict(lib)
+
+
     def add_lib(self, partition, path, elf):
         lib = ELFLinkData(partition, path, elf,
                           self.tagged_paths.get_path_tag_bit(path))
@@ -2102,7 +2297,7 @@ class ELFLinker(object):
             ignored_patt = ELFLinker._compile_path_matcher(
                 root, ignored_subdirs)
 
-        for path, elf in scan_elf_files(root, unzip_files):
+        for path, elf in scan_elf_files(root, unzip_files=unzip_files):
             # Ignore ELF files with unknown machine ID (eg. DSP).
             if elf.e_machine not in ELF.ELF_MACHINES:
                 continue
@@ -2194,12 +2389,21 @@ class ELFLinker(object):
             self._resolve_lib_deps(lib, resolver, generic_refs)
 
 
+    def _get_apex_bionic_search_paths(self, lib_dir):
+        return ['/apex/com.android.runtime/' + lib_dir + '/bionic']
+
+
+    def _get_apex_search_paths(self, lib_dir):
+        return ['/apex/' + name + '/' + lib_dir
+                for name in sorted(self.apex_module_names)]
+
+
     def _get_system_search_paths(self, lib_dir):
-        return [
-            '/system/' + lib_dir,
-            # To find violating dependencies to vendor partitions.
-            '/vendor/' + lib_dir,
-        ]
+        apex_lib_dirs = (self._get_apex_search_paths(lib_dir) +
+                         self._get_apex_bionic_search_paths(lib_dir))
+        system_lib_dirs = ['/system/' + lib_dir, '/system/product/' + lib_dir]
+        vendor_lib_dirs = ['/vendor/' + lib_dir]
+        return apex_lib_dirs + system_lib_dirs + vendor_lib_dirs
 
 
     def _get_vendor_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs):
@@ -2208,28 +2412,30 @@ class ELFLinker(object):
             '/vendor/' + lib_dir + '/egl',
             '/vendor/' + lib_dir,
         ]
-        system_lib_dirs = [
-            # For degenerated VNDK libs.
-            '/system/' + lib_dir,
-        ]
-        return vendor_lib_dirs + vndk_sp_dirs + vndk_dirs + system_lib_dirs
+        # For degenerated VNDK libs.
+        apex_lib_dirs = (self._get_apex_search_paths(lib_dir) +
+                         self._get_apex_bionic_search_paths(lib_dir))
+        system_lib_dirs = ['/system/' + lib_dir]
+        return (vendor_lib_dirs + vndk_sp_dirs + vndk_dirs + apex_lib_dirs +
+                system_lib_dirs)
 
 
     def _get_vndk_sp_search_paths(self, lib_dir, vndk_sp_dirs):
+        # To find missing dependencies or LL-NDK.
         fallback_lib_dirs = [
-            # To find missing VNDK-SP dependencies.
             '/vendor/' + lib_dir,
-            # To find missing VNDK-SP dependencies or LL-NDK.
             '/system/' + lib_dir,
         ]
+        fallback_lib_dirs += self._get_apex_search_paths(lib_dir)
+        fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir)
         return vndk_sp_dirs + fallback_lib_dirs
 
 
     def _get_vndk_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs):
-        fallback_lib_dirs = [
-            # To find missing VNDK dependencies or LL-NDK.
-            '/system/' + lib_dir,
-        ]
+        # To find missing dependencies or LL-NDK.
+        fallback_lib_dirs = ['/system/' + lib_dir]
+        fallback_lib_dirs += self._get_apex_search_paths(lib_dir)
+        fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir)
         return vndk_sp_dirs + vndk_dirs + fallback_lib_dirs
 
 
@@ -2353,6 +2559,22 @@ class ELFLinker(object):
                     lib.hide_dlopen_dep(dep)
 
 
+    def rewrite_apex_modules(self):
+        """Rename ELF files under `/system/apex/${name}.apex` to
+        `/apex/${name}/...` and collect apex module names."""
+        APEX_PREFIX = '/system/apex/'
+        APEX_PREFIX_LEN = len(APEX_PREFIX)
+        for lib in list(self.all_libs()):
+            if not lib.path.startswith(APEX_PREFIX):
+                continue
+
+            apex_name_end = lib.path.find('/', APEX_PREFIX_LEN)
+            apex_name = lib.path[APEX_PREFIX_LEN:apex_name_end]
+            self.apex_module_names.add(apex_name)
+
+            self._rename_lib(lib, '/apex/' + lib.path[APEX_PREFIX_LEN:])
+
+
     @staticmethod
     def _parse_action_on_ineligible_lib(arg):
         follow = False
@@ -2708,6 +2930,7 @@ class ELFLinker(object):
             for path in extra_deps:
                 graph.add_dlopen_deps(path)
 
+        graph.rewrite_apex_modules()
         graph.resolve_deps(generic_refs)
 
         return graph