add script for looking for and copying missing files and dirty uprev

BUG=chromium:1048060
TEST=./libchrome_tools/uprev/copy_new_files.py --dry_run a0dacdac4496c584edc7ad103fa12a89f4d9e124
TEST=./libchrome_tools/uprev/dirty_uprev.py --is_browser --dry_run a0dacdac4496c584edc7ad103fa12a89f4d9e124 d971bc701bcd2032bb1ae788a8a52fa9db6d8682

Change-Id: I94ba35c892882c748a14d953b99451faf2e87603

4 files changed, 392 insertions, 0 deletions

diff --git a/libchrome_tools/uprev/copy_new_files.py b/libchrome_tools/uprev/copy_new_files.py
new file mode 100755
index 000000000..67da28f4a
--- /dev/null
+++ b/libchrome_tools/uprev/copy_new_files.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Utility to copy missing files from Chromium tree to Chromium OS libchrome tree
+based on hard coded rules.
+
+This utility is used to diff current HEAD against given commit in Chromium
+browser master branch, copy missing files after hard-coded filter rules and
+remove unnecessary files. libchrome original files in hard-coded filter rules
+will be untounched.
+"""
+
+import argparse
+import os
+import os.path
+import subprocess
+import sys
+
+import filters
+import utils
+
+def main():
+    # Init args
+    parser = argparse.ArgumentParser(
+        description='Copy file from given commits')
+    parser.add_argument(
+        'commit_hash',
+        metavar='commit',
+        type=str,
+        nargs=1,
+        help='commit hash to copy files from')
+    parser.add_argument(
+        '--dry_run',
+        dest='dry_run',
+        action='store_const',
+        const=True,
+        default=False)
+    arg = parser.parse_args(sys.argv[1:])
+
+    # Read file list from HEAD and upstream commit.
+    upstream_files = utils.get_file_list(arg.commit_hash[0])
+    our_files = utils.get_file_list('HEAD')
+
+    # Calculate target file list
+    target_files = filters.filter_file(our_files, upstream_files)
+
+    # Calculate operations needed
+    ops = utils.gen_op(our_files, target_files)
+
+    if arg.dry_run:
+        # Print ops only on dry-run mode.
+        print('\n'.join(repr(x) for x in ops))
+        return
+    for op, f in ops:
+        # Ignore if op is REP because we only want to copy missing files, not to
+        # revert custom Chromium OS libchrome patch.
+        assert type(op) == utils.DiffOperations
+        if op == utils.DiffOperations.DEL:
+            subprocess.check_call(['git', 'rm', f.path]),
+        elif op == utils.DiffOperations.ADD:
+            # Create directory recursively if not exist.
+            os.makedirs(os.path.dirname(f.path), exist_ok=True)
+            # Read file by git cat-file with blob object id to avoid heavy git checkout.
+            with open(f.path, 'wb') as outfile:
+                subprocess.check_call(['git', 'cat-file', 'blob', f.id],
+                                      stdout=outfile)
+            # Add to git index
+            subprocess.check_call(['git', 'add', f.path])
+
+if __name__ == '__main__':
+    main()
diff --git a/libchrome_tools/uprev/dirty_uprev.py b/libchrome_tools/uprev/dirty_uprev.py
new file mode 100755
index 000000000..aa8e2bd08
--- /dev/null
+++ b/libchrome_tools/uprev/dirty_uprev.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Utility to apply diffs between given two upstream commit hashes to current
+workspace.
+
+This utility diffs files between old_commit and new_commit, with hard-coded
+filter rules, and apply the diff to current HEAD with 3-way-merge supported.
+
+This can be used to uprev a libchrome directory when this is not git history for
+git merge to work.
+"""
+
+import argparse
+import subprocess
+import sys
+
+import filters
+import utils
+
+def main():
+    # Init args
+    parser = argparse.ArgumentParser(
+        description='Copy file from given commits')
+    parser.add_argument(
+        'old_commit', metavar='old_commit', type=str, nargs=1,
+        help='commit hash in upstream branch or browser repository '
+             'we want to uprev from')
+    parser.add_argument(
+        'new_commit', metavar='new_commit', type=str, nargs=1,
+        help='commit hash in upstream branch or browser repository '
+             'we want ot uprev to')
+    parser.add_argument(
+        '--dry_run', dest='dry_run', action='store_const', const=True, default=False)
+    parser.add_argument(
+        '--is_browser', dest='is_browser', action='store_const', const=True, default=False,
+        help='is the commit hash in browser repository')
+    arg = parser.parse_args(sys.argv[1:])
+
+    # Get old and new files.
+    old_files = utils.get_file_list(arg.old_commit[0])
+    new_files = utils.get_file_list(arg.new_commit[0])
+
+    if arg.is_browser:
+        old_files = filters.filter_file([], old_files)
+        new_files = filters.filter_file([], new_files)
+        assert filters.filter_file(old_files, []) == []
+        assert filters.filter_file(new_files, []) == []
+
+    # Generate a tree object for new files.
+    old_tree = utils.git_mktree(old_files)
+    new_tree = utils.git_mktree(new_files)
+    newroot = utils.git_commit(old_tree, [])
+    squashed = utils.git_commit(new_tree, [newroot])
+
+    # Generate patch for git am
+    patch = subprocess.check_output(['git', 'format-patch', '--stdout', newroot+b'..'+squashed])
+
+    if arg.dry_run:
+        print(patch.decode('utf-8'))
+    else:
+        subprocess.run(['git', 'am', '-3'], input=patch)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/libchrome_tools/uprev/filters.py b/libchrome_tools/uprev/filters.py
new file mode 100644
index 000000000..eaf829e65
--- /dev/null
+++ b/libchrome_tools/uprev/filters.py
@@ -0,0 +1,105 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Provide filters for libchrome tools."""
+
+import re
+
+# Libchrome wants WANT but mot WANT_EXCLUDE
+# aka files matching WANT will be copied from upstream_files
+WANT = [
+    re.compile(rb'base/((?!(allocator|third_party)/).*$)'),
+    re.compile(
+        rb'base/allocator/(allocator_shim.cc|allocator_shim_override_linker_wrapped_symbols.h|allocator_shim_override_cpp_symbols.h|allocator_shim_override_libc_symbols.h|allocator_shim_default_dispatch_to_glibc.cc|allocator_shim.h|allocator_shim_default_dispatch_to_linker_wrapped_symbols.cc|allocator_extension.cc|allocator_extension.h|allocator_shim_internals.h)$'
+    ),
+    re.compile(rb'base/third_party/(dynamic_annotation|icu|nspr|valgrind)'),
+    re.compile(rb'mojo/'),
+    re.compile(rb'dbus/'),
+    re.compile(rb'ipc/.*(\.cc|\.h|\.mojom)$'),
+    re.compile(rb'ui/gfx/(gfx_export.h|geometry|range)'),
+    re.compile(rb'testing/[^/]*\.(cc|h)$'),
+    re.compile(rb'third_party/(jinja2|markupsafe|ply)'),
+    re.compile(
+        rb'components/(json_schema|policy/core/common/[^/]*$|policy/policy_export.h|timers)'
+    ),
+    re.compile(
+        rb'device/bluetooth/bluetooth_(common|advertisement|uuid|export)\.*(h|cc)'
+    ),
+    re.compile(
+        rb'device/bluetooth/bluez/bluetooth_service_attribute_value_bluez.(h|cc)'
+    ),
+]
+
+# WANT_EXCLUDE will be excluded from WANT
+WANT_EXCLUDE = [
+    re.compile(rb'(.*/)?BUILD.gn$'),
+    re.compile(rb'(.*/)?PRESUBMIT.py$'),
+    re.compile(rb'(.*/)?OWNERS$'),
+    re.compile(rb'(.*/)?SECURITY_OWNERS$'),
+    re.compile(rb'(.*/)?DEPS$'),
+    re.compile(rb'base/(.*/)?(ios|win|fuchsia|mac|openbsd|freebsd|nacl)/.*'),
+    re.compile(rb'.*_(ios|win|mac|fuchsia|openbsd|freebsd|nacl)[_./]'),
+    re.compile(rb'.*/(ios|win|mac|fuchsia|openbsd|freebsd|nacl)_'),
+    re.compile(rb'base/test/android'),
+    re.compile(rb'base/android/javatests'),
+    re.compile(rb'dbus/(test_serv(er|ice)\.cc|test_service\.h)$')
+]
+
+# Files matching KEEP should not be touched.
+# aka files matching KEEP will keep its our_files version,
+# and it will be kept even it doesn't exist in upstream.
+# KEEP-KEEP_EXCLUDE must NOT intersect with WANT-WANT_EXCLUDE
+KEEP = [
+    re.compile(
+        b'(Android.bp|build|BUILD.gn|crypto|libchrome_tools|MODULE_LICENSE_BSD|NOTICE|OWNERS|PRESUBMIT.cfg|soong|testrunner.cc|third_party)(/.*)?$'
+    ),
+    re.compile(rb'[^/]*$'),
+    re.compile(rb'.*buildflags.h'),
+    re.compile(rb'base/android/java/src/org/chromium/base/BuildConfig.java'),
+    re.compile(rb'testing/(gmock|gtest)/'),
+    re.compile(rb'base/third_party/(libevent|symbolize)'),
+]
+
+# KEEP_EXCLUDE wil be excluded from KEEP.
+KEEP_EXCLUDE = [
+    re.compile(rb'third_party/(jinja2|markupsafe|ply)'),
+]
+
+
+def filter_file(our_files, upstream_files):
+    """
+    Generates a list of files we want based on hard-coded rules.
+
+    File list must be a list of GitFile.
+
+    our_files: files in Chromium OS libchrome repository.
+    upstream_files: files in Chromium browser repository.
+    """
+
+    files = []
+    for upstream_file in upstream_files:
+        wanted = False
+        for want_file_regex in WANT:
+            if want_file_regex.match(upstream_file.path):
+                wanted = True
+                break
+        for exclude_file_regex in WANT_EXCLUDE:
+            if exclude_file_regex.match(upstream_file.path):
+                wanted = False
+                break
+        if wanted:
+            files.append(upstream_file)
+    for our_file in our_files:
+        keep = False
+        for keep_file_regex in KEEP:
+            if keep_file_regex.match(our_file.path):
+                keep = True
+                break
+        for exclude_file_regex in KEEP_EXCLUDE:
+            if exclude_file_regex.match(our_file.path):
+                keep = False
+                break
+        if keep:
+            files.append(our_file)
+    return files
diff --git a/libchrome_tools/uprev/utils.py b/libchrome_tools/uprev/utils.py
new file mode 100644
index 000000000..5fdff5d8a
--- /dev/null
+++ b/libchrome_tools/uprev/utils.py
@@ -0,0 +1,144 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Provide some basic utility functions for libchrome tools."""
+
+import collections
+import enum
+import os
+import re
+import subprocess
+
+class DiffOperations(enum.Enum):
+    """
+    Describes operations on files
+    """
+    ADD = 1
+    DEL = 2
+    REP = 3
+
+GitFile = collections.namedtuple(
+    'GitFile',
+    ['path', 'mode', 'id',]
+)
+
+
+def _reverse(files):
+    """
+    Creates a reverse map from file path to file.
+    Assert if a file path exist only once in files.
+
+    files: list of files.
+    """
+    files_map = {}
+    for i in files:
+        if i.path in files_map:
+            assert i.path not in files_map
+        files_map[i.path] = i
+    return files_map
+
+
+def get_file_list(commit):
+    """
+    Gets list of files of a commit
+
+    commit: commit hash or refs
+    """
+
+    output = subprocess.check_output(['git', 'ls-tree', '-r',
+                                      commit]).split(b'\n')
+    files = []
+    # Line looks like
+    # mode<space>type<space>id<tab>file name
+    # split by tab first, and by space.
+    re_line = re.compile(rb'^([^ ]*) ([^ ]*) ([^ ]*)\t(.*)$')
+    for line in output:
+        if not line:
+            continue
+        match = re_line.match(line)
+        mode, gittype, blobhash, path = match.groups()
+        if gittype == b'commit':
+            continue
+        assert gittype == b'blob', '%s\n\n%s' % (str(output), line)
+        files.append(GitFile(path, mode, blobhash))
+    return files
+
+
+def gen_op(current_files, target_files):
+    """
+    Generates list of operations (add/delete/replace files) if we want to
+    convert current_files in directory to target_files
+
+    current_files: list of files in current directory.
+    target_files: list of files we want it to be in current directory.
+    """
+    current_file_map = _reverse(current_files)
+    target_file_map = _reverse(target_files)
+    op = []
+    for i in sorted(current_file_map):
+        if i not in target_file_map:
+            op.append((DiffOperations.DEL, current_file_map[i]))
+    for i in sorted(target_file_map):
+        if i in current_file_map and current_file_map[i] != target_file_map[i]:
+            op.append((DiffOperations.REP, target_file_map[i]))
+        elif i not in current_file_map:
+            op.append((DiffOperations.ADD, target_file_map[i]))
+    return op
+
+
+def git_mktree(files):
+    """
+    Returns a git tree object hash after mktree recursively
+    """
+
+    def recursive_default_dict():
+        return collections.defaultdict(recursive_default_dict)
+
+    tree = recursive_default_dict()
+    for f in files:
+        directories = f.path.split(b'/')
+        directories, filename = directories[:-1], directories[-1]
+        cwd = tree
+        for directory in directories:
+            # If cwd is a GitFile, which means a file and a directory shares the
+            # same name.
+            assert type(cwd) == collections.defaultdict
+            cwd = cwd[directory]
+        assert filename not in cwd
+        cwd[filename] = f
+
+    def _mktree(prefix, node):
+        objects = []
+        for name, val in node.items():
+            prefix.append(name)
+            if isinstance(val, collections.defaultdict):
+                tree_hash = _mktree(prefix, val)
+                objects.append(b'\t'.join(
+                    [b' '.join([b'040000', b'tree', tree_hash]), name]))
+            else:
+                path = b'/'.join(prefix)
+                assert path == val.path, '%s\n%s' % (str(path), str(val.path))
+                objects.append(b'\t'.join(
+                    [b' '.join([val.mode, b'blob', val.id]), name]))
+            prefix.pop(-1)
+        return subprocess.check_output(['git', 'mktree'],
+                                       input=b'\n'.join(objects)).strip(b'\n')
+
+    return _mktree([], tree)
+
+
+def git_commit(tree, parents):
+    """
+    Create commit
+
+    tree: tree object id
+    parents: parent commit id
+    """
+    parent_args = []
+    for parent in parents:
+        parent_args.append('-p')
+        parent_args.append(parent)
+    return subprocess.check_output(
+        ['git', 'commit-tree', tree] + parent_args,
+        stdin=subprocess.DEVNULL).strip(b'\n')