diff options
Diffstat (limited to 'chromium/tools/merge_to_master.py')
-rwxr-xr-x | chromium/tools/merge_to_master.py | 427 |
1 files changed, 298 insertions, 129 deletions
diff --git a/chromium/tools/merge_to_master.py b/chromium/tools/merge_to_master.py index 9f8e07a..9a2e78d 100755 --- a/chromium/tools/merge_to_master.py +++ b/chromium/tools/merge_to_master.py @@ -21,153 +21,318 @@ import optparse import os import re import shutil +import subprocess import sys import merge_common AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.' +WEBVIEW_PROJECT = 'frameworks/webview' -def _MergeProjects(svn_revision, target): - """Merges the Chromium projects from master-chromium to target. +def _GetAbsPath(project): + """Returns the full path to a given project (either Chromium or Android).""" + if project in merge_common.ALL_PROJECTS: + abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project) + else: + abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project) + if not os.path.exists(abs_path): + raise merge_common.MergeError('Cannot find path ' + abs_path) + return abs_path - The larger projects' histories are flattened in the process. + +def _CheckoutSingleProject(project, target_branch): + """Checks out the tip of the target_branch into a local branch (merge-to-XXX). Args: - svn_revision: The SVN revision for the main Chromium repository + project: a Chromium project (., third_party/foo) or frameworks/webview. + target_branch: name of the target branch (in the goog remote). """ - for path in merge_common.PROJECTS_WITH_FLAT_HISTORY: - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - merge_common.GetCommandStdout(['git', 'remote', 'update', - 'goog', 'history'], cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'checkout', - '-b', 'merge-to-' + target, - '-t', 'goog/' + target], cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'fetch', 'history', - 'refs/archive/chromium-%s' % svn_revision], - cwd=dest_dir) - merge_sha1 = merge_common.GetCommandStdout(['git', 'rev-parse', - 'FETCH_HEAD'], - cwd=dest_dir).strip() - old_sha1 = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], - cwd=dest_dir).strip() + dest_dir = _GetAbsPath(project) + tracking_branch = 'goog/' + target_branch + logging.debug('Check out %-45s at %-16s', project, tracking_branch) + merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'], + cwd=dest_dir) + merge_common.GetCommandStdout(['git', 'checkout', + '-b', 'merge-to-' + target_branch, + '-t', tracking_branch], cwd=dest_dir) + + +def _FetchSingleProject(project, remote, remote_ref): + """Fetches a remote ref for the given project and returns the fetched SHA. + + Args: + project: a Chromium project (., third_party/foo) or frameworks/webview. + remote: Git remote name (goog for most projects, history for squashed ones). + remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX). + + Returns: + The SHA1 of the FETCH_HEAD. + """ + dest_dir = _GetAbsPath(project) + logging.debug('Fetch %-45s %s:%s', project, remote, remote_ref) + merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref], + cwd=dest_dir) + return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'], + cwd=dest_dir).strip() + + +def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten): + """Merges a single project at a given SHA. + + Args: + project: a Chromium project (., third_party/foo) or frameworks/webview. + merge_sha: the SHA to merge. + revision: Abbrev. commitish in the main Chromium repository. + target_branch: name of the target branch. + flatten: True: squash history while merging; False: perform a normal merge. + """ + dest_dir = _GetAbsPath(project) + if flatten: # Make the previous merges into grafts so we can do a correct merge. + old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], + cwd=dest_dir).strip() merge_log = os.path.join(dest_dir, '.merged-revisions') if os.path.exists(merge_log): shutil.copyfile(merge_log, os.path.join(dest_dir, '.git', 'info', 'grafts')) - if merge_common.GetCommandStdout(['git', 'rev-list', '-1', - 'HEAD..' + merge_sha1], cwd=dest_dir): - logging.debug('Merging project %s ...', path) - # Merge conflicts cause 'git merge' to return 1, so ignore errors - merge_common.GetCommandStdout(['git', 'merge', '--no-commit', '--squash', - merge_sha1], - cwd=dest_dir, ignore_errors=True) - dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(path, []) - if dirs_to_prune: - merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] + - dirs_to_prune, cwd=dest_dir) - merge_common.CheckNoConflictsAndCommitMerge( - 'Merge from Chromium at DEPS revision %s\n\n%s' % - (svn_revision, AUTOGEN_MESSAGE), cwd=dest_dir) - new_sha1 = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], - cwd=dest_dir).strip() - with open(merge_log, 'a+') as f: - f.write('%s %s %s\n' % (new_sha1, old_sha1, merge_sha1)) - merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'], - cwd=dest_dir) - merge_common.GetCommandStdout( - ['git', 'commit', '-m', - 'Record Chromium merge at DEPS revision %s\n\n%s' % - (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir) - else: - logging.debug('No new commits to merge in project %s', path) - - for path in merge_common.PROJECTS_WITH_FULL_HISTORY: - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'], - cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'checkout', - '-b', 'merge-to-' + target, - '-t', 'goog/' + target], cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'fetch', 'goog', - 'refs/archive/chromium-%s' % svn_revision], + + # Early out if there is nothing to merge. + if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', + 'HEAD..' + merge_sha], cwd=dest_dir): + logging.debug('No new commits to merge in project %s', project) + return + + logging.debug('Merging project %s (flatten: %s)...', project, flatten) + merge_cmd = ['git', 'merge', '--no-commit'] + merge_cmd += ['--squash'] if flatten else ['--no-ff'] + merge_cmd += [merge_sha] + # Merge conflicts cause 'git merge' to return 1, so ignore errors + merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True) + + if flatten: + dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, []) + if dirs_to_prune: + merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] + + dirs_to_prune, cwd=dest_dir) + + if project in merge_common.ALL_PROJECTS: + commit_msg = 'Merge from Chromium at DEPS revision %s' % revision + else: + commit_msg = 'Merge master-chromium into %s at %s' % (target_branch, + revision) + commit_msg += '\n\n' + AUTOGEN_MESSAGE + merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir) + + if flatten: + # Generate the new grafts file and commit it on top of the merge. + new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], + cwd=dest_dir).strip() + with open(merge_log, 'a+') as f: + f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha)) + merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'], cwd=dest_dir) - if merge_common.GetCommandStdout(['git', 'rev-list', '-1', - 'HEAD..FETCH_HEAD'], - cwd=dest_dir): - logging.debug('Merging project %s ...', path) - # Merge conflicts cause 'git merge' to return 1, so ignore errors - merge_common.GetCommandStdout(['git', 'merge', '--no-commit', '--no-ff', - 'FETCH_HEAD'], - cwd=dest_dir, ignore_errors=True) - merge_common.CheckNoConflictsAndCommitMerge( - 'Merge from Chromium at DEPS revision %s\n\n%s' % - (svn_revision, AUTOGEN_MESSAGE), cwd=dest_dir) - else: - logging.debug('No new commits to merge in project %s', path) - - -def _GetSVNRevision(commitish='history/master-chromium'): - logging.debug('Getting SVN revision ...') - commit = merge_common.GetCommandStdout([ - 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b', commitish]) - svn_revision = re.search(r'^git-svn-id: .*@([0-9]+)', commit, - flags=re.MULTILINE).group(1) - return svn_revision - - -def _MergeWithRepoProp(repo_prop_file, target): + merge_common.GetCommandStdout( + ['git', 'commit', '-m', + 'Record Chromium merge at DEPS revision %s\n\n%s' % + (revision, AUTOGEN_MESSAGE)], cwd=dest_dir) + + +def _IsAncestor(ref1, ref2, cwd): + """Checks whether ref1 is a ancestor of ref2 in the given Git repo.""" + cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2] + ret = subprocess.call(cmd, cwd=cwd) + if ret == 0: + return True + elif ret == 1: + return False + else: + raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A') + + +def _MergeChromiumProjects(revision, target_branch, repo_shas=None, + force=False): + """Merges the Chromium projects from master-chromium to target_branch. + + The larger projects' histories are flattened in the process. + When repo_shas != None, it checks that the SHAs of the projects in the + archive match exactly the SHAs of the projects in repo.prop. + + Args: + revision: Abbrev. commitish in the main Chromium repository. + target_branch: target branch name to merge and push to. + repo_shas: optional dict. of expected revisions (only for --repo-prop). + force: True: merge anyways using the SHAs from repo.prop; False: bail out if + projects mismatch (archive vs repo.prop). + """ + # Sync and checkout ToT for all projects (creating the merge-to-XXX branch) + # and fetch the archive snapshot. + fetched_shas = {} + remote_ref = 'refs/archive/chromium-%s' % revision + for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: + _CheckoutSingleProject(project, target_branch) + fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref) + for project in merge_common.PROJECTS_WITH_FULL_HISTORY: + _CheckoutSingleProject(project, target_branch) + fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref) + + if repo_shas: + project_shas_mismatch = False + for project, merge_sha in fetched_shas.items(): # the dict can be modified. + expected_sha = repo_shas.get(project) + if expected_sha != merge_sha: + logging.warn('The SHA for project %s specified in the repo.prop (%s) ' + 'and the one in the archive (%s) differ.', + project, expected_sha, merge_sha) + dest_dir = _GetAbsPath(project) + if expected_sha is None: + reason = 'cannot find a SHA in the repo.pro for %s' % project + elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir): + reason = 'the SHA in repo.prop is ahead of the SHA in the archive. ' + log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10', + '%s..%s' % (merge_sha, expected_sha)] + log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir) + reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd), + log_cmd_output) + elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir): + reason = 'The SHA is already merged in the archive' + else: + reason = 'The project history diverged. Consult your Git historian.' + + project_shas_mismatch = True + if force: + logging.debug('Merging the SHA in repo.prop anyways (due to --force)') + fetched_shas[project] = expected_sha + else: + logging.debug('Reason: %s', reason) + if not force and project_shas_mismatch: + raise merge_common.MergeError( + 'The revision of some projects in the archive is different from the ' + 'one provided in build.prop. See the log for more details. Re-run ' + 'with --force to continue.') + + for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: + _MergeSingleProject(project, fetched_shas[project], revision, target_branch, + flatten=True) + for project in merge_common.PROJECTS_WITH_FULL_HISTORY: + _MergeSingleProject(project, fetched_shas[project], revision, target_branch, + flatten=False) + + +def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'): + """Returns the abbrev. upstream SHA which closest to the given reference.""" + logging.debug('Getting upstream SHA for %s...', reference) + merge_common.GetCommandStdout(['git', 'remote', 'update', 'history']) + upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([ + 'git', 'merge-base', 'history/upstream-master', reference])) + + # Pedantic check: look for the existence of a merge commit which contains the + # |upstream_commit| in its message and is its children. + merge_parents = merge_common.GetCommandStdout([ + 'git', 'rev-list', reference, '--grep', upstream_commit, '--merges', + '--parents', '-1']) + if upstream_commit not in merge_parents: + raise merge_common.MergeError( + 'Found upstream commit %s, but the merge child (%s) could not be found ' + 'or is not a parent of the upstream SHA') + logging.debug('Found nearest Chromium revision %s', upstream_commit) + return upstream_commit + + +def _MergeWithRepoProp(repo_prop_file, target_branch, force): + """Performs a merge using a repo.prop file (from Android build waterfall). + + This does NOT merge (unless forced with force=True) the pinned + revisions in repo.prop, as a repo.prop can snapshot an intermediate state + (between two automerger cycles). Instead, this looks up the archived snapshot + (generated by the chromium->master-chromium auto-merger) which is closest to + the given repo.prop (following the main Chromium project) and merges that one. + If the projects revisions don't match, it fails with detailed error messages. + + Args: + repo_prop_file: Path to a downloaded repo.prop file. + target_branch: name of the target branch to merget to. + force: ignores the aforementioned check and merged anyways. + """ chromium_sha = None webview_sha = None + repo_shas = {} # 'project/path' -> 'sha' with open(repo_prop_file) as prop: for line in prop: - project, sha = line.split() - if project == 'platform/external/chromium_org-history': - chromium_sha = sha - elif project == 'platform/frameworks/webview': - webview_sha = sha + repo, sha = line.split() + # Translate the Android repo paths into the relative project paths used in + # merge_common (e.g., platform/external/chromium_org/foo -> foo). + m = ( + re.match(r'^platform/(frameworks/.+)$', repo) or + re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo)) + if m: + project = m.group(1) if m.group(1) else '.' # '.' = Main project. + repo_shas[project] = sha + + chromium_sha = repo_shas.get('.') + webview_sha = repo_shas.get(WEBVIEW_PROJECT) if not chromium_sha or not webview_sha: - logging.error('SHA1s for projects not found; invalid build.prop?') - return 1 - chromium_revision = _GetSVNRevision(chromium_sha) - logging.info('Merging Chromium at r%s and WebView at %s', chromium_revision, - webview_sha) - _MergeProjects(chromium_revision, target) + raise merge_common.MergeError('SHAs for projects not found; ' + 'invalid build.prop?') - dest_dir = os.path.join(os.environ['ANDROID_BUILD_TOP'], 'frameworks/webview') - merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'], - cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'checkout', - '-b', 'merge-to-' + target, - '-t', 'goog/' + target], cwd=dest_dir) - if merge_common.GetCommandStdout(['git', 'rev-list', '-1', - 'HEAD..' + webview_sha], cwd=dest_dir): - logging.debug('Creating merge for framework...') - # Merge conflicts cause 'git merge' to return 1, so ignore errors - merge_common.GetCommandStdout(['git', 'merge', '--no-commit', '--no-ff', - webview_sha], cwd=dest_dir, - ignore_errors=True) - merge_common.CheckNoConflictsAndCommitMerge( - 'Merge master-chromium into %s at r%s\n\n%s' % - (target, chromium_revision, AUTOGEN_MESSAGE), cwd=dest_dir) - upload = merge_common.GetCommandStdout(['git', 'push', 'goog', - 'HEAD:refs/for/' + target], - cwd=dest_dir) - logging.info(upload) - else: - logging.debug('No new commits to merge in framework') - return 0 + # Check that the revisions in repo.prop and the on in the archive match. + archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha) + logging.info('Merging Chromium at %s and WebView at %s', + archived_chromium_revision, webview_sha) + _MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas, + force) + + _CheckoutSingleProject(WEBVIEW_PROJECT, target_branch) + _MergeSingleProject(WEBVIEW_PROJECT, webview_sha, + archived_chromium_revision, target_branch, flatten=False) -def Push(target): - """Push the finished snapshot to the Android repository.""" - logging.debug('Pushing to server ...') - refspec = 'merge-to-%s:%s' % (target, target) +def Push(target_branch): + """Push the finished snapshot to the Android repository. + + Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists) + then wait for user confirmation and pushes the Chromium merges. This is to + give an opportunity to get a +2 for frameworks/webview and then push both + frameworks/webview and the Chromium projects atomically(ish). + + Args: + target_branch: name of the target branch (in the goog remote). + """ + merge_branch = 'merge-to-%s' % target_branch + + # Create a Gerrit CL for the frameworks/webview project (if needed). + dest_dir = _GetAbsPath(WEBVIEW_PROJECT) + did_upload_webview_cl = False + if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch], + cwd=dest_dir): + # Check that there was actually something to merge. + merge_range = 'goog/%s..%s' % (target_branch, merge_branch) + if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range], + cwd=dest_dir): + logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT) + refspec = '%s:refs/for/%s' % (merge_branch, target_branch) + upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec], + cwd=dest_dir) + logging.info(upload) + did_upload_webview_cl = True + + prompt_msg = 'About push the Chromium projects merge. ' + if not did_upload_webview_cl: + logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT) + else: + prompt_msg += ('At this point you should have the CL +2-ed and merge it ' + 'together with this push.') + prompt_msg += '\nPress "y" to continue: ' + if raw_input(prompt_msg) != 'y': + logging.warn('Push aborted by the user!') + return + + logging.debug('Pushing Chromium projects to %s ...', target_branch) + refspec = '%s:%s' % (merge_branch, target_branch) for path in merge_common.ALL_PROJECTS: logging.debug('Pushing %s', path) - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) + dest_dir = _GetAbsPath(path) # Delete the graft before pushing otherwise git will attempt to push all the # grafted-in objects to the server as well as the ones we want. graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts') @@ -177,22 +342,26 @@ def Push(target): cwd=dest_dir) - def main(): parser = optparse.OptionParser(usage='%prog [options]') parser.epilog = ('Takes the current master-chromium branch of the Chromium ' 'projects in Android and merges them into master to publish ' 'them.') parser.add_option( - '', '--svn_revision', '--release', + '', '--revision', default=None, - help=('Merge to the specified archived master-chromium SVN revision,' - 'rather than using HEAD.')) + help=('Merge to the specified archived master-chromium revision (abbrev. ' + 'SHA or release version) rather than using HEAD. e.g., ' + '--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24')) parser.add_option( '', '--repo-prop', default=None, metavar='FILE', help=('Merge to the revisions specified in this repo.prop file.')) parser.add_option( + '', '--force', + default=False, action='store_true', + help=('Skip history checks and merged anyways (only for --repo-prop).')) + parser.add_option( '', '--push', default=False, action='store_true', help=('Push the result of a previous merge to the server.')) @@ -211,13 +380,13 @@ def main(): if options.push: Push(options.target) elif options.repo_prop: - return _MergeWithRepoProp(os.path.expanduser(options.repo_prop), - options.target) - elif options.svn_revision: - _MergeProjects(options.svn_revision, options.target) + _MergeWithRepoProp(os.path.expanduser(options.repo_prop), + options.target, options.force) + elif options.revision: + _MergeChromiumProjects(options.revision, options.target) else: - svn_revision = _GetSVNRevision() - _MergeProjects(svn_revision, options.target) + first_upstream_sha = _GetNearestUpstreamAbbrevSHA() + _MergeChromiumProjects(first_upstream_sha, options.target) return 0 |