From 753233e1666d4a8f69a4fae4345f9590f19589ef Mon Sep 17 00:00:00 2001 From: Primiano Tucci Date: Mon, 25 Aug 2014 21:26:24 +0100 Subject: Update merge_from_chromium.py after Chromium Git migration. Chromium has switched to Git. SVN is not the source of truth anymore. The following change removes --svn support from merge_from_chromium.py and makes it understand git. Also, this is removing the --lkgr option, as LKGR is being deprecated in chromium. Change-Id: I879e9cda2cee313f1e2409a1a85b173db59efd8c --- chromium/tools/merge_from_chromium.py | 180 +++++++++++++--------------------- 1 file changed, 69 insertions(+), 111 deletions(-) (limited to 'chromium/tools') diff --git a/chromium/tools/merge_from_chromium.py b/chromium/tools/merge_from_chromium.py index 6a7a67c..0ab20f9 100755 --- a/chromium/tools/merge_from_chromium.py +++ b/chromium/tools/merge_from_chromium.py @@ -34,7 +34,7 @@ webview_licenses = None AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master' - +GIT_ABBREV_LENGTH = 12 def _ReadGitFile(sha1, path, git_url=None, git_branch=None): """Reads a file from a (possibly remote) git project at a specific revision. @@ -53,12 +53,12 @@ def _ReadGitFile(sha1, path, git_url=None, git_branch=None): def _ParseDEPS(deps_content): - """Parses the .DEPS.git file from Chromium and returns its contents. + """Parses the DEPS file from Chromium and returns its contents. Args: - deps_content: The contents of the .DEPS.git file as text. + deps_content: The contents of the DEPS file as text. Returns: - A dictionary of the contents of .DEPS.git at the specified revision + A dictionary of the contents of DEPS at the specified revision """ class FromImpl(object): @@ -91,15 +91,15 @@ def _ParseDEPS(deps_content): def _GetProjectMergeInfo(projects, deps_vars): - """Gets the git URL and SHA1 for each project based on .DEPS.git. + """Gets the git URL and SHA1 for each project based on DEPS. Args: projects: The list of projects to consider. - deps_vars: The dictionary of dependencies from .DEPS.git. + deps_vars: The dictionary of dependencies from DEPS. Returns: A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' Raises: - TemporaryMergeError: if a project to be merged is not found in .DEPS.git. + TemporaryMergeError: if a project to be merged is not found in DEPS. """ deps_fallback_order = [ deps_vars['deps'], @@ -118,7 +118,7 @@ def _GetProjectMergeInfo(projects, deps_vars): break else: raise merge_common.TemporaryMergeError( - 'Could not find .DEPS.git entry for project %s. This probably ' + 'Could not find DEPS entry for project %s. This probably ' 'means that the project list in merge_from_chromium.py needs to be ' 'updated.' % path) match = re.match('(.*?)@(.*)', url_plus_sha1) @@ -132,7 +132,7 @@ def _GetProjectMergeInfo(projects, deps_vars): def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): """Merges each required Chromium project into the Android repository. - .DEPS.git is consulted to determine which revision each project must be merged + DEPS is consulted to determine which revision each project must be merged at. Only a whitelist of required projects are merged. Args: @@ -155,8 +155,10 @@ def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): logging.debug('Parsing DEPS ...') if root_sha1: - deps_content = _ReadGitFile(root_sha1, '.DEPS.git') + deps_content = _ReadGitFile(root_sha1, 'DEPS') else: + # TODO: At some point the release branches will use DEPS as well, instead of + # .DEPS.git. Rename below when that day will come. deps_content = _ReadGitFile('FETCH_HEAD', 'releases/' + version + '/.DEPS.git', buildspec_url, @@ -206,13 +208,18 @@ def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): url = merge_info['']['url'] root_sha1 = merge_info['']['sha1'] merge_common.GetCommandStdout(['git', 'fetch', url, root_sha1]) + merge_msg_version = '%s (%s)' % (version, root_sha1) + else: + merge_msg_version = root_sha1 + + logging.debug('Merging Chromium at %s ...', root_sha1) # Merge conflicts make git merge return 1, so ignore errors merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1], ignore_errors=True) merge_common.CheckNoConflictsAndCommitMerge( - 'Merge Chromium at %s (%s)\n\n%s' - % (version, root_sha1, AUTOGEN_MESSAGE), unattended=unattended) + 'Merge Chromium at %s\n\n%s' + % (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended) logging.debug('Getting directories to exclude ...') @@ -331,19 +338,19 @@ def _GenerateNoticeFile(version): % (version, AUTOGEN_MESSAGE)]) -def _GenerateLastChange(version): +def _GenerateLastChange(version, root_sha1): """Write a build/util/LASTCHANGE file containing the current revision. The revision number is compiled into the binary at build time from this file. Args: version: The version to mention in generated commit messages. + root_sha1: The SHA1 of the main project (before the merge). """ logging.debug('Updating LASTCHANGE ...') - svn_revision, sha1 = _GetSVNRevisionAndSHA1('HEAD', 'HEAD') with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 'w') as f: - f.write('LASTCHANGE=%s\n' % svn_revision) + f.write('LASTCHANGE=%s\n' % root_sha1[:GIT_ABBREV_LENGTH]) merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) logging.debug('Updating LASTCHANGE.blink ...') with open(os.path.join(merge_common.REPOSITORY_ROOT, @@ -358,27 +365,13 @@ def _GenerateLastChange(version): % (version, AUTOGEN_MESSAGE)]) -def GetLKGR(): - """Fetch the last known good release from Chromium's dashboard. - - Returns: - The last known good SVN revision. - """ - with contextlib.closing( - urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr: - return int(lkgr.read()) - - def GetHEAD(): - """Fetch the latest HEAD revision from the git mirror of the Chromium svn - repo. + """Fetch the latest HEAD revision from the Chromium Git mirror. Returns: - The latest HEAD SVN revision. + The latest HEAD revision (A Git abbrev SHA1). """ - (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(SRC_GIT_BRANCH, - 'HEAD') - return int(svn_revision) + return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD') def _ParseSvnRevisionFromGitCommitMessage(commit_message): @@ -386,60 +379,44 @@ def _ParseSvnRevisionFromGitCommitMessage(commit_message): flags=re.MULTILINE).group(1) -def _GetSVNRevisionFromSha(sha1): - commit = merge_common.GetCommandStdout([ - 'git', 'show', '--format=%H%n%b', sha1]) - return _ParseSvnRevisionFromGitCommitMessage(commit) +def _GetGitAbbrevSHA1(git_branch, revision): + assert(revision) + logging.debug('Getting Git revision for %s ...' % revision) + + upstream = git_branch if revision == 'HEAD' else revision + # Make sure the |upstream| Git object has been mirrored. + try: + merge_common.GetCommandStdout([ + 'git', 'merge-base', '--is-ancestor', upstream, git_branch]) + except merge_common.CommandError: + raise merge_common.TemporaryMergeError( + 'Upstream object (%s) not reachable from %s' % (upstream, git_branch)) -def _GetSVNRevisionAndSHA1(git_branch, svn_revision): - logging.debug('Getting SVN revision and SHA1 ...') - - if svn_revision == 'HEAD': - # Just use the latest commit. - commit = merge_common.GetCommandStdout([ - 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b', - git_branch]) - sha1 = commit.split()[0] - svn_revision = _ParseSvnRevisionFromGitCommitMessage(commit) - return (svn_revision, sha1) - - if svn_revision is None: - # Fetch LKGR from upstream. - svn_revision = GetLKGR() - output = merge_common.GetCommandStdout([ - 'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision, - '--format=%H', git_branch]) - if not output: - raise merge_common.TemporaryMergeError('Revision %s not found in git repo.' - % svn_revision) - # The log grep will sometimes match reverts/reapplies of commits. We take the - # oldest (last) match because the first time it appears in history is - # overwhelmingly likely to be the correct commit. - sha1 = output.split()[-1] - return (svn_revision, sha1) + abbrev_sha = merge_common.GetCommandStdout(['git', 'rev-list', + '--abbrev-commit', '--abbrev=%d' % GIT_ABBREV_LENGTH, + '--max-count=1', upstream]) + return abbrev_sha.split()[0] def _GetBlinkRevision(): + # TODO: Switch to Git as soon as Blink gets migrated as well. commit = merge_common.GetCommandStdout([ 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'], cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit')) return _ParseSvnRevisionFromGitCommitMessage(commit) -def Snapshot(svn_revision, root_sha1, release, target, unattended, - buildspec_url): +def Snapshot(root_sha1, release, target, unattended, buildspec_url): """Takes a snapshot of the Chromium tree and merges it into Android. Android makefiles and a top-level NOTICE file are generated and committed after the merge. Args: - svn_revision: The SVN revision in the Chromium repository to merge from. - root_sha1: The sha1 in the Chromium git mirror to merge from. + root_sha1: The abbrev sha1 in the Chromium git mirror to merge from. release: The Chromium release version to merge from (e.g. "30.0.1599.20"). - Only one of svn_revision, root_sha1 and release should be - specified. + Only one of root_sha1 and release should be specified. target: The target branch to merge to. unattended: Run in unattended mode. buildspec_url: URL for buildspec repository, used when merging a release. @@ -447,24 +424,19 @@ def Snapshot(svn_revision, root_sha1, release, target, unattended, Returns: True if new commits were merged; False if no new commits were present. """ - if svn_revision: - svn_revision, root_sha1 = _GetSVNRevisionAndSHA1(SRC_GIT_BRANCH, - svn_revision) - elif root_sha1: - svn_revision = _GetSVNRevisionFromSha(root_sha1) - - if svn_revision and root_sha1: - version = svn_revision - if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', - 'HEAD..' + root_sha1]): - logging.info('No new commits to merge at %s (%s)', - svn_revision, root_sha1) - return False - elif release: - version = release + if release: root_sha1 = None + version = release else: - raise merge_common.MergeError('No merge source specified') + root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1) + version = root_sha1 + + assert((root_sha1 is not None and len(root_sha1) > 6) or version == release) + + if root_sha1 and not merge_common.GetCommandStdout( + ['git', 'rev-list', '-1', 'HEAD..' + root_sha1]): + logging.info('No new commits to merge at %s (%s)' % (version, root_sha1)) + return False logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1) @@ -481,7 +453,7 @@ def Snapshot(svn_revision, root_sha1, release, target, unattended, _GenerateNoticeFile(version) # 5. Generate LASTCHANGE file - _GenerateLastChange(version) + _GenerateLastChange(version, root_sha1) return True @@ -513,29 +485,21 @@ def Push(version, target): def main(): parser = optparse.OptionParser(usage='%prog [options]') parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' - 'Chromium SVN revision and merges it into this repository. ' + 'Chromium Git revision and merges it into this repository. ' 'Paths marked as excluded for license reasons are removed ' 'as part of the merge. Also generates Android makefiles and ' 'generates a top-level NOTICE file suitable for use in the ' 'Android build.') - parser.add_option( - '', '--svn_revision', - default=None, - help=('Merge to the specified chromium SVN revision, rather than using ' - 'the current LKGR. Can also pass HEAD to merge from tip of tree. ' - 'Only one of svn_revision, sha1 and release should be specified')) parser.add_option( '', '--sha1', - default=None, - help=('Merge to the specified chromium sha1 revision from ' + SRC_GIT_BRANCH - + ' branch, rather than using the current LKGR. Only one of' - 'svn_revision, sha1 and release should be specified.')) + default='HEAD', + help=('Merge to the specified chromium sha1 revision from ' + + SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.')) parser.add_option( '', '--release', default=None, - help=('Merge to the specified chromium release buildspec (e.g. ' - '"30.0.1599.20"). Only one of svn_revision, sha1 and release ' - 'should be specified.')) + help=('Merge to the specified chromium release buildspec (e.g., "30.0.' + '1599.20"). Only one of --sha1 and --release should be specified')) parser.add_option( '', '--buildspec_url', default=None, @@ -548,11 +512,7 @@ def main(): '', '--push', default=False, action='store_true', help=('Push the result of a previous merge to the server. Note ' - 'svn_revision must be given.')) - parser.add_option( - '', '--get_lkgr', - default=False, action='store_true', - help=('Just print the current LKGR on stdout and exit.')) + '--sha1 must be given.')) parser.add_option( '', '--get_head', default=False, action='store_true', @@ -577,22 +537,20 @@ def main(): logging.basicConfig(format='%(message)s', level=logging.DEBUG, stream=sys.stdout) - if options.get_lkgr: - print GetLKGR() - elif options.get_head: + if options.get_head: logging.disable(logging.CRITICAL) # Prevent log messages print GetHEAD() elif options.push: if options.release: Push(options.release, options.target) - elif options.svn_revision: - Push(options.svn_revision, options.target) + elif options.sha1: + Push(options.sha1, options.target) else: print >>sys.stderr, 'You need to pass the version to push.' return 1 else: - if not Snapshot(options.svn_revision, options.sha1, options.release, - options.target, options.unattended, options.buildspec_url): + if not Snapshot(options.sha1, options.release, options.target, + options.unattended, options.buildspec_url): return options.no_changes_exit return 0 -- cgit v1.2.3