diff options
author | Vignesh Venkatasubramanian <vigneshv@google.com> | 2023-06-12 21:28:09 +0000 |
---|---|---|
committer | Android Build Cherrypicker Worker <android-build-cherrypicker-worker@google.com> | 2023-06-12 21:28:09 +0000 |
commit | 01fadae8618ea360608cc52c823a022c48f855b5 (patch) | |
tree | 303e8eabb435a80ebec36a19c23ca85744d58fb5 /files/tools_libyuv | |
parent | 435db9f11b09187e0d60683813a28d07cc13166b (diff) | |
download | libyuv-01fadae8618ea360608cc52c823a022c48f855b5.tar.gz |
libyuv: Update to r1871 (2a6cb743)
Changes from upstream:
https://chromium.googlesource.com/libyuv/libyuv/+log/d53f1bee..2a6cb743
The intention of the CL is to import the functions necessary to
enable AV1 (and AVIF) 12-bit color conversion.
Bug: 268505204
Test: Builds. Media and Camera CTS tests pass.
(cherry picked from https://googleplex-android-review.googlesource.com/q/commit:27750a13c6eaacb9f716da3fe1734a8d106d7ff4)
Merged-In: I756d3bd5047d4719659f9e1a449217b1940e51a4
Change-Id: I756d3bd5047d4719659f9e1a449217b1940e51a4
Diffstat (limited to 'files/tools_libyuv')
-rwxr-xr-x | files/tools_libyuv/autoroller/roll_deps.py | 582 |
1 files changed, 447 insertions, 135 deletions
diff --git a/files/tools_libyuv/autoroller/roll_deps.py b/files/tools_libyuv/autoroller/roll_deps.py index 977c86de..2b57eb65 100755 --- a/files/tools_libyuv/autoroller/roll_deps.py +++ b/files/tools_libyuv/autoroller/roll_deps.py @@ -1,18 +1,14 @@ #!/usr/bin/env vpython3 -# Copyright 2017 The LibYuv Project Authors. All rights reserved. +# Copyright (c) 2017 The LibYUV project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license # that can be found in the LICENSE file in the root of the source # tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may +# in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +"""Script to automatically roll dependencies in the LibYUV DEPS file.""" -# This is a modified copy of the script in -# https://webrtc.googlesource.com/src/+/master/tools_webrtc/autoroller/roll_deps.py -# customized for libyuv. - -"""Script to automatically roll dependencies in the libyuv DEPS file.""" import argparse import base64 @@ -25,9 +21,46 @@ import sys import urllib.request +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + # Skip these dependencies (list without solution name prefix). DONT_AUTOROLL_THESE = [ - 'src/third_party/gflags/src', + 'src/third_party/gflags/src', + 'src/third_party/mockito/src', +] + +# These dependencies are missing in chromium/src/DEPS, either unused or already +# in-tree. For instance, src/base is a part of the Chromium source git repo, +# but we pull it through a subtree mirror, so therefore it isn't listed in +# Chromium's deps but it is in ours. +LIBYUV_ONLY_DEPS = [ + 'src/base', + 'src/build', + 'src/buildtools', + 'src/ios', + 'src/testing', + 'src/third_party', + 'src/third_party/android_support_test_runner', + 'src/third_party/bazel', + 'src/third_party/bouncycastle', + 'src/third_party/errorprone/lib', + 'src/third_party/findbugs', + 'src/third_party/gson', + 'src/third_party/gtest-parallel', + 'src/third_party/guava', + 'src/third_party/intellij', + 'src/third_party/jsr-305/src', + 'src/third_party/ow2_asm', + 'src/third_party/proguard', + 'src/third_party/ub-uiautomator/lib', + 'src/tools', + 'src/tools/clang/dsymutil', ] LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' @@ -37,16 +70,22 @@ CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') -CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([0-9a-z-]+)\'$') +CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$') ROLL_BRANCH_NAME = 'roll_chromium_revision' SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, - os.pardir)) +CHECKOUT_SRC_DIR = FindSrcDirPath() CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) +# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. +ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' +ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' +# Location of automically gathered android deps. +ANDROID_DEPS_PATH = 'src/third_party/android_deps/' + sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) -import find_depot_tools # pylint: disable=wrong-import-position +import find_depot_tools + find_depot_tools.add_depot_tools_to_path() CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' @@ -56,11 +95,26 @@ CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') ChangedDep = collections.namedtuple('ChangedDep', 'path url current_rev new_rev') +CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') +VersionEntry = collections.namedtuple('VersionEntry', 'version') +ChangedCipdPackage = collections.namedtuple( + 'ChangedCipdPackage', 'path package current_version new_version') +ChangedVersionEntry = collections.namedtuple( + 'ChangedVersionEntry', 'path current_version new_version') + +ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', + ('current_chromium_rev ' + 'new_chromium_rev ')) + class RollError(Exception): pass +def StrExpansion(): + return lambda str_value: str_value + + def VarLookup(local_scope): return lambda var_name: local_scope['vars'][var_name] @@ -68,9 +122,9 @@ def VarLookup(local_scope): def ParseDepsDict(deps_content): local_scope = {} global_scope = { - 'Var': VarLookup(local_scope), - 'Str': lambda s: s, - 'deps_os': {}, + 'Str': StrExpansion(), + 'Var': VarLookup(local_scope), + 'deps_os': {}, } exec(deps_content, global_scope, local_scope) return local_scope @@ -82,11 +136,6 @@ def ParseLocalDepsFile(filename): return ParseDepsDict(deps_content) -def ParseRemoteCrDepsFile(revision): - deps_content = ReadRemoteCrFile('DEPS', revision) - return ParseDepsDict(deps_content) - - def ParseCommitPosition(commit_message): for line in reversed(commit_message.splitlines()): m = COMMIT_POSITION_RE.match(line.strip()) @@ -97,15 +146,18 @@ def ParseCommitPosition(commit_message): sys.exit(-1) -def _RunCommand(command, working_dir=None, ignore_exit_code=False, - extra_env=None, input_data=None): +def _RunCommand(command, + working_dir=None, + ignore_exit_code=False, + extra_env=None, + input_data=None): """Runs a command and returns the output from that command. - If the command fails (exit code != 0), the function will exit the process. + If the command fails (exit code != 0), the function will exit the process. - Returns: - A tuple containing the stdout and stderr outputs as strings. - """ + Returns: + A tuple containing the stdout and stderr outputs as strings. + """ working_dir = working_dir or CHECKOUT_SRC_DIR logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) env = os.environ.copy() @@ -134,9 +186,9 @@ def _RunCommand(command, working_dir=None, ignore_exit_code=False, def _GetBranches(): """Returns a tuple of active,branches. - The 'active' is the name of the currently active branch and 'branches' is a - list of all branches. - """ + The 'active' is the name of the currently active branch and 'branches' is a + list of all branches. + """ lines = _RunCommand(['git', 'branch'])[0].split('\n') branches = [] active = '' @@ -160,9 +212,16 @@ def _ReadGitilesContent(url): def ReadRemoteCrFile(path_below_src, revision): - """Reads a remote Chromium file of a specific revision. Returns a string.""" - return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, - path_below_src)) + """Reads a remote Chromium file of a specific revision. + + Args: + path_below_src: A path to the target file relative to src dir. + revision: Revision to read. + Returns: + A string with file content. + """ + return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % + (revision, path_below_src)) def ReadRemoteCrCommit(revision): @@ -171,7 +230,13 @@ def ReadRemoteCrCommit(revision): def ReadUrlContent(url): - """Connect to a remote host and read the contents. Returns a list of lines.""" + """Connect to a remote host and read the contents. + + Args: + url: URL to connect to. + Returns: + A list of lines. + """ conn = urllib.request.urlopen(url) try: return conn.readlines() @@ -185,52 +250,172 @@ def ReadUrlContent(url): def GetMatchingDepsEntries(depsentry_dict, dir_path): """Gets all deps entries matching the provided path. - This list may contain more than one DepsEntry object. - Example: dir_path='src/testing' would give results containing both - 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. - Example 2: dir_path='src/build' should return 'src/build' but not - 'src/buildtools'. + This list may contain more than one DepsEntry object. + Example: dir_path='src/testing' would give results containing both + 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's + DEPS. + Example 2: dir_path='src/build' should return 'src/build' but not + 'src/buildtools'. - Returns: - A list of DepsEntry objects. - """ + Returns: + A list of DepsEntry objects. + """ result = [] for path, depsentry in depsentry_dict.items(): if path == dir_path: result.append(depsentry) else: parts = path.split('/') - if all(part == parts[i] - for i, part in enumerate(dir_path.split('/'))): + if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): result.append(depsentry) return result + def BuildDepsentryDict(deps_dict): """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" result = {} def AddDepsEntries(deps_subdict): - for path, deps_url_spec in deps_subdict.items(): - if isinstance(deps_url_spec, dict): - if deps_url_spec.get('dep_type') == 'cipd': - continue - deps_url = deps_url_spec['url'] + for path, dep in deps_subdict.items(): + if path in result: + continue + if not isinstance(dep, dict): + dep = {'url': dep} + if dep.get('dep_type') == 'cipd': + result[path] = CipdDepsEntry(path, dep['packages']) else: - deps_url = deps_url_spec - if not path in result: - url, revision = deps_url.split('@') if deps_url else (None, None) + if '@' not in dep['url']: + continue + url, revision = dep['url'].split('@') result[path] = DepsEntry(path, url, revision) + def AddVersionEntry(vars_subdict): + for key, value in vars_subdict.items(): + if key in result: + continue + if not key.endswith('_version'): + continue + key = re.sub('_version$', '', key) + result[key] = VersionEntry(value) + AddDepsEntries(deps_dict['deps']) - for deps_os in ['win', 'mac', 'linux', 'android', 'ios', 'unix']: + for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) + AddVersionEntry(deps_dict.get('vars', {})) return result +def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): + old_pkgs_names = {p['package'] for p in old_pkgs} + new_pkgs_names = {p['package'] for p in new_pkgs} + pkgs_equal = (old_pkgs_names == new_pkgs_names) + added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] + removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] + + assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' + 'and remove/add entries in DEPS so the old and new ' + 'list match.\nMost likely, you should add \"%s\" and ' + 'remove \"%s\"' % + (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) + + for old_pkg in old_pkgs: + for new_pkg in new_pkgs: + old_version = old_pkg['version'] + new_version = new_pkg['version'] + if (old_pkg['package'] == new_pkg['package'] + and old_version != new_version): + logging.debug('Roll dependency %s to %s', path, new_version) + yield ChangedCipdPackage(path, old_pkg['package'], old_version, + new_version) + + +def _FindChangedVars(name, old_version, new_version): + if old_version != new_version: + logging.debug('Roll dependency %s to %s', name, new_version) + yield ChangedVersionEntry(name, old_version, new_version) + + +def _FindNewDeps(old, new): + """ Gather dependencies only in `new` and return corresponding paths. """ + old_entries = set(BuildDepsentryDict(old)) + new_entries = set(BuildDepsentryDict(new)) + return [ + path for path in new_entries - old_entries + if path not in DONT_AUTOROLL_THESE + ] + + +def FindAddedDeps(libyuv_deps, new_cr_deps): + """ + Calculate new deps entries of interest. + + Ideally, that would mean: only appearing in chromium DEPS + but transitively used in LibYUV. + + Since it's hard to compute, we restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + Otherwise, assumes that's a Chromium-only dependency. + + Args: + libyuv_deps: dict of deps as defined in the LibYUV DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a new package in existing dep. + + Returns: + A tuple consisting of: + A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. + A list of paths for other added dependencies. + """ + all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps) + generated_android_deps = [ + path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) + ] + other_deps = [ + path for path in all_added_deps if path not in generated_android_deps + ] + return generated_android_deps, other_deps + + +def FindRemovedDeps(libyuv_deps, new_cr_deps): + """ + Calculate obsolete deps entries. + + Ideally, that would mean: no more appearing in chromium DEPS + and not used in LibYUV. + + Since it's hard to compute: + 1/ We restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + 2/ We rely on existing behavior of CalculateChangeDeps. + I.e. Assumes non-CIPD dependencies are LibYUV-only, don't remove them. + + Args: + libyuv_deps: dict of deps as defined in the LibYUV DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a deleted package in existing dep. + + Returns: + A tuple consisting of: + A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. + A list of paths of unexpected disappearing dependencies. + """ + all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) + generated_android_deps = sorted( + [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) + # Webrtc-only dependencies are handled in CalculateChangedDeps. + other_deps = sorted([ + path for path in all_removed_deps + if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS + ]) + return generated_android_deps, other_deps + + def CalculateChangedDeps(libyuv_deps, new_cr_deps): """ - Calculate changed deps entries based on entries defined in the libyuv DEPS - file: + Calculate changed deps entries based on entries defined in the LibYUV DEPS + file: - If a shared dependency with the Chromium DEPS file: roll it to the same revision as Chromium (i.e. entry in the new_cr_deps dict) - If it's a Chromium sub-directory, roll it to the HEAD revision (notice @@ -239,9 +424,9 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps): - If it's another DEPS entry (not shared with Chromium), roll it to HEAD unless it's configured to be skipped. - Returns: - A list of ChangedDep objects representing the changed deps. - """ + Returns: + A list of ChangedDep objects representing the changed deps. + """ result = [] libyuv_entries = BuildDepsentryDict(libyuv_deps) new_cr_entries = BuildDepsentryDict(new_cr_deps) @@ -250,68 +435,117 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps): continue cr_deps_entry = new_cr_entries.get(path) if cr_deps_entry: + assert type(cr_deps_entry) is type(libyuv_deps_entry) + + if isinstance(cr_deps_entry, CipdDepsEntry): + result.extend( + _FindChangedCipdPackages(path, libyuv_deps_entry.packages, + cr_deps_entry.packages)) + continue + + if isinstance(cr_deps_entry, VersionEntry): + result.extend( + _FindChangedVars(path, libyuv_deps_entry.version, + cr_deps_entry.version)) + continue + # Use the revision from Chromium's DEPS file. new_rev = cr_deps_entry.revision assert libyuv_deps_entry.url == cr_deps_entry.url, ( - 'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' % - (path, libyuv_deps_entry.url, cr_deps_entry.url)) + 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' % + (path, libyuv_deps_entry.url, cr_deps_entry.url)) else: - # Use the HEAD of the deps repo. - stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url, - 'HEAD']) - new_rev = stdout.strip().split('\t')[0] + if isinstance(libyuv_deps_entry, DepsEntry): + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand( + ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']) + new_rev = stdout.strip().split('\t')[0] + else: + # The dependency has been removed from chromium. + # This is handled by FindRemovedDeps. + continue # Check if an update is necessary. if libyuv_deps_entry.revision != new_rev: logging.debug('Roll dependency %s to %s', path, new_rev) - result.append(ChangedDep(path, libyuv_deps_entry.url, - libyuv_deps_entry.revision, new_rev)) + result.append( + ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision, + new_rev)) return sorted(result) def CalculateChangedClang(new_cr_rev): + def GetClangRev(lines): for line in lines: match = CLANG_REVISION_RE.match(line) if match: return match.group(1) - raise RollError('Could not parse Clang revision from:\n' + '\n'.join(' ' + l for l in lines)) + raise RollError('Could not parse Clang revision!') with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: current_lines = f.readlines() current_rev = GetClangRev(current_lines) new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, - new_cr_rev).splitlines() + new_cr_rev).splitlines() new_rev = GetClangRev(new_clang_update_py) return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) -def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, - new_commit_pos, changed_deps_list, clang_change): - current_cr_rev = current_cr_rev[0:10] - new_cr_rev = new_cr_rev[0:10] +def GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps_list, + added_deps_paths=None, + removed_deps_paths=None, + clang_change=None, +): + current_cr_rev = rev_update.current_chromium_rev[0:10] + new_cr_rev = rev_update.new_chromium_rev[0:10] rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) - commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, - git_number_interval)] - commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) - commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % - rev_interval)) + commit_msg = [ + 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), + 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), + 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) + ] + + def Section(adjective, deps): + noun = 'dependency' if len(deps) == 1 else 'dependencies' + commit_msg.append('%s %s' % (adjective, noun)) + if changed_deps_list: - commit_msg.append('Changed dependencies:') + Section('Changed', changed_deps_list) for c in changed_deps_list: - commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, - c.current_rev[0:10], - c.new_rev[0:10])) + if isinstance(c, ChangedCipdPackage): + commit_msg.append('* %s: %s..%s' % + (c.path, c.current_version, c.new_version)) + elif isinstance(c, ChangedVersionEntry): + commit_msg.append('* %s_vesion: %s..%s' % + (c.path, c.current_version, c.new_version)) + else: + commit_msg.append('* %s: %s/+log/%s..%s' % + (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) + + if added_deps_paths: + Section('Added', added_deps_paths) + commit_msg.extend('* %s' % p for p in added_deps_paths) + + if removed_deps_paths: + Section('Removed', removed_deps_paths) + commit_msg.extend('* %s' % p for p in removed_deps_paths) + + if any([changed_deps_list, added_deps_paths, removed_deps_paths]): change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') commit_msg.append('DEPS diff: %s\n' % change_url) else: commit_msg.append('No dependencies changed.') - if clang_change.current_rev != clang_change.new_rev: + if clang_change and clang_change.current_rev != clang_change.new_rev: commit_msg.append('Clang version changed %s:%s' % (clang_change.current_rev, clang_change.new_rev)) change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, @@ -320,38 +554,61 @@ def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, else: commit_msg.append('No update to Clang.\n') - # TBR needs to be non-empty for Gerrit to process it. - git_author = _RunCommand(['git', 'config', 'user.email'], - working_dir=CHECKOUT_SRC_DIR)[0].strip() - commit_msg.append('TBR=%s' % git_author) - commit_msg.append('BUG=None') return '\n'.join(commit_msg) -def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, - changed_deps): +def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): """Update the DEPS file with the new revision.""" - # Update the chromium_revision variable. with open(deps_filename, 'rb') as deps_file: deps_content = deps_file.read().decode('utf-8') - deps_content = deps_content.replace(old_cr_revision, new_cr_revision) + + # Update the chromium_revision variable. + deps_content = deps_content.replace(rev_update.current_chromium_rev, + rev_update.new_chromium_rev) + + # Add and remove dependencies. For now: only generated android deps. + # Since gclient cannot add or remove deps, we on the fact that + # these android deps are located in one place we can copy/paste. + deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) + new_deps = deps_re.search(new_cr_content) + old_deps = deps_re.search(deps_content) + if not new_deps or not old_deps: + faulty = 'Chromium' if not new_deps else 'LibYUV' + raise RollError('Was expecting to find "%s" and "%s"\n' + 'in %s DEPS' % + (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) + deps_content = deps_re.sub(new_deps.group(0), deps_content) + + for dep in changed_deps: + if isinstance(dep, ChangedVersionEntry): + deps_content = deps_content.replace(dep.current_version, dep.new_version) + with open(deps_filename, 'wb') as deps_file: deps_file.write(deps_content.encode('utf-8')) # Update each individual DEPS entry. for dep in changed_deps: + # ChangedVersionEntry types are already been processed. + if isinstance(dep, ChangedVersionEntry): + continue local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) if not os.path.isdir(local_dep_dir): raise RollError( - 'Cannot find local directory %s. Make sure the .gclient file\n' - 'contains all platforms in the target_os list, i.e.\n' + 'Cannot find local directory %s. Either run\n' + 'gclient sync --deps=all\n' + 'or make sure the .gclient file for your solution contains all ' + 'platforms in the target_os list, i.e.\n' 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 'Then run "gclient sync" again.' % local_dep_dir) - _RunCommand( - ['gclient', 'setdep', '--revision', '%s@%s' % (dep.path, dep.new_rev)], - working_dir=CHECKOUT_SRC_DIR) + if isinstance(dep, ChangedCipdPackage): + package = dep.package.format() # Eliminate double curly brackets + update = '%s:%s@%s' % (dep.path, package, dep.new_version) + else: + update = '%s@%s' % (dep.path, dep.new_rev) + _RunCommand(['gclient', 'setdep', '--revision', update], + working_dir=CHECKOUT_SRC_DIR) def _IsTreeClean(): @@ -363,9 +620,9 @@ def _IsTreeClean(): return False -def _EnsureUpdatedMasterBranch(dry_run): - current_branch = _RunCommand( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] +def _EnsureUpdatedMainBranch(dry_run): + current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', + 'HEAD'])[0].splitlines()[0] if current_branch != 'main': logging.error('Please checkout the main branch and re-run this script.') if not dry_run: @@ -407,19 +664,34 @@ def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): return 2 -def _UploadCL(commit_queue_mode): +def _GetCcRecipients(changed_deps_list): + """Returns a list of emails to notify based on the changed deps list. + """ + cc_recipients = [] + for c in changed_deps_list: + pass + return cc_recipients + + +def _UploadCL(commit_queue_mode, add_cc=None): """Upload the committed changes as a changelist to Gerrit. - commit_queue_mode: - - 2: Submit to commit queue. - - 1: Run trybots but do not submit to CQ. - - 0: Skip CQ, upload only. - """ - cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks', '--send-mail'] + commit_queue_mode: + - 2: Submit to commit queue. + - 1: Run trybots but do not submit to CQ. + - 0: Skip CQ, upload only. + + add_cc: A list of email addresses to add as CC recipients. + """ + cc_recipients = [] + if add_cc: + cc_recipients.extend(add_cc) + cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] if commit_queue_mode >= 2: logging.info('Sending the CL to the CQ...') cmd.extend(['-o', 'label=Bot-Commit+1']) cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) elif commit_queue_mode >= 1: logging.info('Starting CQ dry run...') cmd.extend(['-o', 'label=Commit-Queue+1']) @@ -429,31 +701,57 @@ def _UploadCL(commit_queue_mode): } stdout, stderr = _RunCommand(cmd, extra_env=extra_env) logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', - stdout, stderr) + stdout, stderr) + + +def GetRollRevisionRanges(opts, libyuv_deps): + current_cr_rev = libyuv_deps['vars']['chromium_revision'] + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev + + return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) def main(): p = argparse.ArgumentParser() - p.add_argument('--clean', action='store_true', default=False, + p.add_argument('--clean', + action='store_true', + default=False, help='Removes any previous local roll branch.') - p.add_argument('-r', '--revision', + p.add_argument('-r', + '--revision', help=('Chromium Git revision to roll to. Defaults to the ' 'Chromium HEAD revision if omitted.')) - p.add_argument('--dry-run', action='store_true', default=False, + p.add_argument('--dry-run', + action='store_true', + default=False, help=('Calculate changes and modify DEPS, but don\'t create ' 'any local branch, commit, upload CL or send any ' 'tryjobs.')) - p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', + p.add_argument('-i', + '--ignore-unclean-workdir', + action='store_true', default=False, help=('Ignore if the current branch is not main or if there ' 'are uncommitted changes (default: %(default)s).')) grp = p.add_mutually_exclusive_group() - grp.add_argument('--skip-cq', action='store_true', default=False, + grp.add_argument('--skip-cq', + action='store_true', + default=False, help='Skip sending the CL to the CQ (default: %(default)s)') - grp.add_argument('--cq-over', type=int, default=1, + grp.add_argument('--cq-over', + type=int, + default=1, help=('Commit queue dry run if the revision difference ' 'is below this number (default: %(default)s)')) - p.add_argument('-v', '--verbose', action='store_true', default=False, + p.add_argument('-v', + '--verbose', + action='store_true', + default=False, help='Be extra verbose in printing of log messages.') opts = p.parse_args() @@ -470,38 +768,52 @@ def main(): _RemovePreviousRollBranch(opts.dry_run) if not opts.ignore_unclean_workdir: - _EnsureUpdatedMasterBranch(opts.dry_run) - - new_cr_rev = opts.revision - if not new_cr_rev: - stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) - head_rev = stdout.strip().split('\t')[0] - logging.info('No revision specified. Using HEAD: %s', head_rev) - new_cr_rev = head_rev + _EnsureUpdatedMainBranch(opts.dry_run) deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') libyuv_deps = ParseLocalDepsFile(deps_filename) - current_cr_rev = libyuv_deps['vars']['chromium_revision'] - current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) - new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) + rev_update = GetRollRevisionRanges(opts, libyuv_deps) - new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) + current_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.current_chromium_rev)) + new_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.new_chromium_rev)) + + new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) + new_cr_deps = ParseDepsDict(new_cr_content) changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) - clang_change = CalculateChangedClang(new_cr_rev) - commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, - current_commit_pos, new_commit_pos, - changed_deps, clang_change) + # Discard other deps, assumed to be chromium-only dependencies. + new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) + removed_generated_android_deps, other_deps = FindRemovedDeps( + libyuv_deps, new_cr_deps) + if other_deps: + raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n' + 'Remove them or add them to either ' + 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) + clang_change = CalculateChangedClang(rev_update.new_chromium_rev) + commit_msg = GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps, + added_deps_paths=new_generated_android_deps, + removed_deps_paths=removed_generated_android_deps, + clang_change=clang_change) logging.debug('Commit message:\n%s', commit_msg) _CreateRollBranch(opts.dry_run) - UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) - _LocalCommit(commit_msg, opts.dry_run) - commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, - current_commit_pos, new_commit_pos) - logging.info('Uploading CL...') if not opts.dry_run: - _UploadCL(commit_queue_mode) + UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) + if _IsTreeClean(): + logging.info("No DEPS changes detected, skipping CL creation.") + else: + _LocalCommit(commit_msg, opts.dry_run) + commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, + current_commit_pos, new_commit_pos) + logging.info('Uploading CL...') + if not opts.dry_run: + _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) return 0 |