diff options
author | Henrik Kjellander <kjellander@chromium.org> | 2017-02-03 12:33:16 +0100 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2017-02-03 11:36:53 +0000 |
commit | f49fde796febead2ab00511735595d3b94ca6de8 (patch) | |
tree | d631f580e38010c1078b1dc35cdc98bc9597d9f6 /tools_libyuv | |
parent | 76e7f104ae69d9a4f7ed4ac74d6e2c4fd16b1594 (diff) | |
download | libyuv-f49fde796febead2ab00511735595d3b94ca6de8.tar.gz |
Use DEPS for all dependencies + add PRESUBMIT.py
This changes libyuv to use the DEPS file for pulling
down all dependencies (thus no Chromium checkout is needed any more).
Add tools_libyuv directory to contain libyuv-specific tools
(needed to avoid name collision with the now DEPSed tools/ directory
of Chromium, which is needed by the toolchain).
Add tools_libyuv/autoroller/roll_deps.py script to automatically
roll all entries in the DEPS file (copied from WebRTC).
third_party/ is now DEPSed as well, including the gtest configuration
headers that used to live inside the libyuv repo.
Add PRESUBMIT.py with a few simple checks + execution of PyLint and
Python unit tests. For PyLint a pylintrc file was also added.
Valgrind in tools_libyuv/valgrind was updated to make PRESUBMIT.py pass
and remove old tsan suppressions (not used).
Removed util/android/test_runner.py since it's no longer needed.
Buildbot changes in https://chromium-review.googlesource.com/436464
are needed for the Memcheck bot to go green.
BUG=libyuv:676
NOTRY=True
Change-Id: Ib86fea2905a1656bba2933703ce5a59d29d8db6b
Reviewed-on: https://chromium-review.googlesource.com/436264
Commit-Queue: Henrik Kjellander <kjellander@chromium.org>
Reviewed-by: Frank Barchard <fbarchard@google.com>
Diffstat (limited to 'tools_libyuv')
21 files changed, 1245 insertions, 0 deletions
diff --git a/tools_libyuv/OWNERS b/tools_libyuv/OWNERS new file mode 100644 index 00000000..aca046d4 --- /dev/null +++ b/tools_libyuv/OWNERS @@ -0,0 +1 @@ +kjellander@chromium.org diff --git a/tools_libyuv/autoroller/roll_deps.py b/tools_libyuv/autoroller/roll_deps.py new file mode 100755 index 00000000..a9eb307e --- /dev/null +++ b/tools_libyuv/autoroller/roll_deps.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python +# Copyright 2017 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# This is a modified copy of the script in +# https://chromium.googlesource.com/external/webrtc/+/master/tools-webrtc/autoroller/roll_deps.py +# customized for libyuv. + + +"""Script to automatically roll dependencies in the libyuv DEPS file.""" + +import argparse +import base64 +import collections +import logging +import os +import re +import subprocess +import sys +import urllib + + +# Skip these dependencies (list without solution name prefix). +DONT_AUTOROLL_THESE = [ + 'src/third_party/gflags/src', +] + +LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' +CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' +CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' +CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' +CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' + +COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') +CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') +ROLL_BRANCH_NAME = 'roll_chromium_revision' + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, + os.pardir)) +CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) + +sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) +import find_depot_tools +find_depot_tools.add_depot_tools_to_path() +from gclient import GClientKeywords + +CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' +CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', + 'clang', 'scripts', 'update.py') + +DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') +ChangedDep = collections.namedtuple('ChangedDep', + 'path url current_rev new_rev') + +class RollError(Exception): + pass + + +def ParseDepsDict(deps_content): + local_scope = {} + var = GClientKeywords.VarImpl({}, local_scope) + global_scope = { + 'From': GClientKeywords.FromImpl, + 'Var': var.Lookup, + 'deps_os': {}, + } + exec(deps_content, global_scope, local_scope) + return local_scope + + +def ParseLocalDepsFile(filename): + with open(filename, 'rb') as f: + deps_content = f.read() + return ParseDepsDict(deps_content) + + +def ParseRemoteCrDepsFile(revision): + deps_content = ReadRemoteCrFile('DEPS', revision) + return ParseDepsDict(deps_content) + + +def ParseCommitPosition(commit_message): + for line in reversed(commit_message.splitlines()): + m = COMMIT_POSITION_RE.match(line.strip()) + if m: + return m.group(1) + logging.error('Failed to parse commit position id from:\n%s\n', + commit_message) + sys.exit(-1) + + +def _RunCommand(command, working_dir=None, ignore_exit_code=False, + extra_env=None): + """Runs a command and returns the output from that command. + + If the command fails (exit code != 0), the function will exit the process. + + Returns: + A tuple containing the stdout and stderr outputs as strings. + """ + working_dir = working_dir or CHECKOUT_SRC_DIR + logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) + env = os.environ.copy() + if extra_env: + assert all(type(value) == str for value in extra_env.values()) + logging.debug('extra env: %s', extra_env) + env.update(extra_env) + p = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env, + cwd=working_dir, universal_newlines=True) + std_output = p.stdout.read() + err_output = p.stderr.read() + p.wait() + p.stdout.close() + p.stderr.close() + if not ignore_exit_code and p.returncode != 0: + logging.error('Command failed: %s\n' + 'stdout:\n%s\n' + 'stderr:\n%s\n', ' '.join(command), std_output, err_output) + sys.exit(p.returncode) + return std_output, err_output + + +def _GetBranches(): + """Returns a tuple of active,branches. + + The 'active' is the name of the currently active branch and 'branches' is a + list of all branches. + """ + lines = _RunCommand(['git', 'branch'])[0].split('\n') + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches + + +def _ReadGitilesContent(url): + # Download and decode BASE64 content until + # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. + base64_content = ReadUrlContent(url + '?format=TEXT') + return base64.b64decode(base64_content[0]) + + +def ReadRemoteCrFile(path_below_src, revision): + """Reads a remote Chromium file of a specific revision. Returns a string.""" + return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, + path_below_src)) + + +def ReadRemoteCrCommit(revision): + """Reads a remote Chromium commit message. Returns a string.""" + return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) + + +def ReadUrlContent(url): + """Connect to a remote host and read the contents. Returns a list of lines.""" + conn = urllib.urlopen(url) + try: + return conn.readlines() + except IOError as e: + logging.exception('Error connecting to %s. Error: %s', url, e) + raise + finally: + conn.close() + + +def GetMatchingDepsEntries(depsentry_dict, dir_path): + """Gets all deps entries matching the provided path. + + This list may contain more than one DepsEntry object. + Example: dir_path='src/testing' would give results containing both + 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. + Example 2: dir_path='src/build' should return 'src/build' but not + 'src/buildtools'. + + Returns: + A list of DepsEntry objects. + """ + result = [] + for path, depsentry in depsentry_dict.iteritems(): + if path == dir_path: + result.append(depsentry) + else: + parts = path.split('/') + if all(part == parts[i] + for i, part in enumerate(dir_path.split('/'))): + result.append(depsentry) + return result + + +def BuildDepsentryDict(deps_dict): + """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" + result = {} + def AddDepsEntries(deps_subdict): + for path, deps_url in deps_subdict.iteritems(): + if not result.has_key(path): + url, revision = deps_url.split('@') if deps_url else (None, None) + result[path] = DepsEntry(path, url, revision) + + AddDepsEntries(deps_dict['deps']) + for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: + AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) + return result + + +def CalculateChangedDeps(libyuv_deps, new_cr_deps): + """ + Calculate changed deps entries based on entries defined in the libyuv DEPS + file: + - If a shared dependency with the Chromium DEPS file: roll it to the same + revision as Chromium (i.e. entry in the new_cr_deps dict) + - If it's a Chromium sub-directory, roll it to the HEAD revision (notice + this means it may be ahead of the chromium_revision, but generally these + should be close). + - If it's another DEPS entry (not shared with Chromium), roll it to HEAD + unless it's configured to be skipped. + + Returns: + A list of ChangedDep objects representing the changed deps. + """ + result = [] + libyuv_entries = BuildDepsentryDict(libyuv_deps) + new_cr_entries = BuildDepsentryDict(new_cr_deps) + for path, libyuv_deps_entry in libyuv_entries.iteritems(): + if path in DONT_AUTOROLL_THESE: + continue + cr_deps_entry = new_cr_entries.get(path) + if cr_deps_entry: + # Use the revision from Chromium's DEPS file. + new_rev = cr_deps_entry.revision + assert libyuv_deps_entry.url == cr_deps_entry.url, ( + 'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' % + (path, libyuv_deps_entry.url, cr_deps_entry.url)) + else: + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url, + 'HEAD']) + new_rev = stdout.strip().split('\t')[0] + + # Check if an update is necessary. + if libyuv_deps_entry.revision != new_rev: + logging.debug('Roll dependency %s to %s', path, new_rev) + result.append(ChangedDep(path, libyuv_deps_entry.url, + libyuv_deps_entry.revision, new_rev)) + return sorted(result) + + +def CalculateChangedClang(new_cr_rev): + def GetClangRev(lines): + for line in lines: + match = CLANG_REVISION_RE.match(line) + if match: + return match.group(1) + raise RollError('Could not parse Clang revision!') + + with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f: + current_lines = f.readlines() + current_rev = GetClangRev(current_lines) + + new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, + new_cr_rev).splitlines() + new_rev = GetClangRev(new_clang_update_py) + return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) + + +def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, + new_commit_pos, changed_deps_list, clang_change): + current_cr_rev = current_cr_rev[0:10] + new_cr_rev = new_cr_rev[0:10] + rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) + git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) + + commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, + git_number_interval)] + commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) + commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % + rev_interval)) + # TBR field will be empty unless in some custom cases, where some engineers + # are added. + tbr_authors = '' + if changed_deps_list: + commit_msg.append('Changed dependencies:') + + for c in changed_deps_list: + commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, + c.current_rev[0:10], + c.new_rev[0:10])) + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') + commit_msg.append('DEPS diff: %s\n' % change_url) + else: + commit_msg.append('No dependencies changed.') + + if clang_change.current_rev != clang_change.new_rev: + commit_msg.append('Clang version changed %s:%s' % + (clang_change.current_rev, clang_change.new_rev)) + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, + CLANG_UPDATE_SCRIPT_URL_PATH) + commit_msg.append('Details: %s\n' % change_url) + else: + commit_msg.append('No update to Clang.\n') + + commit_msg.append('TBR=%s' % tbr_authors) + commit_msg.append('BUG=None') + return '\n'.join(commit_msg) + + +def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, + changed_deps): + """Update the DEPS file with the new revision.""" + + # Update the chromium_revision variable. + with open(deps_filename, 'rb') as deps_file: + deps_content = deps_file.read() + deps_content = deps_content.replace(old_cr_revision, new_cr_revision) + with open(deps_filename, 'wb') as deps_file: + deps_file.write(deps_content) + + # Update each individual DEPS entry. + for dep in changed_deps: + local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) + if not os.path.isdir(local_dep_dir): + raise RollError( + 'Cannot find local directory %s. Either run\n' + 'gclient sync --deps=all\n' + 'or make sure the .gclient file for your solution contains all ' + 'platforms in the target_os list, i.e.\n' + 'target_os = ["android", "unix", "mac", "ios", "win"];\n' + 'Then run "gclient sync" again.' % local_dep_dir) + _, stderr = _RunCommand( + ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev], + working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True) + if stderr: + logging.warning('roll-dep-svn: %s', stderr) + + +def _IsTreeClean(): + stdout, _ = _RunCommand(['git', 'status', '--porcelain']) + if len(stdout) == 0: + return True + + logging.error('Dirty/unversioned files:\n%s', stdout) + return False + + +def _EnsureUpdatedMasterBranch(dry_run): + current_branch = _RunCommand( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] + if current_branch != 'master': + logging.error('Please checkout the master branch and re-run this script.') + if not dry_run: + sys.exit(-1) + + logging.info('Updating master branch...') + _RunCommand(['git', 'pull']) + + +def _CreateRollBranch(dry_run): + logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) + + +def _RemovePreviousRollBranch(dry_run): + active_branch, branches = _GetBranches() + if active_branch == ROLL_BRANCH_NAME: + active_branch = 'master' + if ROLL_BRANCH_NAME in branches: + logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', active_branch]) + _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) + + +def _LocalCommit(commit_msg, dry_run): + logging.info('Committing changes locally.') + if not dry_run: + _RunCommand(['git', 'add', '--update', '.']) + _RunCommand(['git', 'commit', '-m', commit_msg]) + + +def _UploadCL(dry_run, rietveld_email=None): + logging.info('Uploading CL...') + if not dry_run: + cmd = ['git', 'cl', 'upload', '-f'] + if rietveld_email: + cmd.append('--email=%s' % rietveld_email) + _RunCommand(cmd, extra_env={'EDITOR': 'true'}) + + +def _SendToCQ(dry_run, skip_cq): + logging.info('Sending the CL to the CQ...') + if not dry_run and not skip_cq: + _RunCommand(['git', 'cl', 'set_commit']) + logging.info('Sent the CL to the CQ.') + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--clean', action='store_true', default=False, + help='Removes any previous local roll branch.') + p.add_argument('-r', '--revision', + help=('Chromium Git revision to roll to. Defaults to the ' + 'Chromium HEAD revision if omitted.')) + p.add_argument('-u', '--rietveld-email', + help=('E-mail address to use for creating the CL at Rietveld' + 'If omitted a previously cached one will be used or an ' + 'error will be thrown during upload.')) + p.add_argument('--dry-run', action='store_true', default=False, + help=('Calculate changes and modify DEPS, but don\'t create ' + 'any local branch, commit, upload CL or send any ' + 'tryjobs.')) + p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', + default=False, + help=('Ignore if the current branch is not master or if there ' + 'are uncommitted changes (default: %(default)s).')) + p.add_argument('--skip-cq', action='store_true', default=False, + help='Skip sending the CL to the CQ (default: %(default)s)') + p.add_argument('-v', '--verbose', action='store_true', default=False, + help='Be extra verbose in printing of log messages.') + opts = p.parse_args() + + if opts.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if not opts.ignore_unclean_workdir and not _IsTreeClean(): + logging.error('Please clean your local checkout first.') + return 1 + + if opts.clean: + _RemovePreviousRollBranch(opts.dry_run) + + if not opts.ignore_unclean_workdir: + _EnsureUpdatedMasterBranch(opts.dry_run) + + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev + + deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') + libyuv_deps = ParseLocalDepsFile(deps_filename) + current_cr_rev = libyuv_deps['vars']['chromium_revision'] + + current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) + new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) + + new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + clang_change = CalculateChangedClang(new_cr_rev) + commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, + current_commit_pos, new_commit_pos, + changed_deps, clang_change) + logging.debug('Commit message:\n%s', commit_msg) + + _CreateRollBranch(opts.dry_run) + UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) + _LocalCommit(commit_msg, opts.dry_run) + _UploadCL(opts.dry_run, opts.rietveld_email) + _SendToCQ(opts.dry_run, opts.skip_cq) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_libyuv/autoroller/unittests/.DS_Store b/tools_libyuv/autoroller/unittests/.DS_Store Binary files differnew file mode 100644 index 00000000..70369d69 --- /dev/null +++ b/tools_libyuv/autoroller/unittests/.DS_Store diff --git a/tools_libyuv/autoroller/unittests/roll_deps_test.py b/tools_libyuv/autoroller/unittests/roll_deps_test.py new file mode 100755 index 00000000..025e46e1 --- /dev/null +++ b/tools_libyuv/autoroller/unittests/roll_deps_test.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# Copyright 2017 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import glob +import os +import shutil +import sys +import tempfile +import unittest + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) +sys.path.append(PARENT_DIR) +import roll_deps +from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \ + ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile + + +TEST_DATA_VARS = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +DEPS_ENTRIES = { + 'src/build': 'https://build.com', + 'src/buildtools': 'https://buildtools.com', + 'src/testing/gtest': 'https://gtest.com', + 'src/testing/gmock': 'https://gmock.com', +} + +BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f' +BUILD_NEW_REV = 'HEAD' +BUILDTOOLS_OLD_REV = '64e38f0cebdde27aa0cfb405f330063582f9ac76' +BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b' + + +class TestError(Exception): + pass + + +class FakeCmd(object): + def __init__(self): + self.expectations = [] + + def add_expectation(self, *args, **kwargs): + returns = kwargs.pop('_returns', None) + self.expectations.append((args, kwargs, returns)) + + def __call__(self, *args, **kwargs): + if not self.expectations: + raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) + exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) + if args != exp_args or kwargs != exp_kwargs: + message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) + message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) + raise TestError(message) + return exp_returns + + +class TestRollChromiumRevision(unittest.TestCase): + def setUp(self): + self._output_dir = tempfile.mkdtemp() + for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): + shutil.copy(test_file, self._output_dir) + self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS') + self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old') + self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new') + + self.fake = FakeCmd() + self.old_RunCommand = getattr(roll_deps, '_RunCommand') + setattr(roll_deps, '_RunCommand', self.fake) + + def tearDown(self): + shutil.rmtree(self._output_dir, ignore_errors=True) + self.assertEqual(self.fake.expectations, []) + setattr(roll_deps, '_RunCommand', self.old_RunCommand) + + def testUpdateDepsFile(self): + new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' + + current_rev = TEST_DATA_VARS['chromium_revision'] + UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, []) + with open(self._libyuv_depsfile) as deps_file: + deps_contents = deps_file.read() + self.assertTrue(new_rev in deps_contents, + 'Failed to find %s in\n%s' % (new_rev, deps_contents)) + + def testParseDepsDict(self): + with open(self._libyuv_depsfile) as deps_file: + deps_contents = deps_file.read() + local_scope = ParseDepsDict(deps_contents) + vars_dict = local_scope['vars'] + + def assertVar(variable_name): + self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) + assertVar('chromium_git') + assertVar('chromium_revision') + self.assertEquals(len(local_scope['deps']), 3) + + def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') + self.assertEquals(len(entries), 1) + self.assertEquals(entries[0], DEPS_ENTRIES['src/testing/gtest']) + + def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing') + self.assertEquals(len(entries), 2) + + def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build') + self.assertEquals(len(entries), 1) + self.assertEquals(entries[0], DEPS_ENTRIES['src/build']) + + def testCalculateChangedDeps(self): + _SetupGitLsRemoteCall(self.fake, + 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV) + libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + self.assertEquals(len(changed_deps), 2) + self.assertEquals(changed_deps[0].path, 'src/build') + self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV) + self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV) + + self.assertEquals(changed_deps[1].path, 'src/buildtools') + self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV) + self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV) + + +def _SetupGitLsRemoteCall(cmd_fake, url, revision): + cmd = ['git', 'ls-remote', url, revision] + cmd_fake.add_expectation(cmd, _returns=(revision, None)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS b/tools_libyuv/autoroller/unittests/testdata/DEPS new file mode 100644 index 00000000..9fbb48a7 --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS @@ -0,0 +1,20 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +deps = { + # Entry that is a directory in Chromium, so we're using a Git subtree mirror for it. + 'src/build': + Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f', + + # Entry that's also a DEPS entry in the Chromium DEPS file. + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76', + + # Entry only present in libyuv, not Chromium. + 'src/third_party/gflags/src': + Var('chromium_git') + '/external/github.com/gflags/gflags@03bebcb065c83beff83d50ae025a55a4bf94dfca', +} diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new new file mode 100644 index 00000000..d53083ce --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is updated compared to the DEPS.chromium.old file. + 'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +} diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old new file mode 100644 index 00000000..dd6ddaec --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is and older revision than DEPS.chromium.new file. + 'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +} diff --git a/tools_libyuv/get_landmines.py b/tools_libyuv/get_landmines.py new file mode 100755 index 00000000..3dc78bb9 --- /dev/null +++ b/tools_libyuv/get_landmines.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Copyright 2016 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +""" +This file emits the list of reasons why a particular build needs to be clobbered +(or a list of 'landmines'). +""" + +import os +import sys + +script_dir = os.path.dirname(os.path.realpath(__file__)) +checkout_root = os.path.abspath(os.path.join(script_dir, os.pardir)) +sys.path.insert(0, os.path.join(checkout_root, 'build')) +import landmine_utils + + +distributor = landmine_utils.distributor +gyp_defines = landmine_utils.gyp_defines +gyp_msvs_version = landmine_utils.gyp_msvs_version +platform = landmine_utils.platform + + +def print_landmines(): + """ + ALL LANDMINES ARE EMITTED FROM HERE. + """ + # DO NOT add landmines as part of a regular CL. Landmines are a last-effort + # bandaid fix if a CL that got landed has a build dependency bug and all bots + # need to be cleaned up. If you're writing a new CL that causes build + # dependency problems, fix the dependency problems instead of adding a + # landmine. + # See the Chromium version in src/build/get_landmines.py for usage examples. + print 'Clobber to remove GYP artifacts after switching bots to GN.' + print 'Another try to remove GYP artifacts after switching bots to GN.' + + +def main(): + print_landmines() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_libyuv/msan/OWNERS b/tools_libyuv/msan/OWNERS new file mode 100644 index 00000000..60351e7e --- /dev/null +++ b/tools_libyuv/msan/OWNERS @@ -0,0 +1,3 @@ +pbos@chromium.org +kjellander@chromium.org + diff --git a/tools_libyuv/msan/blacklist.txt b/tools_libyuv/msan/blacklist.txt new file mode 100644 index 00000000..8b5e42a7 --- /dev/null +++ b/tools_libyuv/msan/blacklist.txt @@ -0,0 +1,9 @@ +# The rules in this file are only applied at compile time. +# Because the Chrome buildsystem does not automatically touch the files +# mentioned here, changing this file requires clobbering all MSan bots. +# +# Please think twice before you add or remove these rules. + +# This is a stripped down copy of Chromium's blacklist.txt, to enable +# adding libyuv-specific blacklist entries. + diff --git a/tools_libyuv/ubsan/OWNERS b/tools_libyuv/ubsan/OWNERS new file mode 100644 index 00000000..b608519a --- /dev/null +++ b/tools_libyuv/ubsan/OWNERS @@ -0,0 +1,4 @@ +pbos@webrtc.org +kjellander@webrtc.org +fbarchard@chromium.org + diff --git a/tools_libyuv/ubsan/blacklist.txt b/tools_libyuv/ubsan/blacklist.txt new file mode 100644 index 00000000..8bcb2907 --- /dev/null +++ b/tools_libyuv/ubsan/blacklist.txt @@ -0,0 +1,15 @@ +############################################################################# +# UBSan blacklist. +# Please think twice before you add or remove these rules. + +# This is a stripped down copy of Chromium's blacklist.txt, to enable +# adding WebRTC-specific blacklist entries. + +############################################################################# +# YASM does some funny things that UBsan doesn't like. +# https://crbug.com/489901 +src:*/third_party/yasm/* + +############################################################################# +# Ignore system libraries. +src:*/usr/* diff --git a/tools_libyuv/ubsan/vptr_blacklist.txt b/tools_libyuv/ubsan/vptr_blacklist.txt new file mode 100644 index 00000000..8ed070c0 --- /dev/null +++ b/tools_libyuv/ubsan/vptr_blacklist.txt @@ -0,0 +1,21 @@ +############################################################################# +# UBSan vptr blacklist. +# Function and type based blacklisting use a mangled name, and it is especially +# tricky to represent C++ types. For now, any possible changes by name manglings +# are simply represented as wildcard expressions of regexp, and thus it might be +# over-blacklisted. +# +# Please think twice before you add or remove these rules. +# +# This is a stripped down copy of Chromium's vptr_blacklist.txt, to enable +# adding libyuv-specific blacklist entries. + +############################################################################# +# Using raw pointer values. +# +# A raw pointer value (16) is used to infer the field offset by +# GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET. + +# Example: +# src:*/third_party/protobuf/src/google/protobuf/compiler/plugin.pb.cc + diff --git a/tools_libyuv/valgrind/libyuv_tests.bat b/tools_libyuv/valgrind/libyuv_tests.bat new file mode 100644 index 00000000..e37f09eb --- /dev/null +++ b/tools_libyuv/valgrind/libyuv_tests.bat @@ -0,0 +1,79 @@ +@echo off
+:: Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+::
+:: Use of this source code is governed by a BSD-style license
+:: that can be found in the LICENSE file in the root of the source
+:: tree. An additional intellectual property rights grant can be found
+:: in the file PATENTS. All contributing project authors may
+:: be found in the AUTHORS file in the root of the source tree.
+
+:: This script is a copy of chrome_tests.bat with the following changes:
+:: - Invokes libyuv_tests.py instead of chrome_tests.py
+:: - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make
+:: it possible to execute the Python scripts properly.
+
+:: TODO(timurrrr): batch files 'export' all the variables to the parent shell
+set THISDIR=%~dp0
+set TOOL_NAME="unknown"
+
+:: Get the tool name and put it into TOOL_NAME {{{1
+:: NB: SHIFT command doesn't modify %*
+:PARSE_ARGS_LOOP
+ if %1 == () GOTO:TOOLNAME_NOT_FOUND
+ if %1 == --tool GOTO:TOOLNAME_FOUND
+ SHIFT
+ goto :PARSE_ARGS_LOOP
+
+:TOOLNAME_NOT_FOUND
+echo "Please specify a tool (tsan or drmemory) by using --tool flag"
+exit /B 1
+
+:TOOLNAME_FOUND
+SHIFT
+set TOOL_NAME=%1
+:: }}}
+if "%TOOL_NAME%" == "drmemory" GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_light" GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_full" GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_pattern" GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "tsan" GOTO :SETUP_TSAN
+echo "Unknown tool: `%TOOL_NAME%`! Only tsan and drmemory are supported."
+exit /B 1
+
+:SETUP_DRMEMORY
+if NOT "%DRMEMORY_COMMAND%"=="" GOTO :RUN_TESTS
+:: Set up DRMEMORY_COMMAND to invoke Dr. Memory {{{1
+set DRMEMORY_PATH=%THISDIR%..\..\third_party\drmemory
+set DRMEMORY_SFX=%DRMEMORY_PATH%\drmemory-windows-sfx.exe
+if EXIST %DRMEMORY_SFX% GOTO DRMEMORY_BINARY_OK
+echo "Can't find Dr. Memory executables."
+echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
+echo "for the instructions on how to get them."
+exit /B 1
+
+:DRMEMORY_BINARY_OK
+%DRMEMORY_SFX% -o%DRMEMORY_PATH%\unpacked -y
+set DRMEMORY_COMMAND=%DRMEMORY_PATH%\unpacked\bin\drmemory.exe
+:: }}}
+goto :RUN_TESTS
+
+:SETUP_TSAN
+:: Set up PIN_COMMAND to invoke TSan {{{1
+set TSAN_PATH=%THISDIR%..\..\third_party\tsan
+set TSAN_SFX=%TSAN_PATH%\tsan-x86-windows-sfx.exe
+if EXIST %TSAN_SFX% GOTO TSAN_BINARY_OK
+echo "Can't find ThreadSanitizer executables."
+echo "See http://www.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/threadsanitizer-on-windows"
+echo "for the instructions on how to get them."
+exit /B 1
+
+:TSAN_BINARY_OK
+%TSAN_SFX% -o%TSAN_PATH%\unpacked -y
+set PIN_COMMAND=%TSAN_PATH%\unpacked\tsan-x86-windows\tsan.bat
+:: }}}
+goto :RUN_TESTS
+
+:RUN_TESTS
+set PYTHONPATH=%THISDIR%..\python\google;%THISDIR%..\valgrind
+set RUNNING_ON_VALGRIND=yes
+python %THISDIR%libyuv_tests.py %*
diff --git a/tools_libyuv/valgrind/libyuv_tests.py b/tools_libyuv/valgrind/libyuv_tests.py new file mode 100755 index 00000000..e780bd95 --- /dev/null +++ b/tools_libyuv/valgrind/libyuv_tests.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Runs various libyuv tests through valgrind_test.py. + +This script inherits the chrome_tests.py in Chrome, but allows running any test +instead of only the hard-coded ones. It uses the -t cmdline flag to do this, and +only supports specifying a single test for each run. + +Suppression files: +The Chrome valgrind directory we use as a DEPS dependency contains the following +suppression files: + valgrind/memcheck/suppressions.txt + valgrind/memcheck/suppressions_mac.txt + valgrind/tsan/suppressions.txt + valgrind/tsan/suppressions_mac.txt + valgrind/tsan/suppressions_win32.txt +Since they're referenced from the chrome_tests.py script, we have similar files +below the directory of this script. When executing, this script will setup both +Chrome's suppression files and our own, so we can easily maintain libyuv +specific suppressions in our own files. +""" + +import logging +import optparse +import os +import sys + +import logging_utils +import path_utils + +import chrome_tests + + +class LibyuvTest(chrome_tests.ChromeTests): + """Class that handles setup of suppressions for libyuv. + + Everything else is inherited from chrome_tests.ChromeTests. + """ + + def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None): + """Override command-building method so we can add more suppressions.""" + cmd = chrome_tests.ChromeTests._DefaultCommand(self, tool, exe, + valgrind_test_args) + # When ChromeTests._DefaultCommand has executed, it has setup suppression + # files based on what's found in the memcheck/ or tsan/ subdirectories of + # this script's location. If Mac or Windows is executing, additional + # platform specific files have also been added. + # Since only the ones located below this directory is added, we must also + # add the ones maintained by Chrome, located in ../../tools/valgrind. + + # The idea is to look for --suppression arguments in the cmd list and add a + # modified copy of each suppression file, for the corresponding file in + # ../../tools/valgrind. + script_dir = path_utils.ScriptDir() + old_base, _ = os.path.split(script_dir) + + checkout_src = os.path.abspath(os.path.join(script_dir, os.pardir, + os.pardir)) + new_dir = os.path.join(checkout_src, 'tools', 'valgrind') + add_suppressions = [] + for token in cmd: + if '--suppressions' in token: + add_suppressions.append(token.replace(script_dir, new_dir)) + return add_suppressions + cmd + + +def main(_): + parser = optparse.OptionParser('usage: %prog -b <dir> -t <test> <test args>') + parser.disable_interspersed_args() + parser.add_option('-b', '--build-dir', + help=('Location of the compiler output. Can only be used ' + 'when the test argument does not contain this path.')) + parser.add_option("--target", help="Debug or Release") + parser.add_option('-t', '--test', help='Test to run.') + parser.add_option('', '--baseline', action='store_true', default=False, + help='Generate baseline data instead of validating') + parser.add_option('', '--gtest_filter', + help='Additional arguments to --gtest_filter') + parser.add_option('', '--gtest_repeat', + help='Argument for --gtest_repeat') + parser.add_option("--gtest_shuffle", action="store_true", default=False, + help="Randomize tests' orders on every iteration.") + parser.add_option("--gtest_break_on_failure", action="store_true", + default=False, + help="Drop in to debugger on assertion failure. Also " + "useful for forcing tests to exit with a stack dump " + "on the first assertion failure when running with " + "--gtest_repeat=-1") + parser.add_option('-v', '--verbose', action='store_true', default=False, + help='Verbose output - enable debug log messages') + parser.add_option('', '--tool', dest='valgrind_tool', default='memcheck', + help='Specify a valgrind tool to run the tests under') + parser.add_option('', '--tool_flags', dest='valgrind_tool_flags', default='', + help='Specify custom flags for the selected valgrind tool') + parser.add_option('', '--keep_logs', action='store_true', default=False, + help=('Store memory tool logs in the <tool>.logs directory ' + 'instead of /tmp.\nThis can be useful for tool ' + 'developers/maintainers.\nPlease note that the <tool>' + '.logs directory will be clobbered on tool startup.')) + parser.add_option("--test-launcher-bot-mode", action="store_true", + help="run the tests with --test-launcher-bot-mode") + parser.add_option("--test-launcher-total-shards", type=int, + help="run the tests with --test-launcher-total-shards") + parser.add_option("--test-launcher-shard-index", type=int, + help="run the tests with --test-launcher-shard-index") + options, args = parser.parse_args() + + if options.verbose: + logging_utils.config_root(logging.DEBUG) + else: + logging_utils.config_root() + + if not options.test: + parser.error('--test not specified') + + # Support build dir both with and without the target. + if (options.target and options.build_dir and + not options.build_dir.endswith(options.target)): + options.build_dir = os.path.join(options.build_dir, options.target) + + # If --build_dir is provided, prepend it to the test executable if needed. + test_executable = options.test + if options.build_dir and not test_executable.startswith(options.build_dir): + test_executable = os.path.join(options.build_dir, test_executable) + args = [test_executable] + args + + test = LibyuvTest(options, args, 'cmdline') + return test.Run() + +if __name__ == '__main__': + return_code = main(sys.argv) + sys.exit(return_code) diff --git a/tools_libyuv/valgrind/libyuv_tests.sh b/tools_libyuv/valgrind/libyuv_tests.sh new file mode 100755 index 00000000..4fee7dae --- /dev/null +++ b/tools_libyuv/valgrind/libyuv_tests.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +# Set up some paths and re-direct the arguments to libyuv_tests.py + +# This script is a copy of the chrome_tests.sh wrapper script with the following +# changes: +# - The locate_valgrind.sh of Chromium's Valgrind scripts dir is used to locate +# the Valgrind framework install. +# - libyuv_tests.py is invoked instead of chrome_tests.py. +# - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make it +# possible to execute the Python scripts properly. + +export THISDIR=`dirname $0` +ARGV_COPY="$@" + +# We need to set CHROME_VALGRIND iff using Memcheck or TSan-Valgrind: +# tools/valgrind-libyuv/libyuv_tests.sh --tool memcheck +# or +# tools/valgrind-libyuv/libyuv_tests.sh --tool=memcheck +# (same for "--tool=tsan") +tool="memcheck" # Default to memcheck. +while (( "$#" )) +do + if [[ "$1" == "--tool" ]] + then + tool="$2" + shift + elif [[ "$1" =~ --tool=(.*) ]] + then + tool="${BASH_REMATCH[1]}" + fi + shift +done + +NEEDS_VALGRIND=0 +NEEDS_DRMEMORY=0 + +case "$tool" in + "memcheck") + NEEDS_VALGRIND=1 + ;; + "tsan" | "tsan_rv") + if [ "`uname -s`" == CYGWIN* ] + then + NEEDS_PIN=1 + else + NEEDS_VALGRIND=1 + fi + ;; + "drmemory" | "drmemory_light" | "drmemory_full" | "drmemory_pattern") + NEEDS_DRMEMORY=1 + ;; +esac + +# For Libyuv, we'll use the locate_valgrind.sh script in Chromium's Valgrind +# scripts dir to locate the Valgrind framework install +CHROME_VALGRIND_SCRIPTS=$THISDIR/../valgrind + +if [ "$NEEDS_VALGRIND" == "1" ] +then + CHROME_VALGRIND=`sh $CHROME_VALGRIND_SCRIPTS/locate_valgrind.sh` + if [ "$CHROME_VALGRIND" = "" ] + then + # locate_valgrind.sh failed + exit 1 + fi + echo "Using valgrind binaries from ${CHROME_VALGRIND}" + + PATH="${CHROME_VALGRIND}/bin:$PATH" + # We need to set these variables to override default lib paths hard-coded into + # Valgrind binary. + export VALGRIND_LIB="$CHROME_VALGRIND/lib/valgrind" + export VALGRIND_LIB_INNER="$CHROME_VALGRIND/lib/valgrind" + + # Clean up some /tmp directories that might be stale due to interrupted + # chrome_tests.py execution. + # FYI: + # -mtime +1 <- only print files modified more than 24h ago, + # -print0/-0 are needed to handle possible newlines in the filenames. + echo "Cleanup /tmp from Valgrind stuff" + find /tmp -maxdepth 1 \(\ + -name "vgdb-pipe-*" -or -name "vg_logs_*" -or -name "valgrind.*" \ + \) -mtime +1 -print0 | xargs -0 rm -rf +fi + +if [ "$NEEDS_DRMEMORY" == "1" ] +then + if [ -z "$DRMEMORY_COMMAND" ] + then + DRMEMORY_PATH="$THISDIR/../../third_party/drmemory" + DRMEMORY_SFX="$DRMEMORY_PATH/drmemory-windows-sfx.exe" + if [ ! -f "$DRMEMORY_SFX" ] + then + echo "Can't find Dr. Memory executables." + echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory" + echo "for the instructions on how to get them." + exit 1 + fi + + chmod +x "$DRMEMORY_SFX" # Cygwin won't run it without +x. + "$DRMEMORY_SFX" -o"$DRMEMORY_PATH/unpacked" -y + export DRMEMORY_COMMAND="$DRMEMORY_PATH/unpacked/bin/drmemory.exe" + fi +fi + +if [ "$NEEDS_PIN" == "1" ] +then + if [ -z "$PIN_COMMAND" ] + then + # Set up PIN_COMMAND to invoke TSan. + TSAN_PATH="$THISDIR/../../third_party/tsan" + TSAN_SFX="$TSAN_PATH/tsan-x86-windows-sfx.exe" + echo "$TSAN_SFX" + if [ ! -f $TSAN_SFX ] + then + echo "Can't find ThreadSanitizer executables." + echo "See http://www.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/threadsanitizer-on-windows" + echo "for the instructions on how to get them." + exit 1 + fi + + chmod +x "$TSAN_SFX" # Cygwin won't run it without +x. + "$TSAN_SFX" -o"$TSAN_PATH"/unpacked -y + export PIN_COMMAND="$TSAN_PATH/unpacked/tsan-x86-windows/tsan.bat" + fi +fi + +# Add Chrome's Valgrind scripts dir to the PYTHON_PATH since it contains +# the scripts that are needed for this script to run +PYTHONPATH=$THISDIR/../python/google:$CHROME_VALGRIND_SCRIPTS python \ + "$THISDIR/libyuv_tests.py" $ARGV_COPY diff --git a/tools_libyuv/valgrind/memcheck/OWNERS b/tools_libyuv/valgrind/memcheck/OWNERS new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/tools_libyuv/valgrind/memcheck/OWNERS @@ -0,0 +1 @@ +* diff --git a/tools_libyuv/valgrind/memcheck/PRESUBMIT.py b/tools_libyuv/valgrind/memcheck/PRESUBMIT.py new file mode 100644 index 00000000..03329214 --- /dev/null +++ b/tools_libyuv/valgrind/memcheck/PRESUBMIT.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +""" +Copied from Chrome's src/tools/valgrind/memcheck/PRESUBMIT.py + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts +for more details on the presubmit API built into gcl. +""" + +import os +import re +import sys + +def CheckChange(input_api, output_api): + """Checks the memcheck suppressions files for bad data.""" + + # Add the path to the Chrome valgrind dir to the import path: + tools_vg_path = os.path.join(input_api.PresubmitLocalPath(), '..', '..', '..', + 'tools', 'valgrind') + sys.path.append(tools_vg_path) + import suppressions + + sup_regex = re.compile('suppressions.*\.txt$') + suppressions = {} + errors = [] + check_for_memcheck = False + # skip_next_line has 3 possible values: + # - False: don't skip the next line. + # - 'skip_suppression_name': the next line is a suppression name, skip. + # - 'skip_param': the next line is a system call parameter error, skip. + skip_next_line = False + for f in filter(lambda x: sup_regex.search(x.LocalPath()), + input_api.AffectedFiles()): + for line, line_num in zip(f.NewContents(), + xrange(1, len(f.NewContents()) + 1)): + line = line.lstrip() + if line.startswith('#') or not line: + continue + + if skip_next_line: + if skip_next_line == 'skip_suppression_name': + if 'insert_a_suppression_name_here' in line: + errors.append('"insert_a_suppression_name_here" is not a valid ' + 'suppression name') + if suppressions.has_key(line): + if f.LocalPath() == suppressions[line][1]: + errors.append('suppression with name "%s" at %s line %s ' + 'has already been defined at line %s' % + (line, f.LocalPath(), line_num, + suppressions[line][1])) + else: + errors.append('suppression with name "%s" at %s line %s ' + 'has already been defined at %s line %s' % + (line, f.LocalPath(), line_num, + suppressions[line][0], suppressions[line][1])) + else: + suppressions[line] = (f, line_num) + check_for_memcheck = True; + skip_next_line = False + continue + if check_for_memcheck: + if not line.startswith('Memcheck:'): + errors.append('"%s" should be "Memcheck:..." in %s line %s' % + (line, f.LocalPath(), line_num)) + check_for_memcheck = False; + if line == '{': + skip_next_line = 'skip_suppression_name' + continue + if line == "Memcheck:Param": + skip_next_line = 'skip_param' + continue + + if (line.startswith('fun:') or line.startswith('obj:') or + line.startswith('Memcheck:') or line == '}' or + line == '...'): + continue + errors.append('"%s" is probably wrong: %s line %s' % (line, f.LocalPath(), + line_num)) + if errors: + return [output_api.PresubmitError('\n'.join(errors))] + return [] + +def CheckChangeOnUpload(input_api, output_api): + return CheckChange(input_api, output_api) + +def CheckChangeOnCommit(input_api, output_api): + return CheckChange(input_api, output_api) + +def GetPreferredTrySlaves(): + # We don't have any memcheck slaves yet, so there's no use for this method. + # When we have, the slave name(s) should be put into this list. + return [] diff --git a/tools_libyuv/valgrind/memcheck/suppressions.txt b/tools_libyuv/valgrind/memcheck/suppressions.txt new file mode 100644 index 00000000..3ad0c8cc --- /dev/null +++ b/tools_libyuv/valgrind/memcheck/suppressions.txt @@ -0,0 +1,5 @@ +# This file is used in addition to the one already maintained in Chrome. +# It acts as a place holder for future additions for this project. +# It must exist for the Python wrapper script to work properly. + + diff --git a/tools_libyuv/valgrind/memcheck/suppressions_mac.txt b/tools_libyuv/valgrind/memcheck/suppressions_mac.txt new file mode 100644 index 00000000..3ad0c8cc --- /dev/null +++ b/tools_libyuv/valgrind/memcheck/suppressions_mac.txt @@ -0,0 +1,5 @@ +# This file is used in addition to the one already maintained in Chrome. +# It acts as a place holder for future additions for this project. +# It must exist for the Python wrapper script to work properly. + + diff --git a/tools_libyuv/valgrind/memcheck/suppressions_win32.txt b/tools_libyuv/valgrind/memcheck/suppressions_win32.txt new file mode 100644 index 00000000..3ad0c8cc --- /dev/null +++ b/tools_libyuv/valgrind/memcheck/suppressions_win32.txt @@ -0,0 +1,5 @@ +# This file is used in addition to the one already maintained in Chrome. +# It acts as a place holder for future additions for this project. +# It must exist for the Python wrapper script to work properly. + + |