From 8bf4e0ad584cbeb6fce2a0da1b2ab2f971bfb9c3 Mon Sep 17 00:00:00 2001 From: David Neto Date: Mon, 8 Jul 2019 13:53:33 -0400 Subject: Add Clang warning -Wextra-semi Also remove the one extra semicolon it found. --- cmake/utils.cmake | 3 +++ effcee/cursor.h | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 291be3c..19e81f2 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -37,6 +37,9 @@ function(effcee_default_c_compile_options TARGET) if (UNIX AND NOT MINGW) target_link_libraries(${TARGET} PUBLIC -pthread) endif() + if (${CMAKE_C_COMPILER_ID} MATCHES "Clang") + target_compile_options(${TARGET} PRIVATE -Wextra-semi) + endif() else() # disable warning C4800: 'int' : forcing value to bool 'true' or 'false' # (performance warning) diff --git a/effcee/cursor.h b/effcee/cursor.h index 82133f9..7776b7c 100644 --- a/effcee/cursor.h +++ b/effcee/cursor.h @@ -60,7 +60,7 @@ class Cursor { ++line_num_; } return *this; - }; + } private: // The remaining text, after all previous advancements. References the -- cgit v1.2.3 From 4bef5dbed590d1edfd3e34bc83d4141f41b998b0 Mon Sep 17 00:00:00 2001 From: David Neto Date: Mon, 8 Jul 2019 16:04:20 -0400 Subject: Require Python 3 --- .appveyor.yml | 1 + README.md | 4 ++-- cmake/setup_build.cmake | 24 +----------------------- examples/CMakeLists.txt | 2 +- examples/effcee-example-driver.py | 2 +- 5 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 6fb6bc5..20b9447 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -29,6 +29,7 @@ matrix: install: - git clone --depth=1 https://github.com/google/googletest.git third_party/googletest - git clone --depth=1 https://github.com/google/re2.git third_party/re2 + - set PATH=c:\Python36;%PATH% before_build: - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" (call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_amd64) diff --git a/README.md b/README.md index b7cd0ac..c447ecf 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ for more information. See also the [`AUTHORS`](AUTHORS) and Effcee depends on the [RE2][RE2] regular expression library. -Effcee tests depend on [Googletest][Googletest] and [Python][Python]. +Effcee tests depend on [Googletest][Googletest] and [Python 3][Python]. In the following sections, `$SOURCE_DIR` is the directory containing the Effcee source code. @@ -229,7 +229,7 @@ installed regardless of your OS: - A compiler supporting C++11. - [CMake][CMake]: for generating compilation targets. -- [Python][Python]: for a test script. +- [Python 3][Python]: for a test script. On Linux, if cross compiling to Windows: - [MinGW][MinGW]: A GCC-based cross compiler targeting Windows diff --git a/cmake/setup_build.cmake b/cmake/setup_build.cmake index 40749fc..9867de9 100644 --- a/cmake/setup_build.cmake +++ b/cmake/setup_build.cmake @@ -12,35 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# For cross-compilation, we need to use find_host_package -# in the remaining setup. But if not cross-compiling, then we -# need to alias find_host_package to find_package. -# Similar for find_host_program. if(NOT COMMAND find_host_package) macro(find_host_package) find_package(${ARGN}) endmacro() endif() -if(NOT COMMAND find_host_program) - macro(find_host_program) - find_program(${ARGN}) - endmacro() -endif() - -if (ANDROID) - # For android let's preemptively find the correct packages so that - # child projects (e.g. googletest) do not fail to find them. - find_host_package(PythonInterp) -endif() -foreach(PROGRAM echo python) - string(TOUPPER ${PROGRAM} PROG_UC) - if (ANDROID) - find_host_program(${PROG_UC}_EXE ${PROGRAM} REQUIRED) - else() - find_program(${PROG_UC}_EXE ${PROGRAM} REQUIRED) - endif() -endforeach(PROGRAM) +find_host_package(PythonInterp 3 REQUIRED) option(DISABLE_RTTI "Disable RTTI in builds") if(DISABLE_RTTI) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 32cdd29..3072df0 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,7 +13,7 @@ endif(WIN32 AND NOT MSVC) if(EFFCEE_BUILD_TESTING) add_test(NAME effcee-example - COMMAND ${PYTHON_EXE} + COMMAND ${PYTHON_EXECUTABLE} effcee-example-driver.py $ example_data.txt diff --git a/examples/effcee-example-driver.py b/examples/effcee-example-driver.py index e1b0eff..6c60bfb 100644 --- a/examples/effcee-example-driver.py +++ b/examples/effcee-example-driver.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2017 The Effcee Authors. # -- cgit v1.2.3 From 3842fdc5b596d1ea37ce1c1676fedc3d16f98f79 Mon Sep 17 00:00:00 2001 From: Diego Novillo Date: Mon, 19 Aug 2019 15:03:24 -0400 Subject: Add Bazel build rules. This adds support for building effcee with Bazel (https://bazel.build). It automatically pulls the googletest and re2 repositories when building with Bazel. I did not add a sha256 attribute to fix the version of the external repos to get, so by default this tracks top-of-master for those repos. It's easy to fix the versions by adding the sha256 attribute that Bazel suggests when it starts. --- BUILD.bazel | 91 +++++++++++++++++++++++++ DEVELOPMENT.howto.md | 49 +++++++------- README.md | 186 ++++++++++++++++++++++++++++----------------------- WORKSPACE | 21 ++++++ 4 files changed, 241 insertions(+), 106 deletions(-) create mode 100644 BUILD.bazel create mode 100644 WORKSPACE diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 index 0000000..ef23da9 --- /dev/null +++ b/BUILD.bazel @@ -0,0 +1,91 @@ +package( + default_visibility = ["//visibility:public"], +) + +# Description: +# +# Effcee is a C++ library for stateful pattern matching of strings inspired by +# LLVM's FileCheck. + +licenses(["notice"]) # Apache 2.0 + +exports_files([ + "CHANGES", + "LICENSE", +]) + +cc_library( + name = "effcee", + srcs = glob( + ["effcee/*.cc"], + exclude = ["effcee/*_test.cc"], + ), + hdrs = glob(["effcee/*.h"]), + compatible_with = [ + ], + deps = [ + "@com_googlesource_code_re2//:re2", + ], +) + +# Tests + +cc_test( + name = "check_test", + srcs = ["effcee/check_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "cursor_test", + srcs = ["effcee/cursor_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "diagnostic_test", + srcs = ["effcee/diagnostic_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "match_test", + srcs = ["effcee/match_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "options_test", + srcs = ["effcee/options_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "result_test", + srcs = ["effcee/result_test.cc"], + deps = [ + ":effcee", + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + ], +) diff --git a/DEVELOPMENT.howto.md b/DEVELOPMENT.howto.md index ce2afdd..6f066c4 100644 --- a/DEVELOPMENT.howto.md +++ b/DEVELOPMENT.howto.md @@ -1,54 +1,55 @@ # Developing for Effcee -Thank you for considering Effcee development! Please make sure you review +Thank you for considering Effcee development! Please make sure you review [`CONTRIBUTING.md`](CONTRIBUTING.md) for important preliminary info. ## Building Instructions for first-time building can be found in [`README.md`](README.md). -Incremental build after a source change can be done using `ninja` (or +Incremental build after a source change can be done using `bazel` or `ninja` (or `cmake --build`) and `ctest` exactly as in the first-time procedure. ## Issue tracking -We use GitHub issues to track bugs, enhancement requests, and questions. -See [the project's Issues page](https://github.com/google/effcee/issues). +We use GitHub issues to track bugs, enhancement requests, and questions. See +[the project's Issues page](https://github.com/google/effcee/issues). For all but the most trivial changes, we prefer that you file an issue before -submitting a pull request. An issue gives us context for your change: what -problem are you solving, and why. It also allows us to provide feedback on -your proposed solution before you invest a lot of effort implementing it. +submitting a pull request. An issue gives us context for your change: what +problem are you solving, and why. It also allows us to provide feedback on your +proposed solution before you invest a lot of effort implementing it. ## Code reviews All submissions are subject to review via the GitHub pull review process. Reviews will cover: -* *Correctness:* Does it work? Does it work in a multithreaded context? -* *Testing:* New functionality should be accompanied by tests. -* *Testability:* Can it easily be tested? This is proven with accompanying tests. -* *Design:* Is the solution fragile? Does it fit with the existing code? - Would it easily accommodate anticipated changes? -* *Ease of use:* Can a client get their work done with a minimum of fuss? - Are there unnecessarily surprising details? -* *Consistency:* Does it follow the style guidelines and the rest of the code? - Consistency reduces the work of future readers and maintainers. -* *Portability:* Does it work in many environments? +* *Correctness:* Does it work? Does it work in a multithreaded context? +* *Testing:* New functionality should be accompanied by tests. +* *Testability:* Can it easily be tested? This is proven with accompanying + tests. +* *Design:* Is the solution fragile? Does it fit with the existing code? Would + it easily accommodate anticipated changes? +* *Ease of use:* Can a client get their work done with a minimum of fuss? Are + there unnecessarily surprising details? +* *Consistency:* Does it follow the style guidelines and the rest of the code? + Consistency reduces the work of future readers and maintainers. +* *Portability:* Does it work in many environments? To respond to feedback, submit one or more *new* commits to the pull request branch. The project maintainer will normally clean up the submission by -squashing feedback response commits. We maintain a linear commit history, -so submission will be rebased onto master before merging. +squashing feedback response commits. We maintain a linear commit history, so +submission will be rebased onto master before merging. ## Testing There is a lot we won't say about testing. However: -* Most tests should be small scale, i.e. unit tests. -* Tests should run quickly. -* A test should: - * Check a single behaviour. This often corresponds to a use case. - * Have a three phase structure: setup, action, check. +* Most tests should be small scale, i.e. unit tests. +* Tests should run quickly. +* A test should: + * Check a single behaviour. This often corresponds to a use case. + * Have a three phase structure: setup, action, check. ## Coding style diff --git a/README.md b/README.md index c447ecf..663025b 100644 --- a/README.md +++ b/README.md @@ -2,16 +2,18 @@ [![Linux and OSX Build Status](https://travis-ci.org/google/effcee.svg)](https://travis-ci.org/google/effcee "Linux and OSX Build Status") -Effcee is a C++ library for stateful pattern matching of strings, -inspired by LLVM's [FileCheck][FileCheck] command. +Effcee is a C++ library for stateful pattern matching of strings, inspired by +LLVM's [FileCheck][FileCheck] command. Effcee: -- Is a library, so it can be used for quickly running tests in your own process. -- Is largely compatible with FileCheck, so tests and test-writing skills are - transferable. -- Has few dependencies: - - The C++11 standard library, and - - [RE2][RE2] for regular expression matching. + +- Is a library, so it can be used for quickly running tests in your own + process. +- Is largely compatible with FileCheck, so tests and test-writing skills are + transferable. +- Has few dependencies: + - The C++11 standard library, and + - [RE2][RE2] for regular expression matching. ## Example @@ -72,8 +74,8 @@ The following is from [examples/main.cc](examples/main.cc): ``` -For more examples, see the matching tests -in [effcee/match_test.cc](effcee/match_test.cc). +For more examples, see the matching tests in +[effcee/match_test.cc](effcee/match_test.cc). ## Status @@ -81,55 +83,58 @@ Effcee is mature enough to be relied upon by [third party projects](#what-uses-effcee), but could be improved. What works: -* All check types: CHECK, CHECK-NEXT, CHECK-SAME, CHECK-DAG, CHECK-LABEL, CHECK-NOT. -* Check strings can contain: - * fixed strings - * regular expressions - * variable definitions and uses -* Setting a custom check prefix. -* Accurate and helpful reporting of match failures. + +* All check types: CHECK, CHECK-NEXT, CHECK-SAME, CHECK-DAG, CHECK-LABEL, + CHECK-NOT. +* Check strings can contain: + * fixed strings + * regular expressions + * variable definitions and uses +* Setting a custom check prefix. +* Accurate and helpful reporting of match failures. What is left to do: -* Add an option to define shorthands for regular expressions. - * For example, you could express that if the string `%%` appears where a - regular expression is expected, then it expands to the regular expression - for a local identifier in LLVM assembly language, i.e. - `%[-a-zA-Z$._][-a-zA-Z$._0-9]*`. - This enables you to write precise tests with less fuss. -* Better error reporting for failure to parse the checks list. -* Write a check language reference and tutorial. + +* Add an option to define shorthands for regular expressions. + * For example, you could express that if the string `%%` appears where a + regular expression is expected, then it expands to the regular + expression for a local identifier in LLVM assembly language, i.e. + `%[-a-zA-Z$._][-a-zA-Z$._0-9]*`. This enables you to write precise tests + with less fuss. +* Better error reporting for failure to parse the checks list. +* Write a check language reference and tutorial. What is left to do, but lower priority: -* Match full lines. -* Strict whitespace. -* Implicit check-not. -* Variable scoping. + +* Match full lines. +* Strict whitespace. +* Implicit check-not. +* Variable scoping. ## Licensing and contributing -Effcee is licensed under terms of the [Apache 2.0 license](LICENSE). If you -are interested in contributing to this project, please see +Effcee is licensed under terms of the [Apache 2.0 license](LICENSE). If you are +interested in contributing to this project, please see [`CONTRIBUTING.md`](CONTRIBUTING.md). This is not an official Google product (experimental or otherwise), it is just -code that happens to be owned by Google. That may change if Effcee gains -contributions from others. See the [`CONTRIBUTING.md`](CONTRIBUTING.md) file -for more information. See also the [`AUTHORS`](AUTHORS) and +code that happens to be owned by Google. That may change if Effcee gains +contributions from others. See the [`CONTRIBUTING.md`](CONTRIBUTING.md) file for +more information. See also the [`AUTHORS`](AUTHORS) and [`CONTRIBUTORS`](CONTRIBUTORS) files. ## File organization -- [`effcee`/](effcee) : library source code, and tests -- `third_party/`: third party open source packages, downloaded - separately -- [`examples/`](examples): example programs +- [`effcee`/](effcee) : library source code, and tests +- `third_party/`: third party open source packages, downloaded separately +- [`examples/`](examples): example programs Effcee depends on the [RE2][RE2] regular expression library. Effcee tests depend on [Googletest][Googletest] and [Python 3][Python]. -In the following sections, `$SOURCE_DIR` is the directory containing the -Effcee source code. +In the following sections, `$SOURCE_DIR` is the directory containing the Effcee +source code. ## Getting and building Effcee @@ -144,17 +149,24 @@ cd $SOURCE_DIR/ ``` Note: There are two other ways to manage third party sources: -- If you are building Effcee as part of a larger CMake-based project, - add the RE2 and `googletest` projects before adding Effcee. -- Otherwise, you can set CMake variables to point to third party sources - if they are located somewhere else. See the [Build options](#build-options) below. + +- If you are building Effcee with Bazel (https://bazel.build), you do not need + to clone the repositories for `googletest` and `re2`. They will be + automatically downloaded by Bazel during build. Bazel will suggest adding + `sha256` attributes to each repository rule to get hermetic builds (these + notices are safe to ignore if you are not interested in hermetic builds). +- If you are building Effcee as part of a larger CMake-based project, add the + RE2 and `googletest` projects before adding Effcee. +- Otherwise, you can set CMake variables to point to third party sources if + they are located somewhere else. See the [Build options](#build-options) + below. 2) Ensure you have the requisite tools -- see the tools subsection below. 3) Decide where to place the build output. In the following steps, we'll call it - `$BUILD_DIR`. Any new directory should work. We recommend building outside - the source tree, but it is also common to build in a (new) subdirectory of - `$SOURCE_DIR`, such as `$SOURCE_DIR/build`. +`$BUILD_DIR`. Any new directory should work. We recommend building outside the +source tree, but it is also common to build in a (new) subdirectory of +`$SOURCE_DIR`, such as `$SOURCE_DIR/build`. 4a) Build and test with Ninja on Linux or Windows: @@ -174,9 +186,8 @@ cmake --build . --config {Release|Debug|MinSizeRel|RelWithDebInfo} ctest -C {Release|Debug|MinSizeRel|RelWithDebInfo} ``` -4c) Or build with MinGW on Linux for Windows: -(Skip building threaded unit tests due to -[Googletest bug 606](https://github.com/google/googletest/issues/606)) +4c) Or build with MinGW on Linux for Windows: (Skip building threaded unit tests +due to [Googletest bug 606](https://github.com/google/googletest/issues/606)) ```sh cd $BUILD_DIR @@ -186,19 +197,27 @@ cmake -GNinja -DCMAKE_BUILD_TYPE={Debug|Release|RelWithDebInfo} $SOURCE_DIR \ ninja ``` -After a successful build, you should have a `libeffcee` library under -the `$BUILD_DIR/effcee/` directory. +4d) Or build with Bazel on Linux: + +```sh +cd $SOURCE_DIR +bazel build -c opt :all +``` + +After a successful build, you should have a `libeffcee` library under the +`$BUILD_DIR/effcee/` directory (or `$SOURCE_DIR/bazel-bin` when building with +Bazel). The default behavior on MSVC is to link with the static CRT. If you would like -to change this behavior `-DEFFCEE_ENABLE_SHARED_CRT` may be passed on the -cmake configure line. +to change this behavior `-DEFFCEE_ENABLE_SHARED_CRT` may be passed on the cmake +configure line. ### Tests By default, Effcee registers two tests with `ctest`: -* `effcee-test`: All library tests, based on Googletest. -* `effcee-example`: Executes the example executable with sample inputs. +* `effcee-test`: All library tests, based on Googletest. +* `effcee-example`: Executes the example executable with sample inputs. Running `ctest` without arguments will run the tests for Effcee as well as for RE2. @@ -210,8 +229,8 @@ configuration time: cmake -GNinja -DEFFCEE_BUILD_TESTING=OFF ... ``` -The RE2 tests run much longer, so if you're working on Effcee alone, we -suggest limiting ctest to tests with prefix `effcee`: +The RE2 tests run much longer, so if you're working on Effcee alone, we suggest +limiting ctest to tests with prefix `effcee`: ctest -R effcee @@ -227,39 +246,42 @@ cmake -GNinja -DRE2_BUILD_TESTING=OFF ... For building, testing, and profiling Effcee, the following tools should be installed regardless of your OS: -- A compiler supporting C++11. -- [CMake][CMake]: for generating compilation targets. -- [Python 3][Python]: for a test script. +- A compiler supporting C++11. +- [CMake][CMake]: for generating compilation targets. +- [Python 3][Python]: for a test script. -On Linux, if cross compiling to Windows: -- [MinGW][MinGW]: A GCC-based cross compiler targeting Windows - so that generated executables use the Microsoft C runtime libraries. +On Linux, if cross compiling to Windows: - [MinGW][MinGW]: A GCC-based cross +compiler targeting Windows so that generated executables use the Microsoft C +runtime libraries. On Windows, the following tools should be installed and available on your path: -- Visual Studio 2015 or later. Previous versions of Visual Studio are not usable - with RE2 or Googletest. -- Git - including the associated tools, Bash, `diff`. +- Visual Studio 2015 or later. Previous versions of Visual Studio are not + usable with RE2 or Googletest. +- Git - including the associated tools, Bash, `diff`. ### Build options Third party source locations: -- `EFFCEE_GOOGLETEST_DIR`: Location of `googletest` sources, if not under - `third_party`. -- `EFFCEE_RE2_DIR`: Location of `re2` sources, if not under `third_party`. -- `EFFCEE_THIRD_PARTY_ROOT_DIR`: Alternate location for `googletest` and - `re2` subdirectories. This is used if the sources are not located under - the `third_party` directory, and if the previous two variables are not set. + +- `EFFCEE_GOOGLETEST_DIR`: Location of `googletest` sources, if not under + `third_party`. +- `EFFCEE_RE2_DIR`: Location of `re2` sources, if not under `third_party`. +- `EFFCEE_THIRD_PARTY_ROOT_DIR`: Alternate location for `googletest` and `re2` + subdirectories. This is used if the sources are not located under the + `third_party` directory, and if the previous two variables are not set. Compilation options: -- `DISABLE_RTTI`. Disable runtime type information. Default is enabled. -- `DISABLE_EXCEPTIONS`. Disable exceptions. Default is enabled. -- `EFFCEE_ENABLE_SHARED_CRT`. See above. + +- `DISABLE_RTTI`. Disable runtime type information. Default is enabled. +- `DISABLE_EXCEPTIONS`. Disable exceptions. Default is enabled. +- `EFFCEE_ENABLE_SHARED_CRT`. See above. Controlling samples and tests: -- `EFFCEE_BUILD_SAMPLES`. Should Effcee examples be built? Defaults to `ON`. -- `EFFCEE_BUILD_TESTING`. Should Effcee tests be built? Defaults to `ON`. -- `RE2_BUILD_TESTING`. Should RE2 tests be built? Defaults to `ON`. + +- `EFFCEE_BUILD_SAMPLES`. Should Effcee examples be built? Defaults to `ON`. +- `EFFCEE_BUILD_TESTING`. Should Effcee tests be built? Defaults to `ON`. +- `RE2_BUILD_TESTING`. Should RE2 tests be built? Defaults to `ON`. ## Bug tracking @@ -268,9 +290,9 @@ We track bugs using GitHub -- click on the "Issues" button on ## What uses Effcee? -- [Tests](https://github.com/Microsoft/DirectXShaderCompiler/tree/master/tools/clang/test/CodeGenSPIRV) - for SPIR-V code generation in the [DXC][DXC] HLSL compiler. -- Tests for [SPIRV-Tools][SPIRV-Tools] +- [Tests](https://github.com/Microsoft/DirectXShaderCompiler/tree/master/tools/clang/test/CodeGenSPIRV) + for SPIR-V code generation in the [DXC][DXC] HLSL compiler. +- Tests for [SPIRV-Tools][SPIRV-Tools] ## References diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..8f2869b --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,21 @@ +workspace(name = "effcee") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_cc", + strip_prefix = "rules_cc-master", + urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"], +) + +http_archive( + name = "com_google_googletest", + strip_prefix = "googletest-master", + urls = ["https://github.com/google/googletest/archive/master.zip"], +) + +http_archive( + name = "com_googlesource_code_re2", + strip_prefix = "re2-master", + urls = ["https://github.com/google/re2/archive/master.zip"], +) -- cgit v1.2.3 From ecbc165b47c95ddc07037680c2271190cbc3b86d Mon Sep 17 00:00:00 2001 From: David Neto Date: Fri, 23 Aug 2019 19:07:42 -0400 Subject: Add effcee-fuzz It's only built if you specify a CMake variable pointing at the FuzzedDataProvider.h source file from LLVM's compiler-rt. effcee-fuzz runs a simple matcher against standard input. This can be used to simply and easily reproduce OSS-Fuzz failures. --- CMakeLists.txt | 1 + fuzzer/CMakeLists.txt | 18 ++++++++++++++++++ fuzzer/effcee_fuzz.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 fuzzer/CMakeLists.txt create mode 100644 fuzzer/effcee_fuzz.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 67087b7..a521ac3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ include(cmake/utils.cmake) add_subdirectory(third_party) add_subdirectory(effcee) +add_subdirectory(fuzzer) if(${EFFCEE_BUILD_SAMPLES}) add_subdirectory(examples) diff --git a/fuzzer/CMakeLists.txt b/fuzzer/CMakeLists.txt new file mode 100644 index 0000000..c0d0205 --- /dev/null +++ b/fuzzer/CMakeLists.txt @@ -0,0 +1,18 @@ +if (EXISTS "${EFFCEE_FUZZED_DATA_PROVIDER_DIR}/FuzzedDataProvider.h") + message(STATUS "effcee: configuring effcee-fuzz") + add_executable(effcee-fuzz effcee_fuzz.cc) + effcee_default_compile_options(effcee-fuzz) + target_include_directories(effcee-fuzz PRIVATE "${EFFCEE_FUZZED_DATA_PROVIDER_DIR}") + target_link_libraries(effcee-fuzz PRIVATE effcee) + + if(UNIX AND NOT MINGW) + set_target_properties(effcee-fuzz PROPERTIES LINK_FLAGS -pthread) + endif() + if (WIN32 AND NOT MSVC) + # For MinGW cross-compile, statically link to the C++ runtime + set_target_properties(effcee-fuzz PROPERTIES + LINK_FLAGS "-static -static-libgcc -static-libstdc++") + endif(WIN32 AND NOT MSVC) +else() + message(STATUS "effcee: effcee-fuzz won't be built. Can't find FuzzedDataProvider.h") +endif() diff --git a/fuzzer/effcee_fuzz.cc b/fuzzer/effcee_fuzz.cc new file mode 100644 index 0000000..6ffe274 --- /dev/null +++ b/fuzzer/effcee_fuzz.cc @@ -0,0 +1,49 @@ +// Copyright 2019 The Effcee Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "FuzzedDataProvider.h" +#include "effcee/effcee.h" + +// Consumes standard input as a fuzzer input, breaks it apart into text +// and a check, then runs a basic match. +int main(int argc, char* argv[]) { + std::vector input; + // Read standard input into a buffer. + { + if (FILE* fp = freopen(nullptr, "rb", stdin)) { + uint8_t chunk[1024]; + while (size_t len = fread(chunk, sizeof(uint8_t), sizeof(chunk), fp)) { + input.insert(input.end(), chunk, chunk + len); + } + if (ftell(fp) == -1L) { + if (ferror(fp)) { + fprintf(stderr, "error: error reading standard input"); + } + return 1; + } + } else { + fprintf(stderr, "error: couldn't reopen stdin for binary reading"); + } + } + + // This is very basic, but can find bugs. + FuzzedDataProvider stream(input.data(), input.size()); + std::string text = stream.ConsumeRandomLengthString(input.size()); + std::string checks = stream.ConsumeRemainingBytesAsString(); + effcee::Match(text, checks); + return 0; +} -- cgit v1.2.3 From 0eb6499adf7a4debb3cdc87f9c469b45f8d65b5a Mon Sep 17 00:00:00 2001 From: David Neto Date: Sat, 24 Aug 2019 01:48:08 -0400 Subject: Fail parsing checks if the regexp is bad. BUG=129514492 --- effcee/check.cc | 21 ++++++++++++++------- effcee/check_test.cc | 7 +++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/effcee/check.cc b/effcee/check.cc index 2751801..08f7b7a 100644 --- a/effcee/check.cc +++ b/effcee/check.cc @@ -130,10 +130,10 @@ bool Check::Matches(StringPiece* input, StringPiece* captured, } namespace { -// Returns a parts list for the given pattern. This splits out regular -// expressions as delimited by {{ and }}, and also variable uses and -// definitions. -Check::Parts PartsForPattern(StringPiece pattern) { +// Returns a Result and a parts list for the given pattern. This splits out +// regular expressions as delimited by {{ and }}, and also variable uses and +// definitions. This can fail when a regular expression is invalid. +std::pair PartsForPattern(StringPiece pattern) { Check::Parts parts; StringPiece fixed, regex, var; @@ -160,6 +160,12 @@ Check::Parts PartsForPattern(StringPiece pattern) { } if (!regex.empty()) { parts.emplace_back(make_unique(Type::Regex, regex)); + if (parts.back()->NumCapturingGroups() < 0) { + return std::make_pair( + Result(Result::Status::BadRule, + std::string("invalid regex: ") + ToString(regex)), + Check::Parts()); + } } } else if (var_exists && (!regex_exists || var_start < regex_start)) { const auto consumed = @@ -191,7 +197,7 @@ Check::Parts PartsForPattern(StringPiece pattern) { } } - return parts; + return std::make_pair(Result(Result::Status::Ok), std::move(parts)); } } // namespace @@ -234,8 +240,9 @@ std::pair ParseChecks(StringPiece str, StringPiece suffix; if (RE2::PartialMatch(line, regexp, &suffix, &matched_param)) { const Type type = TypeForSuffix(suffix); - auto parts(PartsForPattern(matched_param)); - check_list.push_back(Check(type, matched_param, std::move(parts))); + auto parts = PartsForPattern(matched_param); + if (!parts.first) return std::make_pair(parts.first, CheckList()); + check_list.push_back(Check(type, matched_param, std::move(parts.second))); } cursor.AdvanceLine(); } diff --git a/effcee/check_test.cc b/effcee/check_test.cc index c9d0674..f7089ef 100644 --- a/effcee/check_test.cc +++ b/effcee/check_test.cc @@ -246,6 +246,13 @@ INSTANTIATE_TEST_SUITE_P(AllCheckTypes, ParseChecksTypeFailTest, "FOO"}), ValuesIn(AllCheckTypesAsPairs()))); +TEST(ParseChecks, BadRegexpFails) { + const auto parsed = ParseChecks("CHECK: {{\\}}", Options()); + EXPECT_THAT(parsed.first.status(), Eq(Status::BadRule)); + EXPECT_THAT(parsed.first.message(), HasSubstr("invalid regex: \\")); + EXPECT_THAT(parsed.second, Eq(CheckList({}))); +} + TEST(ParseChecks, CheckSameCantBeFirst) { const auto parsed = ParseChecks("CHECK-SAME: now", Options()); EXPECT_THAT(parsed.first.status(), Eq(Status::BadRule)); -- cgit v1.2.3 From 6527fb25482ee16f0ae8c735d1d0c33f7d5f220a Mon Sep 17 00:00:00 2001 From: David Neto Date: Mon, 26 Aug 2019 14:15:44 -0400 Subject: Fail parsing checks if var def regexp is bad e.g. bad pattern [[FOO:?]] The question mark is a quantifier that doesn't quantify anything. BUG=137632977 --- effcee/check.cc | 8 ++++++++ effcee/check_test.cc | 10 +++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/effcee/check.cc b/effcee/check.cc index 08f7b7a..c078eff 100644 --- a/effcee/check.cc +++ b/effcee/check.cc @@ -188,6 +188,14 @@ std::pair PartsForPattern(StringPiece pattern) { StringPiece expression = var.substr(colon + 1, StringPiece::npos); parts.emplace_back( make_unique(Type::VarDef, var, name, expression)); + if (parts.back()->NumCapturingGroups() < 0) { + return std::make_pair( + Result( + Result::Status::BadRule, + std::string("invalid regex in variable definition for ") + + ToString(name) + ": " + ToString(expression)), + Check::Parts()); + } } } } else { diff --git a/effcee/check_test.cc b/effcee/check_test.cc index f7089ef..d5c118a 100644 --- a/effcee/check_test.cc +++ b/effcee/check_test.cc @@ -246,13 +246,21 @@ INSTANTIATE_TEST_SUITE_P(AllCheckTypes, ParseChecksTypeFailTest, "FOO"}), ValuesIn(AllCheckTypesAsPairs()))); -TEST(ParseChecks, BadRegexpFails) { +TEST(ParseChecks, BadRegexpMatchTrailingSlashFails) { const auto parsed = ParseChecks("CHECK: {{\\}}", Options()); EXPECT_THAT(parsed.first.status(), Eq(Status::BadRule)); EXPECT_THAT(parsed.first.message(), HasSubstr("invalid regex: \\")); EXPECT_THAT(parsed.second, Eq(CheckList({}))); } +TEST(ParseChecks, BadRegexpVardefUnboundOptionalFails) { + const auto parsed = ParseChecks("CHECK: [[VAR:?]]", Options()); + EXPECT_THAT(parsed.first.status(), Eq(Status::BadRule)); + EXPECT_THAT(parsed.first.message(), + HasSubstr("invalid regex in variable definition for VAR: ?")); + EXPECT_THAT(parsed.second, Eq(CheckList({}))); +} + TEST(ParseChecks, CheckSameCantBeFirst) { const auto parsed = ParseChecks("CHECK-SAME: now", Options()); EXPECT_THAT(parsed.first.status(), Eq(Status::BadRule)); -- cgit v1.2.3