llvm.org GIT mirror llvm / 18c6fab
Pull google/benchmark library to the LLVM tree This patch pulls google/benchmark v1.4.1 into the LLVM tree so that any project could use it for benchmark generation. A dummy benchmark is added to `llvm/benchmarks/DummyYAML.cpp` to validate the correctness of the build process. The current version does not utilize LLVM LNT and LLVM CMake infrastructure, but that might be sufficient for most users. Two introduced CMake variables: * `LLVM_INCLUDE_BENCHMARKS` (`ON` by default) generates benchmark targets * `LLVM_BUILD_BENCHMARKS` (`OFF` by default) adds generated benchmark targets to the list of default LLVM targets (i.e. if `ON` benchmarks will be built upon standard build invocation, e.g. `ninja` or `make` with no specific targets) List of modifications: * `BENCHMARK_ENABLE_TESTING` is disabled * `BENCHMARK_ENABLE_EXCEPTIONS` is disabled * `BENCHMARK_ENABLE_INSTALL` is disabled * `BENCHMARK_ENABLE_GTEST_TESTS` is disabled * `BENCHMARK_DOWNLOAD_DEPENDENCIES` is disabled Original discussion can be found here: http://lists.llvm.org/pipermail/llvm-dev/2018-August/125023.html Reviewed by: dberris, lebedev.ri Subscribers: ilya-biryukov, ioeric, EricWF, lebedev.ri, srhines, dschuff, mgorny, krytarowski, fedor.sergeev, mgrang, jfb, llvm-commits Differential Revision: https://reviews.llvm.org/D50894 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@340809 91177308-0d34-0410-b5e6-96231b3b80d8 Kirill Bobyrev 1 year, 19 days ago
106 changed file(s) with 14142 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
491491 "Build LLVM unit tests. If OFF, just generate build targets." OFF)
492492 option(LLVM_INCLUDE_TESTS "Generate build targets for the LLVM unit tests." ON)
493493 option(LLVM_INCLUDE_GO_TESTS "Include the Go bindings tests in test build targets." ON)
494
495 option(LLVM_BUILD_BENCHMARKS "Add LLVM benchmark targets to the list of default
496 targets. If OFF, benchmarks still could be built using Benchmarks target." OFF)
497 option(LLVM_INCLUDE_BENCHMARKS "Generate benchmark targets. If OFF, benchmarks can't be built." ON)
494498
495499 option (LLVM_BUILD_DOCS "Build the llvm documentation." OFF)
496500 option (LLVM_INCLUDE_DOCS "Generate build targets for llvm documentation." ON)
10121016 if (MSVC AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows" AND CMAKE_INSTALL_UCRT_LIBRARIES)
10131017 include(InstallRequiredSystemLibraries)
10141018 endif()
1019
1020 if (LLVM_INCLUDE_BENCHMARKS)
1021 # Override benchmark defaults so that when the library itself is updated these
1022 # modifications are not lost.
1023 set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing" FORCE)
1024 set(BENCHMARK_ENABLE_EXCEPTIONS OFF CACHE BOOL "Disable benchmark exceptions" FORCE)
1025 set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Don't install benchmark" FORCE)
1026 set(BENCHMARK_DOWNLOAD_DEPENDENCIES OFF CACHE BOOL "Don't download dependencies" FORCE)
1027 set(BENCHMARK_ENABLE_GTEST_TESTS OFF CACHE BOOL "Disable Google Test in benchmark" FORCE)
1028
1029 add_subdirectory(utils/benchmark)
1030 add_subdirectory(benchmarks)
1031 endif()
0 set(LLVM_LINK_COMPONENTS
1 Support)
2
3 add_benchmark(DummyYAML DummyYAML.cpp)
0 #include "benchmark/benchmark.h"
1 #include "llvm/Support/YAMLTraits.h"
2
3 static void BM_YAMLDummyIsNumeric(benchmark::State& state) {
4 std::string x = "hello";
5 for (auto _ : state) {
6 std::string copy(x);
7 llvm::yaml::isNumeric(copy);
8 }
9 }
10 BENCHMARK(BM_YAMLDummyIsNumeric);
11
12 BENCHMARK_MAIN();
10831083 # Our current version of gtest does not properly recognize C++11 support
10841084 # with MSVC, so it falls back to tr1 / experimental classes. Since LLVM
10851085 # itself requires C++11, we can safely force it on unconditionally so that
1086 # we don't have to fight with the buggy gtest check.
1086 # we don't have to fight with the buggy gtest check.
10871087 add_definitions(-DGTEST_LANG_CXX11=1)
10881088 add_definitions(-DGTEST_HAS_TR1_TUPLE=0)
10891089
11171117 if (NOT ${test_suite_folder} STREQUAL "NOTFOUND")
11181118 set_property(TARGET ${test_name} PROPERTY FOLDER "${test_suite_folder}")
11191119 endif ()
1120 endfunction()
1121
1122 # Generic support for adding a benchmark.
1123 function(add_benchmark benchmark_name)
1124 if( NOT LLVM_BUILD_BENCHMARKS )
1125 set(EXCLUDE_FROM_ALL ON)
1126 endif()
1127
1128 add_llvm_executable(${benchmark_name} IGNORE_EXTERNALIZE_DEBUGINFO NO_INSTALL_RPATH ${ARGN})
1129 set(outdir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR})
1130 set_output_directory(${benchmark_name} BINARY_DIR ${outdir} LIBRARY_DIR ${outdir})
1131 target_link_libraries(${benchmark_name} PRIVATE benchmark)
11201132 endfunction()
11211133
11221134 function(llvm_add_go_executable binary pkgpath)
248248 Generate build targets for the LLVM unit tests. Defaults to ON. You can use
249249 this option to disable the generation of build targets for the LLVM unit
250250 tests.
251
252 **LLVM_BUILD_BENCHMARKS**:BOOL
253 Adds benchmarks to the list of default targets. Defaults to OFF.
254
255 **LLVM_INCLUDE_BENCHMARKS**:BOOL
256 Generate build targets for the LLVM benchmarks. Defaults to ON.
251257
252258 **LLVM_APPEND_VC_REV**:BOOL
253259 Embed version control revision info (svn revision number or Git revision id).
0 # This is the official list of benchmark authors for copyright purposes.
1 # This file is distinct from the CONTRIBUTORS files.
2 # See the latter for an explanation.
3 #
4 # Names should be added to this file as:
5 # Name or Organization
6 # The email address is not required for organizations.
7 #
8 # Please keep the list sorted.
9
10 Albert Pretorius
11 Arne Beer
12 Carto
13 Christopher Seymour
14 David Coeurjolly
15 Deniz Evrenci
16 Dirac Research
17 Dominik Czarnota
18 Eric Fiselier
19 Eugene Zhuk
20 Evgeny Safronov
21 Felix Homann
22 Google Inc.
23 International Business Machines Corporation
24 Ismael Jimenez Martinez
25 Jern-Kuan Leong
26 JianXiong Zhou
27 Joao Paulo Magalhaes
28 Jussi Knuuttila
29 Kaito Udagawa
30 Kishan Kumar
31 Lei Xu
32 Matt Clarkson
33 Maxim Vafin
34 MongoDB Inc.
35 Nick Hutchinson
36 Oleksandr Sochka
37 Paul Redmond
38 Radoslav Yovchev
39 Roman Lebedev
40 Shuo Chen
41 Steinar H. Gunderson
42 Stripe, Inc.
43 Yixuan Qiu
44 Yusuke Suzuki
45 Zbigniew Skowron
0 cmake_minimum_required (VERSION 2.8.12)
1
2 project (benchmark)
3
4 foreach(p
5 CMP0054 # CMake 3.1
6 CMP0056 # export EXE_LINKER_FLAGS to try_run
7 CMP0057 # Support no if() IN_LIST operator
8 )
9 if(POLICY ${p})
10 cmake_policy(SET ${p} NEW)
11 endif()
12 endforeach()
13
14 option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." ON)
15 option(BENCHMARK_ENABLE_EXCEPTIONS "Enable the use of exceptions in the benchmark library." ON)
16 option(BENCHMARK_ENABLE_LTO "Enable link time optimisation of the benchmark library." OFF)
17 option(BENCHMARK_USE_LIBCXX "Build and test using libc++ as the standard library." OFF)
18 option(BENCHMARK_BUILD_32_BITS "Build a 32 bit version of the library." OFF)
19 option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark. (Projects embedding benchmark may want to turn this OFF.)" ON)
20
21 # Allow unmet dependencies to be met using CMake's ExternalProject mechanics, which
22 # may require downloading the source code.
23 option(BENCHMARK_DOWNLOAD_DEPENDENCIES "Allow the downloading and in-tree building of unmet dependencies" OFF)
24
25 # This option can be used to disable building and running unit tests which depend on gtest
26 # in cases where it is not possible to build or find a valid version of gtest.
27 option(BENCHMARK_ENABLE_GTEST_TESTS "Enable building the unit tests which depend on gtest" OFF)
28
29 set(ENABLE_ASSEMBLY_TESTS_DEFAULT OFF)
30 function(should_enable_assembly_tests)
31 if(CMAKE_BUILD_TYPE)
32 string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
33 if (${CMAKE_BUILD_TYPE_LOWER} MATCHES "coverage")
34 # FIXME: The --coverage flag needs to be removed when building assembly
35 # tests for this to work.
36 return()
37 endif()
38 endif()
39 if (MSVC)
40 return()
41 elseif(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
42 return()
43 elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8)
44 # FIXME: Make these work on 32 bit builds
45 return()
46 elseif(BENCHMARK_BUILD_32_BITS)
47 # FIXME: Make these work on 32 bit builds
48 return()
49 endif()
50 find_program(LLVM_FILECHECK_EXE FileCheck)
51 if (LLVM_FILECHECK_EXE)
52 set(LLVM_FILECHECK_EXE "${LLVM_FILECHECK_EXE}" CACHE PATH "llvm filecheck" FORCE)
53 message(STATUS "LLVM FileCheck Found: ${LLVM_FILECHECK_EXE}")
54 else()
55 message(STATUS "Failed to find LLVM FileCheck")
56 return()
57 endif()
58 set(ENABLE_ASSEMBLY_TESTS_DEFAULT ON PARENT_SCOPE)
59 endfunction()
60 should_enable_assembly_tests()
61
62 # This option disables the building and running of the assembly verification tests
63 option(BENCHMARK_ENABLE_ASSEMBLY_TESTS "Enable building and running the assembly tests"
64 ${ENABLE_ASSEMBLY_TESTS_DEFAULT})
65
66 # Make sure we can import out CMake functions
67 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules")
68 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
69
70
71 # Read the git tags to determine the project version
72 include(GetGitVersion)
73 get_git_version(GIT_VERSION)
74
75 # Tell the user what versions we are using
76 string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" VERSION ${GIT_VERSION})
77 message("-- Version: ${VERSION}")
78
79 # The version of the libraries
80 set(GENERIC_LIB_VERSION ${VERSION})
81 string(SUBSTRING ${VERSION} 0 1 GENERIC_LIB_SOVERSION)
82
83 # Import our CMake modules
84 include(CheckCXXCompilerFlag)
85 include(AddCXXCompilerFlag)
86 include(CXXFeatureCheck)
87
88 if (BENCHMARK_BUILD_32_BITS)
89 add_required_cxx_compiler_flag(-m32)
90 endif()
91
92 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
93 # Turn compiler warnings up to 11
94 string(REGEX REPLACE "[-/]W[1-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
95 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4")
96 add_definitions(-D_CRT_SECURE_NO_WARNINGS)
97
98 if (NOT BENCHMARK_ENABLE_EXCEPTIONS)
99 add_cxx_compiler_flag(-EHs-)
100 add_cxx_compiler_flag(-EHa-)
101 endif()
102 # Link time optimisation
103 if (BENCHMARK_ENABLE_LTO)
104 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GL")
105 set(CMAKE_STATIC_LINKER_FLAGS_RELEASE "${CMAKE_STATIC_LINKER_FLAGS_RELEASE} /LTCG")
106 set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG")
107 set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG")
108
109 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /GL")
110 string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO}")
111 set(CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_STATIC_LINKER_FLAGS_RELWITHDEBINFO} /LTCG")
112 string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO}")
113 set(CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_SHARED_LINKER_FLAGS_RELWITHDEBINFO} /LTCG")
114 string(REGEX REPLACE "[-/]INCREMENTAL" "/INCREMENTAL:NO" CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO}")
115 set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} /LTCG")
116
117 set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} /GL")
118 set(CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL "${CMAKE_STATIC_LINKER_FLAGS_MINSIZEREL} /LTCG")
119 set(CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL "${CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL} /LTCG")
120 set(CMAKE_EXE_LINKER_FLAGS_MINSIZEREL "${CMAKE_EXE_LINKER_FLAGS_MINSIZEREL} /LTCG")
121 endif()
122 else()
123 # Try and enable C++11. Don't use C++14 because it doesn't work in some
124 # configurations.
125 add_cxx_compiler_flag(-std=c++11)
126 if (NOT HAVE_CXX_FLAG_STD_CXX11)
127 add_cxx_compiler_flag(-std=c++0x)
128 endif()
129
130 # Turn compiler warnings up to 11
131 add_cxx_compiler_flag(-Wall)
132
133 add_cxx_compiler_flag(-Wextra)
134 add_cxx_compiler_flag(-Wshadow)
135 # FIXME(kbobyrev): Document this change.
136 # add_cxx_compiler_flag(-Werror RELEASE)
137 # add_cxx_compiler_flag(-Werror RELWITHDEBINFO)
138 # add_cxx_compiler_flag(-Werror MINSIZEREL)
139 add_cxx_compiler_flag(-pedantic)
140 add_cxx_compiler_flag(-pedantic-errors)
141 add_cxx_compiler_flag(-Wshorten-64-to-32)
142 add_cxx_compiler_flag(-Wfloat-equal)
143 add_cxx_compiler_flag(-fstrict-aliasing)
144 if (NOT BENCHMARK_ENABLE_EXCEPTIONS)
145 add_cxx_compiler_flag(-fno-exceptions)
146 endif()
147
148 if (HAVE_CXX_FLAG_FSTRICT_ALIASING)
149 if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") #ICC17u2: Many false positives for Wstrict-aliasing
150 add_cxx_compiler_flag(-Wstrict-aliasing)
151 endif()
152 endif()
153 # ICC17u2: overloaded virtual function "benchmark::Fixture::SetUp" is only partially overridden
154 # (because of deprecated overload)
155 add_cxx_compiler_flag(-wd654)
156 add_cxx_compiler_flag(-Wthread-safety)
157 if (HAVE_CXX_FLAG_WTHREAD_SAFETY)
158 cxx_feature_check(THREAD_SAFETY_ATTRIBUTES)
159 endif()
160
161 # On most UNIX like platforms g++ and clang++ define _GNU_SOURCE as a
162 # predefined macro, which turns on all of the wonderful libc extensions.
163 # However g++ doesn't do this in Cygwin so we have to define it ourselfs
164 # since we depend on GNU/POSIX/BSD extensions.
165 if (CYGWIN)
166 add_definitions(-D_GNU_SOURCE=1)
167 endif()
168
169 # Link time optimisation
170 if (BENCHMARK_ENABLE_LTO)
171 add_cxx_compiler_flag(-flto)
172 if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
173 find_program(GCC_AR gcc-ar)
174 if (GCC_AR)
175 set(CMAKE_AR ${GCC_AR})
176 endif()
177 find_program(GCC_RANLIB gcc-ranlib)
178 if (GCC_RANLIB)
179 set(CMAKE_RANLIB ${GCC_RANLIB})
180 endif()
181 elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
182 include(llvm-toolchain)
183 endif()
184 endif()
185
186 # Coverage build type
187 set(BENCHMARK_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_DEBUG}"
188 CACHE STRING "Flags used by the C++ compiler during coverage builds."
189 FORCE)
190 set(BENCHMARK_EXE_LINKER_FLAGS_COVERAGE "${CMAKE_EXE_LINKER_FLAGS_DEBUG}"
191 CACHE STRING "Flags used for linking binaries during coverage builds."
192 FORCE)
193 set(BENCHMARK_SHARED_LINKER_FLAGS_COVERAGE "${CMAKE_SHARED_LINKER_FLAGS_DEBUG}"
194 CACHE STRING "Flags used by the shared libraries linker during coverage builds."
195 FORCE)
196 mark_as_advanced(
197 BENCHMARK_CXX_FLAGS_COVERAGE
198 BENCHMARK_EXE_LINKER_FLAGS_COVERAGE
199 BENCHMARK_SHARED_LINKER_FLAGS_COVERAGE)
200 set(CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}" CACHE STRING
201 "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel Coverage.")
202 add_cxx_compiler_flag(--coverage COVERAGE)
203 endif()
204
205 if (BENCHMARK_USE_LIBCXX)
206 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
207 add_cxx_compiler_flag(-stdlib=libc++)
208 elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" OR
209 "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
210 add_cxx_compiler_flag(-nostdinc++)
211 message("libc++ header path must be manually specified using CMAKE_CXX_FLAGS")
212 # Adding -nodefaultlibs directly to CMAKE__LINKER_FLAGS will break
213 # configuration checks such as 'find_package(Threads)'
214 list(APPEND BENCHMARK_CXX_LINKER_FLAGS -nodefaultlibs)
215 # -lc++ cannot be added directly to CMAKE__LINKER_FLAGS because
216 # linker flags appear before all linker inputs and -lc++ must appear after.
217 list(APPEND BENCHMARK_CXX_LIBRARIES c++)
218 else()
219 message(FATAL_ERROR "-DBENCHMARK_USE_LIBCXX:BOOL=ON is not supported for compiler")
220 endif()
221 endif(BENCHMARK_USE_LIBCXX)
222
223 # C++ feature checks
224 # Determine the correct regular expression engine to use
225 cxx_feature_check(STD_REGEX)
226 cxx_feature_check(GNU_POSIX_REGEX)
227 cxx_feature_check(POSIX_REGEX)
228 if(NOT HAVE_STD_REGEX AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX)
229 message(FATAL_ERROR "Failed to determine the source files for the regular expression backend")
230 endif()
231 if (NOT BENCHMARK_ENABLE_EXCEPTIONS AND HAVE_STD_REGEX
232 AND NOT HAVE_GNU_POSIX_REGEX AND NOT HAVE_POSIX_REGEX)
233 message(WARNING "Using std::regex with exceptions disabled is not fully supported")
234 endif()
235 cxx_feature_check(STEADY_CLOCK)
236 # Ensure we have pthreads
237 find_package(Threads REQUIRED)
238
239 # Set up directories
240 include_directories(${PROJECT_SOURCE_DIR}/include)
241
242 # Build the targets
243 add_subdirectory(src)
244
245 if (BENCHMARK_ENABLE_TESTING)
246 enable_testing()
247 if (BENCHMARK_ENABLE_GTEST_TESTS)
248 include(HandleGTest)
249 endif()
250 add_subdirectory(test)
251 endif()
0 # How to contribute #
1
2 We'd love to accept your patches and contributions to this project. There are
3 a just a few small guidelines you need to follow.
4
5
6 ## Contributor License Agreement ##
7
8 Contributions to any Google project must be accompanied by a Contributor
9 License Agreement. This is not a copyright **assignment**, it simply gives
10 Google permission to use and redistribute your contributions as part of the
11 project.
12
13 * If you are an individual writing original source code and you're sure you
14 own the intellectual property, then you'll need to sign an [individual
15 CLA][].
16
17 * If you work for a company that wants to allow you to contribute your work,
18 then you'll need to sign a [corporate CLA][].
19
20 You generally only need to submit a CLA once, so if you've already submitted
21 one (even if it was for a different project), you probably don't need to do it
22 again.
23
24 [individual CLA]: https://developers.google.com/open-source/cla/individual
25 [corporate CLA]: https://developers.google.com/open-source/cla/corporate
26
27 Once your CLA is submitted (or if you already submitted one for
28 another Google project), make a commit adding yourself to the
29 [AUTHORS][] and [CONTRIBUTORS][] files. This commit can be part
30 of your first [pull request][].
31
32 [AUTHORS]: AUTHORS
33 [CONTRIBUTORS]: CONTRIBUTORS
34
35
36 ## Submitting a patch ##
37
38 1. It's generally best to start by opening a new issue describing the bug or
39 feature you're intending to fix. Even if you think it's relatively minor,
40 it's helpful to know what people are working on. Mention in the initial
41 issue that you are planning to work on that bug or feature so that it can
42 be assigned to you.
43
44 1. Follow the normal process of [forking][] the project, and setup a new
45 branch to work in. It's important that each group of changes be done in
46 separate branches in order to ensure that a pull request only includes the
47 commits related to that bug or feature.
48
49 1. Do your best to have [well-formed commit messages][] for each change.
50 This provides consistency throughout the project, and ensures that commit
51 messages are able to be formatted properly by various git tools.
52
53 1. Finally, push the commits to your fork and submit a [pull request][].
54
55 [forking]: https://help.github.com/articles/fork-a-repo
56 [well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html
57 [pull request]: https://help.github.com/articles/creating-a-pull-request
0 # People who have agreed to one of the CLAs and can contribute patches.
1 # The AUTHORS file lists the copyright holders; this file
2 # lists people. For example, Google employees are listed here
3 # but not in AUTHORS, because Google holds the copyright.
4 #
5 # Names should be added to this file only after verifying that
6 # the individual or the individual's organization has agreed to
7 # the appropriate Contributor License Agreement, found here:
8 #
9 # https://developers.google.com/open-source/cla/individual
10 # https://developers.google.com/open-source/cla/corporate
11 #
12 # The agreement for individuals can be filled out on the web.
13 #
14 # When adding J Random Contributor's name to this file,
15 # either J's name or J's organization's name should be
16 # added to the AUTHORS file, depending on whether the
17 # individual or corporate CLA was used.
18 #
19 # Names should be added to this file as:
20 # Name
21 #
22 # Please keep the list sorted.
23
24 Albert Pretorius
25 Arne Beer
26 Billy Robert O'Neal III
27 Chris Kennelly
28 Christopher Seymour
29 David Coeurjolly
30 Deniz Evrenci
31 Dominic Hamon
32 Dominik Czarnota
33 Eric Fiselier
34 Eugene Zhuk
35 Evgeny Safronov
36 Felix Homann
37 Ismael Jimenez Martinez
38 Jern-Kuan Leong
39 JianXiong Zhou
40 Joao Paulo Magalhaes
41 John Millikin
42 Jussi Knuuttila
43 Kai Wolf
44 Kishan Kumar
45 Kaito Udagawa
46 Lei Xu
47 Matt Clarkson
48 Maxim Vafin
49 Nick Hutchinson
50 Oleksandr Sochka
51 Pascal Leroy
52 Paul Redmond
53 Pierre Phaneuf
54 Radoslav Yovchev
55 Raul Marin
56 Ray Glover
57 Robert Guo
58 Roman Lebedev
59 Shuo Chen
60 Tobias Ulvgård
61 Tom Madams
62 Yixuan Qiu
63 Yusuke Suzuki
64 Zbigniew Skowron
0
1 Apache License
2 Version 2.0, January 2004
3 http://www.apache.org/licenses/
4
5 TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
7 1. Definitions.
8
9 "License" shall mean the terms and conditions for use, reproduction,
10 and distribution as defined by Sections 1 through 9 of this document.
11
12 "Licensor" shall mean the copyright owner or entity authorized by
13 the copyright owner that is granting the License.
14
15 "Legal Entity" shall mean the union of the acting entity and all
16 other entities that control, are controlled by, or are under common
17 control with that entity. For the purposes of this definition,
18 "control" means (i) the power, direct or indirect, to cause the
19 direction or management of such entity, whether by contract or
20 otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 outstanding shares, or (iii) beneficial ownership of such entity.
22
23 "You" (or "Your") shall mean an individual or Legal Entity
24 exercising permissions granted by this License.
25
26 "Source" form shall mean the preferred form for making modifications,
27 including but not limited to software source code, documentation
28 source, and configuration files.
29
30 "Object" form shall mean any form resulting from mechanical
31 transformation or translation of a Source form, including but
32 not limited to compiled object code, generated documentation,
33 and conversions to other media types.
34
35 "Work" shall mean the work of authorship, whether in Source or
36 Object form, made available under the License, as indicated by a
37 copyright notice that is included in or attached to the work
38 (an example is provided in the Appendix below).
39
40 "Derivative Works" shall mean any work, whether in Source or Object
41 form, that is based on (or derived from) the Work and for which the
42 editorial revisions, annotations, elaborations, or other modifications
43 represent, as a whole, an original work of authorship. For the purposes
44 of this License, Derivative Works shall not include works that remain
45 separable from, or merely link (or bind by name) to the interfaces of,
46 the Work and Derivative Works thereof.
47
48 "Contribution" shall mean any work of authorship, including
49 the original version of the Work and any modifications or additions
50 to that Work or Derivative Works thereof, that is intentionally
51 submitted to Licensor for inclusion in the Work by the copyright owner
52 or by an individual or Legal Entity authorized to submit on behalf of
53 the copyright owner. For the purposes of this definition, "submitted"
54 means any form of electronic, verbal, or written communication sent
55 to the Licensor or its representatives, including but not limited to
56 communication on electronic mailing lists, source code control systems,
57 and issue tracking systems that are managed by, or on behalf of, the
58 Licensor for the purpose of discussing and improving the Work, but
59 excluding communication that is conspicuously marked or otherwise
60 designated in writing by the copyright owner as "Not a Contribution."
61
62 "Contributor" shall mean Licensor and any individual or Legal Entity
63 on behalf of whom a Contribution has been received by Licensor and
64 subsequently incorporated within the Work.
65
66 2. Grant of Copyright License. Subject to the terms and conditions of
67 this License, each Contributor hereby grants to You a perpetual,
68 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 copyright license to reproduce, prepare Derivative Works of,
70 publicly display, publicly perform, sublicense, and distribute the
71 Work and such Derivative Works in Source or Object form.
72
73 3. Grant of Patent License. Subject to the terms and conditions of
74 this License, each Contributor hereby grants to You a perpetual,
75 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 (except as stated in this section) patent license to make, have made,
77 use, offer to sell, sell, import, and otherwise transfer the Work,
78 where such license applies only to those patent claims licensable
79 by such Contributor that are necessarily infringed by their
80 Contribution(s) alone or by combination of their Contribution(s)
81 with the Work to which such Contribution(s) was submitted. If You
82 institute patent litigation against any entity (including a
83 cross-claim or counterclaim in a lawsuit) alleging that the Work
84 or a Contribution incorporated within the Work constitutes direct
85 or contributory patent infringement, then any patent licenses
86 granted to You under this License for that Work shall terminate
87 as of the date such litigation is filed.
88
89 4. Redistribution. You may reproduce and distribute copies of the
90 Work or Derivative Works thereof in any medium, with or without
91 modifications, and in Source or Object form, provided that You
92 meet the following conditions:
93
94 (a) You must give any other recipients of the Work or
95 Derivative Works a copy of this License; and
96
97 (b) You must cause any modified files to carry prominent notices
98 stating that You changed the files; and
99
100 (c) You must retain, in the Source form of any Derivative Works
101 that You distribute, all copyright, patent, trademark, and
102 attribution notices from the Source form of the Work,
103 excluding those notices that do not pertain to any part of
104 the Derivative Works; and
105
106 (d) If the Work includes a "NOTICE" text file as part of its
107 distribution, then any Derivative Works that You distribute must
108 include a readable copy of the attribution notices contained
109 within such NOTICE file, excluding those notices that do not
110 pertain to any part of the Derivative Works, in at least one
111 of the following places: within a NOTICE text file distributed
112 as part of the Derivative Works; within the Source form or
113 documentation, if provided along with the Derivative Works; or,
114 within a display generated by the Derivative Works, if and
115 wherever such third-party notices normally appear. The contents
116 of the NOTICE file are for informational purposes only and
117 do not modify the License. You may add Your own attribution
118 notices within Derivative Works that You distribute, alongside
119 or as an addendum to the NOTICE text from the Work, provided
120 that such additional attribution notices cannot be construed
121 as modifying the License.
122
123 You may add Your own copyright statement to Your modifications and
124 may provide additional or different license terms and conditions
125 for use, reproduction, or distribution of Your modifications, or
126 for any such Derivative Works as a whole, provided Your use,
127 reproduction, and distribution of the Work otherwise complies with
128 the conditions stated in this License.
129
130 5. Submission of Contributions. Unless You explicitly state otherwise,
131 any Contribution intentionally submitted for inclusion in the Work
132 by You to the Licensor shall be under the terms and conditions of
133 this License, without any additional terms or conditions.
134 Notwithstanding the above, nothing herein shall supersede or modify
135 the terms of any separate license agreement you may have executed
136 with Licensor regarding such Contributions.
137
138 6. Trademarks. This License does not grant permission to use the trade
139 names, trademarks, service marks, or product names of the Licensor,
140 except as required for reasonable and customary use in describing the
141 origin of the Work and reproducing the content of the NOTICE file.
142
143 7. Disclaimer of Warranty. Unless required by applicable law or
144 agreed to in writing, Licensor provides the Work (and each
145 Contributor provides its Contributions) on an "AS IS" BASIS,
146 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 implied, including, without limitation, any warranties or conditions
148 of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 PARTICULAR PURPOSE. You are solely responsible for determining the
150 appropriateness of using or redistributing the Work and assume any
151 risks associated with Your exercise of permissions under this License.
152
153 8. Limitation of Liability. In no event and under no legal theory,
154 whether in tort (including negligence), contract, or otherwise,
155 unless required by applicable law (such as deliberate and grossly
156 negligent acts) or agreed to in writing, shall any Contributor be
157 liable to You for damages, including any direct, indirect, special,
158 incidental, or consequential damages of any character arising as a
159 result of this License or out of the use or inability to use the
160 Work (including but not limited to damages for loss of goodwill,
161 work stoppage, computer failure or malfunction, or any and all
162 other commercial damages or losses), even if such Contributor
163 has been advised of the possibility of such damages.
164
165 9. Accepting Warranty or Additional Liability. While redistributing
166 the Work or Derivative Works thereof, You may choose to offer,
167 and charge a fee for, acceptance of support, warranty, indemnity,
168 or other liability obligations and/or rights consistent with this
169 License. However, in accepting such obligations, You may act only
170 on Your own behalf and on Your sole responsibility, not on behalf
171 of any other Contributor, and only if You agree to indemnify,
172 defend, and hold each Contributor harmless for any liability
173 incurred by, or claims asserted against, such Contributor by reason
174 of your accepting any such warranty or additional liability.
175
176 END OF TERMS AND CONDITIONS
177
178 APPENDIX: How to apply the Apache License to your work.
179
180 To apply the Apache License to your work, attach the following
181 boilerplate notice, with the fields enclosed by brackets "[]"
182 replaced with your own identifying information. (Don't include
183 the brackets!) The text should be enclosed in the appropriate
184 comment syntax for the file format. We also recommend that a
185 file or class name and description of purpose be included on the
186 same "printed page" as the copyright notice for easier
187 identification within third-party archives.
188
189 Copyright [yyyy] [name of copyright owner]
190
191 Licensed under the Apache License, Version 2.0 (the "License");
192 you may not use this file except in compliance with the License.
193 You may obtain a copy of the License at
194
195 http://www.apache.org/licenses/LICENSE-2.0
196
197 Unless required by applicable law or agreed to in writing, software
198 distributed under the License is distributed on an "AS IS" BASIS,
199 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 See the License for the specific language governing permissions and
201 limitations under the License.
0 LLVM notes
1 ----------
2
3 This directory contains the Google Benchmark source code. This directory is
4 under a different license than LLVM.
5
6 Changes:
7
8 * Bazel BUILD files are removed from the library
0 # benchmark
1 [![Build Status](https://travis-ci.org/google/benchmark.svg?branch=master)](https://travis-ci.org/google/benchmark)
2 [![Build status](https://ci.appveyor.com/api/projects/status/u0qsyp7t1tk7cpxs/branch/master?svg=true)](https://ci.appveyor.com/project/google/benchmark/branch/master)
3 [![Coverage Status](https://coveralls.io/repos/google/benchmark/badge.svg)](https://coveralls.io/r/google/benchmark)
4 [![slackin](https://slackin-iqtfqnpzxd.now.sh/badge.svg)](https://slackin-iqtfqnpzxd.now.sh/)
5
6 A library to support the benchmarking of functions, similar to unit-tests.
7
8 Discussion group: https://groups.google.com/d/forum/benchmark-discuss
9
10 IRC channel: https://freenode.net #googlebenchmark
11
12 [Known issues and common problems](#known-issues)
13
14 [Additional Tooling Documentation](docs/tools.md)
15
16 [Assembly Testing Documentation](docs/AssemblyTests.md)
17
18
19 ## Building
20
21 The basic steps for configuring and building the library look like this:
22
23 ```bash
24 $ git clone https://github.com/google/benchmark.git
25 # Benchmark requires Google Test as a dependency. Add the source tree as a subdirectory.
26 $ git clone https://github.com/google/googletest.git benchmark/googletest
27 $ mkdir build && cd build
28 $ cmake -G [options] ../benchmark
29 # Assuming a makefile generator was used
30 $ make
31 ```
32
33 Note that Google Benchmark requires Google Test to build and run the tests. This
34 dependency can be provided two ways:
35
36 * Checkout the Google Test sources into `benchmark/googletest` as above.
37 * Otherwise, if `-DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON` is specified during
38 configuration, the library will automatically download and build any required
39 dependencies.
40
41 If you do not wish to build and run the tests, add `-DBENCHMARK_ENABLE_GTEST_TESTS=OFF`
42 to `CMAKE_ARGS`.
43
44
45 ## Installation Guide
46
47 For Ubuntu and Debian Based System
48
49 First make sure you have git and cmake installed (If not please install it)
50
51 ```
52 sudo apt-get install git
53 sudo apt-get install cmake
54 ```
55
56 Now, let's clone the repository and build it
57
58 ```
59 git clone https://github.com/google/benchmark.git
60 cd benchmark
61 git clone https://github.com/google/googletest.git
62 mkdir build
63 cd build
64 cmake .. -DCMAKE_BUILD_TYPE=RELEASE
65 make
66 ```
67
68 We need to install the library globally now
69
70 ```
71 sudo make install
72 ```
73
74 Now you have google/benchmark installed in your machine
75 Note: Don't forget to link to pthread library while building
76
77 ## Stable and Experimental Library Versions
78
79 The main branch contains the latest stable version of the benchmarking library;
80 the API of which can be considered largely stable, with source breaking changes
81 being made only upon the release of a new major version.
82
83 Newer, experimental, features are implemented and tested on the
84 [`v2` branch](https://github.com/google/benchmark/tree/v2). Users who wish
85 to use, test, and provide feedback on the new features are encouraged to try
86 this branch. However, this branch provides no stability guarantees and reserves
87 the right to change and break the API at any time.
88
89 ##Prerequisite knowledge
90
91 Before attempting to understand this framework one should ideally have some familiarity with the structure and format of the Google Test framework, upon which it is based. Documentation for Google Test, including a "Getting Started" (primer) guide, is available here:
92 https://github.com/google/googletest/blob/master/googletest/docs/Documentation.md
93
94
95 ## Example usage
96 ### Basic usage
97 Define a function that executes the code to be measured.
98
99 ```c++
100 #include
101
102 static void BM_StringCreation(benchmark::State& state) {
103 for (auto _ : state)
104 std::string empty_string;
105 }
106 // Register the function as a benchmark
107 BENCHMARK(BM_StringCreation);
108
109 // Define another benchmark
110 static void BM_StringCopy(benchmark::State& state) {
111 std::string x = "hello";
112 for (auto _ : state)
113 std::string copy(x);
114 }
115 BENCHMARK(BM_StringCopy);
116
117 BENCHMARK_MAIN();
118 ```
119
120 Don't forget to inform your linker to add benchmark library e.g. through
121 `-lbenchmark` compilation flag. Alternatively, you may leave out the
122 `BENCHMARK_MAIN();` at the end of the source file and link against
123 `-lbenchmark_main` to get the same default behavior.
124
125 The benchmark library will reporting the timing for the code within the `for(...)` loop.
126
127 ### Passing arguments
128 Sometimes a family of benchmarks can be implemented with just one routine that
129 takes an extra argument to specify which one of the family of benchmarks to
130 run. For example, the following code defines a family of benchmarks for
131 measuring the speed of `memcpy()` calls of different lengths:
132
133 ```c++
134 static void BM_memcpy(benchmark::State& state) {
135 char* src = new char[state.range(0)];
136 char* dst = new char[state.range(0)];
137 memset(src, 'x', state.range(0));
138 for (auto _ : state)
139 memcpy(dst, src, state.range(0));
140 state.SetBytesProcessed(int64_t(state.iterations()) *
141 int64_t(state.range(0)));
142 delete[] src;
143 delete[] dst;
144 }
145 BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
146 ```
147
148 The preceding code is quite repetitive, and can be replaced with the following
149 short-hand. The following invocation will pick a few appropriate arguments in
150 the specified range and will generate a benchmark for each such argument.
151
152 ```c++
153 BENCHMARK(BM_memcpy)->Range(8, 8<<10);
154 ```
155
156 By default the arguments in the range are generated in multiples of eight and
157 the command above selects [ 8, 64, 512, 4k, 8k ]. In the following code the
158 range multiplier is changed to multiples of two.
159
160 ```c++
161 BENCHMARK(BM_memcpy)->RangeMultiplier(2)->Range(8, 8<<10);
162 ```
163 Now arguments generated are [ 8, 16, 32, 64, 128, 256, 512, 1024, 2k, 4k, 8k ].
164
165 You might have a benchmark that depends on two or more inputs. For example, the
166 following code defines a family of benchmarks for measuring the speed of set
167 insertion.
168
169 ```c++
170 static void BM_SetInsert(benchmark::State& state) {
171 std::set data;
172 for (auto _ : state) {
173 state.PauseTiming();
174 data = ConstructRandomSet(state.range(0));
175 state.ResumeTiming();
176 for (int j = 0; j < state.range(1); ++j)
177 data.insert(RandomNumber());
178 }
179 }
180 BENCHMARK(BM_SetInsert)
181 ->Args({1<<10, 128})
182 ->Args({2<<10, 128})
183 ->Args({4<<10, 128})
184 ->Args({8<<10, 128})
185 ->Args({1<<10, 512})
186 ->Args({2<<10, 512})
187 ->Args({4<<10, 512})
188 ->Args({8<<10, 512});
189 ```
190
191 The preceding code is quite repetitive, and can be replaced with the following
192 short-hand. The following macro will pick a few appropriate arguments in the
193 product of the two specified ranges and will generate a benchmark for each such
194 pair.
195
196 ```c++
197 BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
198 ```
199
200 For more complex patterns of inputs, passing a custom function to `Apply` allows
201 programmatic specification of an arbitrary set of arguments on which to run the
202 benchmark. The following example enumerates a dense range on one parameter,
203 and a sparse range on the second.
204
205 ```c++
206 static void CustomArguments(benchmark::internal::Benchmark* b) {
207 for (int i = 0; i <= 10; ++i)
208 for (int j = 32; j <= 1024*1024; j *= 8)
209 b->Args({i, j});
210 }
211 BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
212 ```
213
214 ### Calculate asymptotic complexity (Big O)
215 Asymptotic complexity might be calculated for a family of benchmarks. The
216 following code will calculate the coefficient for the high-order term in the
217 running time and the normalized root-mean square error of string comparison.
218
219 ```c++
220 static void BM_StringCompare(benchmark::State& state) {
221 std::string s1(state.range(0), '-');
222 std::string s2(state.range(0), '-');
223 for (auto _ : state) {
224 benchmark::DoNotOptimize(s1.compare(s2));
225 }
226 state.SetComplexityN(state.range(0));
227 }
228 BENCHMARK(BM_StringCompare)
229 ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity(benchmark::oN);
230 ```
231
232 As shown in the following invocation, asymptotic complexity might also be
233 calculated automatically.
234
235 ```c++
236 BENCHMARK(BM_StringCompare)
237 ->RangeMultiplier(2)->Range(1<<10, 1<<18)->Complexity();
238 ```
239
240 The following code will specify asymptotic complexity with a lambda function,
241 that might be used to customize high-order term calculation.
242
243 ```c++
244 BENCHMARK(BM_StringCompare)->RangeMultiplier(2)
245 ->Range(1<<10, 1<<18)->Complexity([](int n)->double{return n; });
246 ```
247
248 ### Templated benchmarks
249 Templated benchmarks work the same way: This example produces and consumes
250 messages of size `sizeof(v)` `range_x` times. It also outputs throughput in the
251 absence of multiprogramming.
252
253 ```c++
254 template int BM_Sequential(benchmark::State& state) {
255 Q q;
256 typename Q::value_type v;
257 for (auto _ : state) {
258 for (int i = state.range(0); i--; )
259 q.push(v);
260 for (int e = state.range(0); e--; )
261 q.Wait(&v);
262 }
263 // actually messages, not bytes:
264 state.SetBytesProcessed(
265 static_cast(state.iterations())*state.range(0));
266 }
267 BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10);
268 ```
269
270 Three macros are provided for adding benchmark templates.
271
272 ```c++
273 #ifdef BENCHMARK_HAS_CXX11
274 #define BENCHMARK_TEMPLATE(func, ...) // Takes any number of parameters.
275 #else // C++ < C++11
276 #define BENCHMARK_TEMPLATE(func, arg1)
277 #endif
278 #define BENCHMARK_TEMPLATE1(func, arg1)
279 #define BENCHMARK_TEMPLATE2(func, arg1, arg2)
280 ```
281
282 ### A Faster KeepRunning loop
283
284 In C++11 mode, a ranged-based for loop should be used in preference to
285 the `KeepRunning` loop for running the benchmarks. For example:
286
287 ```c++
288 static void BM_Fast(benchmark::State &state) {
289 for (auto _ : state) {
290 FastOperation();
291 }
292 }
293 BENCHMARK(BM_Fast);
294 ```
295
296 The reason the ranged-for loop is faster than using `KeepRunning`, is
297 because `KeepRunning` requires a memory load and store of the iteration count
298 ever iteration, whereas the ranged-for variant is able to keep the iteration count
299 in a register.
300
301 For example, an empty inner loop of using the ranged-based for method looks like:
302
303 ```asm
304 # Loop Init
305 mov rbx, qword ptr [r14 + 104]
306 call benchmark::State::StartKeepRunning()
307 test rbx, rbx
308 je .LoopEnd
309 .LoopHeader: # =>This Inner Loop Header: Depth=1
310 add rbx, -1
311 jne .LoopHeader
312 .LoopEnd:
313 ```
314
315 Compared to an empty `KeepRunning` loop, which looks like:
316
317 ```asm
318 .LoopHeader: # in Loop: Header=BB0_3 Depth=1
319 cmp byte ptr [rbx], 1
320 jne .LoopInit
321 .LoopBody: # =>This Inner Loop Header: Depth=1
322 mov rax, qword ptr [rbx + 8]
323 lea rcx, [rax + 1]
324 mov qword ptr [rbx + 8], rcx
325 cmp rax, qword ptr [rbx + 104]
326 jb .LoopHeader
327 jmp .LoopEnd
328 .LoopInit:
329 mov rdi, rbx
330 call benchmark::State::StartKeepRunning()
331 jmp .LoopBody
332 .LoopEnd:
333 ```
334
335 Unless C++03 compatibility is required, the ranged-for variant of writing
336 the benchmark loop should be preferred.
337
338 ## Passing arbitrary arguments to a benchmark
339 In C++11 it is possible to define a benchmark that takes an arbitrary number
340 of extra arguments. The `BENCHMARK_CAPTURE(func, test_case_name, ...args)`
341 macro creates a benchmark that invokes `func` with the `benchmark::State` as
342 the first argument followed by the specified `args...`.
343 The `test_case_name` is appended to the name of the benchmark and
344 should describe the values passed.
345
346 ```c++
347 template
348 void BM_takes_args(benchmark::State& state, ExtraArgs&&... extra_args) {
349 [...]
350 }
351 // Registers a benchmark named "BM_takes_args/int_string_test" that passes
352 // the specified values to `extra_args`.
353 BENCHMARK_CAPTURE(BM_takes_args, int_string_test, 42, std::string("abc"));
354 ```
355 Note that elements of `...args` may refer to global variables. Users should
356 avoid modifying global state inside of a benchmark.
357
358 ## Using RegisterBenchmark(name, fn, args...)
359
360 The `RegisterBenchmark(name, func, args...)` function provides an alternative
361 way to create and register benchmarks.
362 `RegisterBenchmark(name, func, args...)` creates, registers, and returns a
363 pointer to a new benchmark with the specified `name` that invokes
364 `func(st, args...)` where `st` is a `benchmark::State` object.
365
366 Unlike the `BENCHMARK` registration macros, which can only be used at the global
367 scope, the `RegisterBenchmark` can be called anywhere. This allows for
368 benchmark tests to be registered programmatically.
369
370 Additionally `RegisterBenchmark` allows any callable object to be registered
371 as a benchmark. Including capturing lambdas and function objects.
372
373 For Example:
374 ```c++
375 auto BM_test = [](benchmark::State& st, auto Inputs) { /* ... */ };
376
377 int main(int argc, char** argv) {
378 for (auto& test_input : { /* ... */ })
379 benchmark::RegisterBenchmark(test_input.name(), BM_test, test_input);
380 benchmark::Initialize(&argc, argv);
381 benchmark::RunSpecifiedBenchmarks();
382 }
383 ```
384
385 ### Multithreaded benchmarks
386 In a multithreaded test (benchmark invoked by multiple threads simultaneously),
387 it is guaranteed that none of the threads will start until all have reached
388 the start of the benchmark loop, and all will have finished before any thread
389 exits the benchmark loop. (This behavior is also provided by the `KeepRunning()`
390 API) As such, any global setup or teardown can be wrapped in a check against the thread
391 index:
392
393 ```c++
394 static void BM_MultiThreaded(benchmark::State& state) {
395 if (state.thread_index == 0) {
396 // Setup code here.
397 }
398 for (auto _ : state) {
399 // Run the test as normal.
400 }
401 if (state.thread_index == 0) {
402 // Teardown code here.
403 }
404 }
405 BENCHMARK(BM_MultiThreaded)->Threads(2);
406 ```
407
408 If the benchmarked code itself uses threads and you want to compare it to
409 single-threaded code, you may want to use real-time ("wallclock") measurements
410 for latency comparisons:
411
412 ```c++
413 BENCHMARK(BM_test)->Range(8, 8<<10)->UseRealTime();
414 ```
415
416 Without `UseRealTime`, CPU time is used by default.
417
418
419 ## Manual timing
420 For benchmarking something for which neither CPU time nor real-time are
421 correct or accurate enough, completely manual timing is supported using
422 the `UseManualTime` function.
423
424 When `UseManualTime` is used, the benchmarked code must call
425 `SetIterationTime` once per iteration of the benchmark loop to
426 report the manually measured time.
427
428 An example use case for this is benchmarking GPU execution (e.g. OpenCL
429 or CUDA kernels, OpenGL or Vulkan or Direct3D draw calls), which cannot
430 be accurately measured using CPU time or real-time. Instead, they can be
431 measured accurately using a dedicated API, and these measurement results
432 can be reported back with `SetIterationTime`.
433
434 ```c++
435 static void BM_ManualTiming(benchmark::State& state) {
436 int microseconds = state.range(0);
437 std::chrono::duration sleep_duration {
438 static_cast(microseconds)
439 };
440
441 for (auto _ : state) {
442 auto start = std::chrono::high_resolution_clock::now();
443 // Simulate some useful workload with a sleep
444 std::this_thread::sleep_for(sleep_duration);
445 auto end = std::chrono::high_resolution_clock::now();
446
447 auto elapsed_seconds =
448 std::chrono::duration_cast>(
449 end - start);
450
451 state.SetIterationTime(elapsed_seconds.count());
452 }
453 }
454 BENCHMARK(BM_ManualTiming)->Range(1, 1<<17)->UseManualTime();
455 ```
456
457 ### Preventing optimisation
458 To prevent a value or expression from being optimized away by the compiler
459 the `benchmark::DoNotOptimize(...)` and `benchmark::ClobberMemory()`
460 functions can be used.
461
462 ```c++
463 static void BM_test(benchmark::State& state) {
464 for (auto _ : state) {
465 int x = 0;
466 for (int i=0; i < 64; ++i) {
467 benchmark::DoNotOptimize(x += i);
468 }
469 }
470 }
471 ```
472
473 `DoNotOptimize()` forces the *result* of `` to be stored in either
474 memory or a register. For GNU based compilers it acts as read/write barrier
475 for global memory. More specifically it forces the compiler to flush pending
476 writes to memory and reload any other values as necessary.
477
478 Note that `DoNotOptimize()` does not prevent optimizations on ``
479 in any way. `` may even be removed entirely when the result is already
480 known. For example:
481
482 ```c++
483 /* Example 1: `` is removed entirely. */
484 int foo(int x) { return x + 42; }
485 while (...) DoNotOptimize(foo(0)); // Optimized to DoNotOptimize(42);
486
487 /* Example 2: Result of '' is only reused */
488 int bar(int) __attribute__((const));
489 while (...) DoNotOptimize(bar(0)); // Optimized to:
490 // int __result__ = bar(0);
491 // while (...) DoNotOptimize(__result__);
492 ```
493
494 The second tool for preventing optimizations is `ClobberMemory()`. In essence
495 `ClobberMemory()` forces the compiler to perform all pending writes to global
496 memory. Memory managed by block scope objects must be "escaped" using
497 `DoNotOptimize(...)` before it can be clobbered. In the below example
498 `ClobberMemory()` prevents the call to `v.push_back(42)` from being optimized
499 away.
500
501 ```c++
502 static void BM_vector_push_back(benchmark::State& state) {
503 for (auto _ : state) {
504 std::vector v;
505 v.reserve(1);
506 benchmark::DoNotOptimize(v.data()); // Allow v.data() to be clobbered.
507 v.push_back(42);
508 benchmark::ClobberMemory(); // Force 42 to be written to memory.
509 }
510 }
511 ```
512
513 Note that `ClobberMemory()` is only available for GNU or MSVC based compilers.
514
515 ### Set time unit manually
516 If a benchmark runs a few milliseconds it may be hard to visually compare the
517 measured times, since the output data is given in nanoseconds per default. In
518 order to manually set the time unit, you can specify it manually:
519
520 ```c++
521 BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
522 ```
523
524 ## Controlling number of iterations
525 In all cases, the number of iterations for which the benchmark is run is
526 governed by the amount of time the benchmark takes. Concretely, the number of
527 iterations is at least one, not more than 1e9, until CPU time is greater than
528 the minimum time, or the wallclock time is 5x minimum time. The minimum time is
529 set as a flag `--benchmark_min_time` or per benchmark by calling `MinTime` on
530 the registered benchmark object.
531
532 ## Reporting the mean, median and standard deviation by repeated benchmarks
533 By default each benchmark is run once and that single result is reported.
534 However benchmarks are often noisy and a single result may not be representative
535 of the overall behavior. For this reason it's possible to repeatedly rerun the
536 benchmark.
537
538 The number of runs of each benchmark is specified globally by the
539 `--benchmark_repetitions` flag or on a per benchmark basis by calling
540 `Repetitions` on the registered benchmark object. When a benchmark is run more
541 than once the mean, median and standard deviation of the runs will be reported.
542
543 Additionally the `--benchmark_report_aggregates_only={true|false}` flag or
544 `ReportAggregatesOnly(bool)` function can be used to change how repeated tests
545 are reported. By default the result of each repeated run is reported. When this
546 option is `true` only the mean, median and standard deviation of the runs is reported.
547 Calling `ReportAggregatesOnly(bool)` on a registered benchmark object overrides
548 the value of the flag for that benchmark.
549
550 ## User-defined statistics for repeated benchmarks
551 While having mean, median and standard deviation is nice, this may not be
552 enough for everyone. For example you may want to know what is the largest
553 observation, e.g. because you have some real-time constraints. This is easy.
554 The following code will specify a custom statistic to be calculated, defined
555 by a lambda function.
556
557 ```c++
558 void BM_spin_empty(benchmark::State& state) {
559 for (auto _ : state) {
560 for (int x = 0; x < state.range(0); ++x) {
561 benchmark::DoNotOptimize(x);
562 }
563 }
564 }
565
566 BENCHMARK(BM_spin_empty)
567 ->ComputeStatistics("max", [](const std::vector& v) -> double {
568 return *(std::max_element(std::begin(v), std::end(v)));
569 })
570 ->Arg(512);
571 ```
572
573 ## Fixtures
574 Fixture tests are created by
575 first defining a type that derives from `::benchmark::Fixture` and then
576 creating/registering the tests using the following macros:
577
578 * `BENCHMARK_F(ClassName, Method)`
579 * `BENCHMARK_DEFINE_F(ClassName, Method)`
580 * `BENCHMARK_REGISTER_F(ClassName, Method)`
581
582 For Example:
583
584 ```c++
585 class MyFixture : public benchmark::Fixture {};
586
587 BENCHMARK_F(MyFixture, FooTest)(benchmark::State& st) {
588 for (auto _ : st) {
589 ...
590 }
591 }
592
593 BENCHMARK_DEFINE_F(MyFixture, BarTest)(benchmark::State& st) {
594 for (auto _ : st) {
595 ...
596 }
597 }
598 /* BarTest is NOT registered */
599 BENCHMARK_REGISTER_F(MyFixture, BarTest)->Threads(2);
600 /* BarTest is now registered */
601 ```
602
603 ### Templated fixtures
604 Also you can create templated fixture by using the following macros:
605
606 * `BENCHMARK_TEMPLATE_F(ClassName, Method, ...)`
607 * `BENCHMARK_TEMPLATE_DEFINE_F(ClassName, Method, ...)`
608
609 For example:
610 ```c++
611 template
612 class MyFixture : public benchmark::Fixture {};
613
614 BENCHMARK_TEMPLATE_F(MyFixture, IntTest, int)(benchmark::State& st) {
615 for (auto _ : st) {
616 ...
617 }
618 }
619
620 BENCHMARK_TEMPLATE_DEFINE_F(MyFixture, DoubleTest, double)(benchmark::State& st) {
621 for (auto _ : st) {
622 ...
623 }
624 }
625
626 BENCHMARK_REGISTER_F(MyFixture, DoubleTest)->Threads(2);
627 ```
628
629 ## User-defined counters
630
631 You can add your own counters with user-defined names. The example below
632 will add columns "Foo", "Bar" and "Baz" in its output:
633
634 ```c++
635 static void UserCountersExample1(benchmark::State& state) {
636 double numFoos = 0, numBars = 0, numBazs = 0;
637 for (auto _ : state) {
638 // ... count Foo,Bar,Baz events
639 }
640 state.counters["Foo"] = numFoos;
641 state.counters["Bar"] = numBars;
642 state.counters["Baz"] = numBazs;
643 }
644 ```
645
646 The `state.counters` object is a `std::map` with `std::string` keys
647 and `Counter` values. The latter is a `double`-like class, via an implicit
648 conversion to `double&`. Thus you can use all of the standard arithmetic
649 assignment operators (`=,+=,-=,*=,/=`) to change the value of each counter.
650
651 In multithreaded benchmarks, each counter is set on the calling thread only.
652 When the benchmark finishes, the counters from each thread will be summed;
653 the resulting sum is the value which will be shown for the benchmark.
654
655 The `Counter` constructor accepts two parameters: the value as a `double`
656 and a bit flag which allows you to show counters as rates and/or as
657 per-thread averages:
658
659 ```c++
660 // sets a simple counter
661 state.counters["Foo"] = numFoos;
662
663 // Set the counter as a rate. It will be presented divided
664 // by the duration of the benchmark.
665 state.counters["FooRate"] = Counter(numFoos, benchmark::Counter::kIsRate);
666
667 // Set the counter as a thread-average quantity. It will
668 // be presented divided by the number of threads.
669 state.counters["FooAvg"] = Counter(numFoos, benchmark::Counter::kAvgThreads);
670
671 // There's also a combined flag:
672 state.counters["FooAvgRate"] = Counter(numFoos,benchmark::Counter::kAvgThreadsRate);
673 ```
674
675 When you're compiling in C++11 mode or later you can use `insert()` with
676 `std::initializer_list`:
677
678 ```c++
679 // With C++11, this can be done:
680 state.counters.insert({{"Foo", numFoos}, {"Bar", numBars}, {"Baz", numBazs}});
681 // ... instead of:
682 state.counters["Foo"] = numFoos;
683 state.counters["Bar"] = numBars;
684 state.counters["Baz"] = numBazs;
685 ```
686
687 ### Counter reporting
688
689 When using the console reporter, by default, user counters are are printed at
690 the end after the table, the same way as ``bytes_processed`` and
691 ``items_processed``. This is best for cases in which there are few counters,
692 or where there are only a couple of lines per benchmark. Here's an example of
693 the default output:
694
695 ```
696 ------------------------------------------------------------------------------
697 Benchmark Time CPU Iterations UserCounters...
698 ------------------------------------------------------------------------------
699 BM_UserCounter/threads:8 2248 ns 10277 ns 68808 Bar=16 Bat=40 Baz=24 Foo=8
700 BM_UserCounter/threads:1 9797 ns 9788 ns 71523 Bar=2 Bat=5 Baz=3 Foo=1024m
701 BM_UserCounter/threads:2 4924 ns 9842 ns 71036 Bar=4 Bat=10 Baz=6 Foo=2
702 BM_UserCounter/threads:4 2589 ns 10284 ns 68012 Bar=8 Bat=20 Baz=12 Foo=4
703 BM_UserCounter/threads:8 2212 ns 10287 ns 68040 Bar=16 Bat=40 Baz=24 Foo=8
704 BM_UserCounter/threads:16 1782 ns 10278 ns 68144 Bar=32 Bat=80 Baz=48 Foo=16
705 BM_UserCounter/threads:32 1291 ns 10296 ns 68256 Bar=64 Bat=160 Baz=96 Foo=32
706 BM_UserCounter/threads:4 2615 ns 10307 ns 68040 Bar=8 Bat=20 Baz=12 Foo=4
707 BM_Factorial 26 ns 26 ns 26608979 40320
708 BM_Factorial/real_time 26 ns 26 ns 26587936 40320
709 BM_CalculatePiRange/1 16 ns 16 ns 45704255 0
710 BM_CalculatePiRange/8 73 ns 73 ns 9520927 3.28374
711 BM_CalculatePiRange/64 609 ns 609 ns 1140647 3.15746
712 BM_CalculatePiRange/512 4900 ns 4901 ns 142696 3.14355
713 ```
714
715 If this doesn't suit you, you can print each counter as a table column by
716 passing the flag `--benchmark_counters_tabular=true` to the benchmark
717 application. This is best for cases in which there are a lot of counters, or
718 a lot of lines per individual benchmark. Note that this will trigger a
719 reprinting of the table header any time the counter set changes between
720 individual benchmarks. Here's an example of corresponding output when
721 `--benchmark_counters_tabular=true` is passed:
722
723 ```
724 ---------------------------------------------------------------------------------------
725 Benchmark Time CPU Iterations Bar Bat Baz Foo
726 ---------------------------------------------------------------------------------------
727 BM_UserCounter/threads:8 2198 ns 9953 ns 70688 16 40 24 8
728 BM_UserCounter/threads:1 9504 ns 9504 ns 73787 2 5 3 1
729 BM_UserCounter/threads:2 4775 ns 9550 ns 72606 4 10 6 2
730 BM_UserCounter/threads:4 2508 ns 9951 ns 70332 8 20 12 4
731 BM_UserCounter/threads:8 2055 ns 9933 ns 70344 16 40 24 8
732 BM_UserCounter/threads:16 1610 ns 9946 ns 70720 32 80 48 16
733 BM_UserCounter/threads:32 1192 ns 9948 ns 70496 64 160 96 32
734 BM_UserCounter/threads:4 2506 ns 9949 ns 70332 8 20 12 4
735 --------------------------------------------------------------
736 Benchmark Time CPU Iterations
737 --------------------------------------------------------------
738 BM_Factorial 26 ns 26 ns 26392245 40320
739 BM_Factorial/real_time 26 ns 26 ns 26494107 40320
740 BM_CalculatePiRange/1 15 ns 15 ns 45571597 0
741 BM_CalculatePiRange/8 74 ns 74 ns 9450212 3.28374
742 BM_CalculatePiRange/64 595 ns 595 ns 1173901 3.15746
743 BM_CalculatePiRange/512 4752 ns 4752 ns 147380 3.14355
744 BM_CalculatePiRange/4k 37970 ns 37972 ns 18453 3.14184
745 BM_CalculatePiRange/32k 303733 ns 303744 ns 2305 3.14162
746 BM_CalculatePiRange/256k 2434095 ns 2434186 ns 288 3.1416
747 BM_CalculatePiRange/1024k 9721140 ns 9721413 ns 71 3.14159
748 BM_CalculatePi/threads:8 2255 ns 9943 ns 70936
749 ```
750 Note above the additional header printed when the benchmark changes from
751 ``BM_UserCounter`` to ``BM_Factorial``. This is because ``BM_Factorial`` does
752 not have the same counter set as ``BM_UserCounter``.
753
754 ## Exiting Benchmarks in Error
755
756 When errors caused by external influences, such as file I/O and network
757 communication, occur within a benchmark the
758 `State::SkipWithError(const char* msg)` function can be used to skip that run
759 of benchmark and report the error. Note that only future iterations of the
760 `KeepRunning()` are skipped. For the ranged-for version of the benchmark loop
761 Users must explicitly exit the loop, otherwise all iterations will be performed.
762 Users may explicitly return to exit the benchmark immediately.
763
764 The `SkipWithError(...)` function may be used at any point within the benchmark,
765 including before and after the benchmark loop.
766
767 For example:
768
769 ```c++
770 static void BM_test(benchmark::State& state) {
771 auto resource = GetResource();
772 if (!resource.good()) {
773 state.SkipWithError("Resource is not good!");
774 // KeepRunning() loop will not be entered.
775 }
776 for (state.KeepRunning()) {
777 auto data = resource.read_data();
778 if (!resource.good()) {
779 state.SkipWithError("Failed to read data!");
780 break; // Needed to skip the rest of the iteration.
781 }
782 do_stuff(data);
783 }
784 }
785
786 static void BM_test_ranged_fo(benchmark::State & state) {
787 state.SkipWithError("test will not be entered");
788 for (auto _ : state) {
789 state.SkipWithError("Failed!");
790 break; // REQUIRED to prevent all further iterations.
791 }
792 }
793 ```
794
795 ## Running a subset of the benchmarks
796
797 The `--benchmark_filter=` option can be used to only run the benchmarks
798 which match the specified ``. For example:
799
800 ```bash
801 $ ./run_benchmarks.x --benchmark_filter=BM_memcpy/32
802 Run on (1 X 2300 MHz CPU )
803 2016-06-25 19:34:24
804 Benchmark Time CPU Iterations
805 ----------------------------------------------------
806 BM_memcpy/32 11 ns 11 ns 79545455
807 BM_memcpy/32k 2181 ns 2185 ns 324074
808 BM_memcpy/32 12 ns 12 ns 54687500
809 BM_memcpy/32k 1834 ns 1837 ns 357143
810 ```
811
812
813 ## Output Formats
814 The library supports multiple output formats. Use the
815 `--benchmark_format=` flag to set the format type. `console`
816 is the default format.
817
818 The Console format is intended to be a human readable format. By default
819 the format generates color output. Context is output on stderr and the
820 tabular data on stdout. Example tabular output looks like:
821 ```
822 Benchmark Time(ns) CPU(ns) Iterations
823 ----------------------------------------------------------------------
824 BM_SetInsert/1024/1 28928 29349 23853 133.097kB/s 33.2742k items/s
825 BM_SetInsert/1024/8 32065 32913 21375 949.487kB/s 237.372k items/s
826 BM_SetInsert/1024/10 33157 33648 21431 1.13369MB/s 290.225k items/s
827 ```
828
829 The JSON format outputs human readable json split into two top level attributes.
830 The `context` attribute contains information about the run in general, including
831 information about the CPU and the date.
832 The `benchmarks` attribute contains a list of every benchmark run. Example json
833 output looks like:
834 ```json
835 {
836 "context": {
837 "date": "2015/03/17-18:40:25",
838 "num_cpus": 40,
839 "mhz_per_cpu": 2801,
840 "cpu_scaling_enabled": false,
841 "build_type": "debug"
842 },
843 "benchmarks": [
844 {
845 "name": "BM_SetInsert/1024/1",
846 "iterations": 94877,
847 "real_time": 29275,
848 "cpu_time": 29836,
849 "bytes_per_second": 134066,
850 "items_per_second": 33516
851 },
852 {
853 "name": "BM_SetInsert/1024/8",
854 "iterations": 21609,
855 "real_time": 32317,
856 "cpu_time": 32429,
857 "bytes_per_second": 986770,
858 "items_per_second": 246693
859 },
860 {
861 "name": "BM_SetInsert/1024/10",
862 "iterations": 21393,
863 "real_time": 32724,
864 "cpu_time": 33355,
865 "bytes_per_second": 1199226,
866 "items_per_second": 299807
867 }
868 ]
869 }
870 ```
871
872 The CSV format outputs comma-separated values. The `context` is output on stderr
873 and the CSV itself on stdout. Example CSV output looks like:
874 ```
875 name,iterations,real_time,cpu_time,bytes_per_second,items_per_second,label
876 "BM_SetInsert/1024/1",65465,17890.7,8407.45,475768,118942,
877 "BM_SetInsert/1024/8",116606,18810.1,9766.64,3.27646e+06,819115,
878 "BM_SetInsert/1024/10",106365,17238.4,8421.53,4.74973e+06,1.18743e+06,
879 ```
880
881 ## Output Files
882 The library supports writing the output of the benchmark to a file specified
883 by `--benchmark_out=`. The format of the output can be specified
884 using `--benchmark_out_format={json|console|csv}`. Specifying
885 `--benchmark_out` does not suppress the console output.
886
887 ## Debug vs Release
888 By default, benchmark builds as a debug library. You will see a warning in the output when this is the case. To build it as a release library instead, use:
889
890 ```
891 cmake -DCMAKE_BUILD_TYPE=Release
892 ```
893
894 To enable link-time optimisation, use
895
896 ```
897 cmake -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true
898 ```
899
900 If you are using gcc, you might need to set `GCC_AR` and `GCC_RANLIB` cmake cache variables, if autodetection fails.
901 If you are using clang, you may need to set `LLVMAR_EXECUTABLE`, `LLVMNM_EXECUTABLE` and `LLVMRANLIB_EXECUTABLE` cmake cache variables.
902
903 ## Linking against the library
904
905 When the library is built using GCC it is necessary to link with `-pthread`,
906 due to how GCC implements `std::thread`.
907
908 For GCC 4.x failing to link to pthreads will lead to runtime exceptions, not linker errors.
909 See [issue #67](https://github.com/google/benchmark/issues/67) for more details.
910
911 ## Compiler Support
912
913 Google Benchmark uses C++11 when building the library. As such we require
914 a modern C++ toolchain, both compiler and standard library.
915
916 The following minimum versions are strongly recommended build the library:
917
918 * GCC 4.8
919 * Clang 3.4
920 * Visual Studio 2013
921 * Intel 2015 Update 1
922
923 Anything older *may* work.
924
925 Note: Using the library and its headers in C++03 is supported. C++11 is only
926 required to build the library.
927
928 ## Disable CPU frequency scaling
929 If you see this error:
930 ```
931 ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will incur extra overhead.
932 ```
933 you might want to disable the CPU frequency scaling while running the benchmark:
934 ```bash
935 sudo cpupower frequency-set --governor performance
936 ./mybench
937 sudo cpupower frequency-set --governor powersave
938 ```
939
940 # Known Issues
941
942 ### Windows with CMake
943
944 * Users must manually link `shlwapi.lib`. Failure to do so may result
945 in unresolved symbols.
946
947 ### Solaris
948
949 * Users must explicitly link with kstat library (-lkstat compilation flag).
0 workspace(name = "com_github_google_benchmark")
1
2 http_archive(
3 name = "com_google_googletest",
4 urls = ["https://github.com/google/googletest/archive/3f0cf6b62ad1eb50d8736538363d3580dd640c3e.zip"],
5 strip_prefix = "googletest-3f0cf6b62ad1eb50d8736538363d3580dd640c3e",
6 )
0 version: '{build}'
1
2 image: Visual Studio 2017
3
4 configuration:
5 - Debug
6 - Release
7
8 environment:
9 matrix:
10 - compiler: msvc-15-seh
11 generator: "Visual Studio 15 2017"
12
13 - compiler: msvc-15-seh
14 generator: "Visual Studio 15 2017 Win64"
15
16 - compiler: msvc-14-seh
17 generator: "Visual Studio 14 2015"
18
19 - compiler: msvc-14-seh
20 generator: "Visual Studio 14 2015 Win64"
21
22 - compiler: msvc-12-seh
23 generator: "Visual Studio 12 2013"
24
25 - compiler: msvc-12-seh
26 generator: "Visual Studio 12 2013 Win64"
27
28 - compiler: gcc-5.3.0-posix
29 generator: "MinGW Makefiles"
30 cxx_path: 'C:\mingw-w64\i686-5.3.0-posix-dwarf-rt_v4-rev0\mingw32\bin'
31 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
32
33 matrix:
34 fast_finish: true
35
36 install:
37 # git bash conflicts with MinGW makefiles
38 - if "%generator%"=="MinGW Makefiles" (set "PATH=%PATH:C:\Program Files\Git\usr\bin;=%")
39 - if not "%cxx_path%"=="" (set "PATH=%PATH%;%cxx_path%")
40
41 build_script:
42 - md _build -Force
43 - cd _build
44 - echo %configuration%
45 - cmake -G "%generator%" "-DCMAKE_BUILD_TYPE=%configuration%" -DBENCHMARK_DOWNLOAD_DEPENDENCIES=ON ..
46 - cmake --build . --config %configuration%
47
48 test_script:
49 - ctest -c %configuration% --timeout 300 --output-on-failure
50
51 artifacts:
52 - path: '_build/CMakeFiles/*.log'
53 name: logs
54 - path: '_build/Testing/**/*.xml'
55 name: test_results
0 # - Adds a compiler flag if it is supported by the compiler
1 #
2 # This function checks that the supplied compiler flag is supported and then
3 # adds it to the corresponding compiler flags
4 #
5 # add_cxx_compiler_flag( [])
6 #
7 # - Example
8 #
9 # include(AddCXXCompilerFlag)
10 # add_cxx_compiler_flag(-Wall)
11 # add_cxx_compiler_flag(-no-strict-aliasing RELEASE)
12 # Requires CMake 2.6+
13
14 if(__add_cxx_compiler_flag)
15 return()
16 endif()
17 set(__add_cxx_compiler_flag INCLUDED)
18
19 include(CheckCXXCompilerFlag)
20
21 function(mangle_compiler_flag FLAG OUTPUT)
22 string(TOUPPER "HAVE_CXX_FLAG_${FLAG}" SANITIZED_FLAG)
23 string(REPLACE "+" "X" SANITIZED_FLAG ${SANITIZED_FLAG})
24 string(REGEX REPLACE "[^A-Za-z_0-9]" "_" SANITIZED_FLAG ${SANITIZED_FLAG})
25 string(REGEX REPLACE "_+" "_" SANITIZED_FLAG ${SANITIZED_FLAG})
26 set(${OUTPUT} "${SANITIZED_FLAG}" PARENT_SCOPE)
27 endfunction(mangle_compiler_flag)
28
29 function(add_cxx_compiler_flag FLAG)
30 mangle_compiler_flag("${FLAG}" MANGLED_FLAG)
31 set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
32 set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}")
33 check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG})
34 set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}")
35 if(${MANGLED_FLAG})
36 set(VARIANT ${ARGV1})
37 if(ARGV1)
38 string(TOUPPER "_${VARIANT}" VARIANT)
39 endif()
40 set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${BENCHMARK_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE)
41 endif()
42 endfunction()
43
44 function(add_required_cxx_compiler_flag FLAG)
45 mangle_compiler_flag("${FLAG}" MANGLED_FLAG)
46 set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
47 set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}")
48 check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG})
49 set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}")
50 if(${MANGLED_FLAG})
51 set(VARIANT ${ARGV1})
52 if(ARGV1)
53 string(TOUPPER "_${VARIANT}" VARIANT)
54 endif()
55 set(CMAKE_CXX_FLAGS${VARIANT} "${CMAKE_CXX_FLAGS${VARIANT}} ${FLAG}" PARENT_SCOPE)
56 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE)
57 set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE)
58 set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${FLAG}" PARENT_SCOPE)
59 set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${FLAG}" PARENT_SCOPE)
60 else()
61 message(FATAL_ERROR "Required flag '${FLAG}' is not supported by the compiler")
62 endif()
63 endfunction()
64
65 function(check_cxx_warning_flag FLAG)
66 mangle_compiler_flag("${FLAG}" MANGLED_FLAG)
67 set(OLD_CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
68 # Add -Werror to ensure the compiler generates an error if the warning flag
69 # doesn't exist.
70 set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror ${FLAG}")
71 check_cxx_compiler_flag("${FLAG}" ${MANGLED_FLAG})
72 set(CMAKE_REQUIRED_FLAGS "${OLD_CMAKE_REQUIRED_FLAGS}")
73 endfunction()
0 # - Compile and run code to check for C++ features
1 #
2 # This functions compiles a source file under the `cmake` folder
3 # and adds the corresponding `HAVE_[FILENAME]` flag to the CMake
4 # environment
5 #
6 # cxx_feature_check( [])
7 #
8 # - Example
9 #
10 # include(CXXFeatureCheck)
11 # cxx_feature_check(STD_REGEX)
12 # Requires CMake 2.8.12+
13
14 if(__cxx_feature_check)
15 return()
16 endif()
17 set(__cxx_feature_check INCLUDED)
18
19 function(cxx_feature_check FILE)
20 string(TOLOWER ${FILE} FILE)
21 string(TOUPPER ${FILE} VAR)
22 string(TOUPPER "HAVE_${VAR}" FEATURE)
23 if (DEFINED HAVE_${VAR})
24 set(HAVE_${VAR} 1 PARENT_SCOPE)
25 add_definitions(-DHAVE_${VAR})
26 return()
27 endif()
28
29 if (NOT DEFINED COMPILE_${FEATURE})
30 message("-- Performing Test ${FEATURE}")
31 if(CMAKE_CROSSCOMPILING)
32 try_compile(COMPILE_${FEATURE}
33 ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp
34 CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}
35 LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES})
36 if(COMPILE_${FEATURE})
37 message(WARNING
38 "If you see build failures due to cross compilation, try setting HAVE_${VAR} to 0")
39 set(RUN_${FEATURE} 0)
40 else()
41 set(RUN_${FEATURE} 1)
42 endif()
43 else()
44 message("-- Performing Test ${FEATURE}")
45 try_run(RUN_${FEATURE} COMPILE_${FEATURE}
46 ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/${FILE}.cpp
47 CMAKE_FLAGS ${BENCHMARK_CXX_LINKER_FLAGS}
48 LINK_LIBRARIES ${BENCHMARK_CXX_LIBRARIES})
49 endif()
50 endif()
51
52 if(RUN_${FEATURE} EQUAL 0)
53 message("-- Performing Test ${FEATURE} -- success")
54 set(HAVE_${VAR} 1 PARENT_SCOPE)
55 add_definitions(-DHAVE_${VAR})
56 else()
57 if(NOT COMPILE_${FEATURE})
58 message("-- Performing Test ${FEATURE} -- failed to compile")
59 else()
60 message("-- Performing Test ${FEATURE} -- compiled but failed to run")
61 endif()
62 endif()
63 endfunction()
0 include("${CMAKE_CURRENT_LIST_DIR}/@targets_export_name@.cmake")
0 # - Returns a version string from Git tags
1 #
2 # This function inspects the annotated git tags for the project and returns a string
3 # into a CMake variable
4 #
5 # get_git_version()
6 #
7 # - Example
8 #
9 # include(GetGitVersion)
10 # get_git_version(GIT_VERSION)
11 #
12 # Requires CMake 2.8.11+
13 find_package(Git)
14
15 if(__get_git_version)
16 return()
17 endif()
18 set(__get_git_version INCLUDED)
19
20 function(get_git_version var)
21 if(GIT_EXECUTABLE)
22 execute_process(COMMAND ${GIT_EXECUTABLE} describe --match "v[0-9]*.[0-9]*.[0-9]*" --abbrev=8
23 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
24 RESULT_VARIABLE status
25 OUTPUT_VARIABLE GIT_VERSION
26 ERROR_QUIET)
27 if(${status})
28 set(GIT_VERSION "v0.0.0")
29 else()
30 string(STRIP ${GIT_VERSION} GIT_VERSION)
31 string(REGEX REPLACE "-[0-9]+-g" "-" GIT_VERSION ${GIT_VERSION})
32 endif()
33
34 # Work out if the repository is dirty
35 execute_process(COMMAND ${GIT_EXECUTABLE} update-index -q --refresh
36 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
37 OUTPUT_QUIET
38 ERROR_QUIET)
39 execute_process(COMMAND ${GIT_EXECUTABLE} diff-index --name-only HEAD --
40 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
41 OUTPUT_VARIABLE GIT_DIFF_INDEX
42 ERROR_QUIET)
43 string(COMPARE NOTEQUAL "${GIT_DIFF_INDEX}" "" GIT_DIRTY)
44 if (${GIT_DIRTY})
45 set(GIT_VERSION "${GIT_VERSION}-dirty")
46 endif()
47 else()
48 set(GIT_VERSION "v0.0.0")
49 endif()
50
51 message("-- git Version: ${GIT_VERSION}")
52 set(${var} ${GIT_VERSION} PARENT_SCOPE)
53 endfunction()
0
1 include(split_list)
2
3 macro(build_external_gtest)
4 include(ExternalProject)
5 set(GTEST_FLAGS "")
6 if (BENCHMARK_USE_LIBCXX)
7 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
8 list(APPEND GTEST_FLAGS -stdlib=libc++)
9 else()
10 message(WARNING "Unsupported compiler (${CMAKE_CXX_COMPILER}) when using libc++")
11 endif()
12 endif()
13 if (BENCHMARK_BUILD_32_BITS)
14 list(APPEND GTEST_FLAGS -m32)
15 endif()
16 if (NOT "${CMAKE_CXX_FLAGS}" STREQUAL "")
17 list(APPEND GTEST_FLAGS ${CMAKE_CXX_FLAGS})
18 endif()
19 string(TOUPPER "${CMAKE_BUILD_TYPE}" GTEST_BUILD_TYPE)
20 if ("${GTEST_BUILD_TYPE}" STREQUAL "COVERAGE")
21 set(GTEST_BUILD_TYPE "DEBUG")
22 endif()
23 # FIXME: Since 10/Feb/2017 the googletest trunk has had a bug where
24 # -Werror=unused-function fires during the build on OS X. This is a temporary
25 # workaround to keep our travis bots from failing. It should be removed
26 # once gtest is fixed.
27 if (NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
28 list(APPEND GTEST_FLAGS "-Wno-unused-function")
29 endif()
30 split_list(GTEST_FLAGS)
31 set(EXCLUDE_FROM_ALL_OPT "")
32 set(EXCLUDE_FROM_ALL_VALUE "")
33 if (${CMAKE_VERSION} VERSION_GREATER "3.0.99")
34 set(EXCLUDE_FROM_ALL_OPT "EXCLUDE_FROM_ALL")
35 set(EXCLUDE_FROM_ALL_VALUE "ON")
36 endif()
37 ExternalProject_Add(googletest
38 ${EXCLUDE_FROM_ALL_OPT} ${EXCLUDE_FROM_ALL_VALUE}
39 GIT_REPOSITORY https://github.com/google/googletest.git
40 GIT_TAG master
41 PREFIX "${CMAKE_BINARY_DIR}/googletest"
42 INSTALL_DIR "${CMAKE_BINARY_DIR}/googletest"
43 CMAKE_CACHE_ARGS
44 -DCMAKE_BUILD_TYPE:STRING=${GTEST_BUILD_TYPE}
45 -DCMAKE_C_COMPILER:STRING=${CMAKE_C_COMPILER}
46 -DCMAKE_CXX_COMPILER:STRING=${CMAKE_CXX_COMPILER}
47 -DCMAKE_INSTALL_PREFIX:PATH=
48 -DCMAKE_INSTALL_LIBDIR:PATH=/lib
49 -DCMAKE_CXX_FLAGS:STRING=${GTEST_FLAGS}
50 -Dgtest_force_shared_crt:BOOL=ON
51 )
52
53 ExternalProject_Get_Property(googletest install_dir)
54 set(GTEST_INCLUDE_DIRS ${install_dir}/include)
55 file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS})
56
57 set(LIB_SUFFIX "${CMAKE_STATIC_LIBRARY_SUFFIX}")
58 set(LIB_PREFIX "${CMAKE_STATIC_LIBRARY_PREFIX}")
59 if("${GTEST_BUILD_TYPE}" STREQUAL "DEBUG")
60 set(LIB_SUFFIX "d${CMAKE_STATIC_LIBRARY_SUFFIX}")
61 endif()
62
63 # Use gmock_main instead of gtest_main because it initializes gtest as well.
64 # Note: The libraries are listed in reverse order of their dependancies.
65 foreach(LIB gtest gmock gmock_main)
66 add_library(${LIB} UNKNOWN IMPORTED)
67 set_target_properties(${LIB} PROPERTIES
68 IMPORTED_LOCATION ${install_dir}/lib/${LIB_PREFIX}${LIB}${LIB_SUFFIX}
69 INTERFACE_INCLUDE_DIRECTORIES ${GTEST_INCLUDE_DIRS}
70 INTERFACE_LINK_LIBRARIES "${GTEST_BOTH_LIBRARIES}"
71 )
72 add_dependencies(${LIB} googletest)
73 list(APPEND GTEST_BOTH_LIBRARIES ${LIB})
74 endforeach()
75 endmacro(build_external_gtest)
76
77 if (BENCHMARK_ENABLE_GTEST_TESTS)
78 if (IS_DIRECTORY ${CMAKE_SOURCE_DIR}/googletest)
79 set(GTEST_ROOT "${CMAKE_SOURCE_DIR}/googletest")
80 set(INSTALL_GTEST OFF CACHE INTERNAL "")
81 set(INSTALL_GMOCK OFF CACHE INTERNAL "")
82 add_subdirectory(${CMAKE_SOURCE_DIR}/googletest)
83 set(GTEST_BOTH_LIBRARIES gtest gmock gmock_main)
84 foreach(HEADER test mock)
85 # CMake 2.8 and older don't respect INTERFACE_INCLUDE_DIRECTORIES, so we
86 # have to add the paths ourselves.
87 set(HFILE g${HEADER}/g${HEADER}.h)
88 set(HPATH ${GTEST_ROOT}/google${HEADER}/include)
89 find_path(HEADER_PATH_${HEADER} ${HFILE}
90 NO_DEFAULT_PATHS
91 HINTS ${HPATH}
92 )
93 if (NOT HEADER_PATH_${HEADER})
94 message(FATAL_ERROR "Failed to find header ${HFILE} in ${HPATH}")
95 endif()
96 list(APPEND GTEST_INCLUDE_DIRS ${HEADER_PATH_${HEADER}})
97 endforeach()
98 elseif(BENCHMARK_DOWNLOAD_DEPENDENCIES)
99 build_external_gtest()
100 else()
101 find_package(GTest REQUIRED)
102 find_path(GMOCK_INCLUDE_DIRS gmock/gmock.h
103 HINTS ${GTEST_INCLUDE_DIRS})
104 if (NOT GMOCK_INCLUDE_DIRS)
105 message(FATAL_ERROR "Failed to find header gmock/gmock.h with hint ${GTEST_INCLUDE_DIRS}")
106 endif()
107 set(GTEST_INCLUDE_DIRS ${GTEST_INCLUDE_DIRS} ${GMOCK_INCLUDE_DIRS})
108 # FIXME: We don't currently require the gmock library to build the tests,
109 # and it's likely we won't find it, so we don't try. As long as we've
110 # found the gmock/gmock.h header and gtest_main that should be good enough.
111 endif()
112 endif()
0 include(FeatureSummary)
1
2 find_program(LLVMAR_EXECUTABLE
3 NAMES llvm-ar
4 DOC "The llvm-ar executable"
5 )
6
7 include(FindPackageHandleStandardArgs)
8 find_package_handle_standard_args(LLVMAr
9 DEFAULT_MSG
10 LLVMAR_EXECUTABLE)
11
12 SET_PACKAGE_PROPERTIES(LLVMAr PROPERTIES
13 URL https://llvm.org/docs/CommandGuide/llvm-ar.html
14 DESCRIPTION "create, modify, and extract from archives"
15 )
0 include(FeatureSummary)
1
2 find_program(LLVMNM_EXECUTABLE
3 NAMES llvm-nm
4 DOC "The llvm-nm executable"
5 )
6
7 include(FindPackageHandleStandardArgs)
8 find_package_handle_standard_args(LLVMNm
9 DEFAULT_MSG
10 LLVMNM_EXECUTABLE)
11
12 SET_PACKAGE_PROPERTIES(LLVMNm PROPERTIES
13 URL https://llvm.org/docs/CommandGuide/llvm-nm.html
14 DESCRIPTION "list LLVM bitcode and object file’s symbol table"
15 )
0 include(FeatureSummary)
1
2 find_program(LLVMRANLIB_EXECUTABLE
3 NAMES llvm-ranlib
4 DOC "The llvm-ranlib executable"
5 )
6
7 include(FindPackageHandleStandardArgs)
8 find_package_handle_standard_args(LLVMRanLib
9 DEFAULT_MSG
10 LLVMRANLIB_EXECUTABLE)
11
12 SET_PACKAGE_PROPERTIES(LLVMRanLib PROPERTIES
13 DESCRIPTION "generate index for LLVM archive"
14 )
0 prefix=@CMAKE_INSTALL_PREFIX@
1 exec_prefix=${prefix}
2 libdir=${prefix}/lib
3 includedir=${prefix}/include
4
5 Name: @PROJECT_NAME@
6 Description: Google microbenchmark framework
7 Version: @VERSION@
8
9 Libs: -L${libdir} -lbenchmark
10 Cflags: -I${includedir}
0 #include
1 #include
2 int main() {
3 std::string str = "test0159";
4 regex_t re;
5 int ec = regcomp(&re, "^[a-z]+[0-9]+$", REG_EXTENDED | REG_NOSUB);
6 if (ec != 0) {
7 return ec;
8 }
9 return regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0;
10 }
0 find_package(LLVMAr REQUIRED)
1 set(CMAKE_AR "${LLVMAR_EXECUTABLE}" CACHE FILEPATH "" FORCE)
2
3 find_package(LLVMNm REQUIRED)
4 set(CMAKE_NM "${LLVMNM_EXECUTABLE}" CACHE FILEPATH "" FORCE)
5
6 find_package(LLVMRanLib REQUIRED)
7 set(CMAKE_RANLIB "${LLVMRANLIB_EXECUTABLE}" CACHE FILEPATH "" FORCE)
0 #include
1 #include
2 int main() {
3 std::string str = "test0159";
4 regex_t re;
5 int ec = regcomp(&re, "^[a-z]+[0-9]+$", REG_EXTENDED | REG_NOSUB);
6 if (ec != 0) {
7 return ec;
8 }
9 int ret = regexec(&re, str.c_str(), 0, nullptr, 0) ? -1 : 0;
10 regfree(&re);
11 return ret;
12 }
0 macro(split_list listname)
1 string(REPLACE ";" " " ${listname} "${${listname}}")
2 endmacro()
0 #include
1 #include
2 int main() {
3 const std::string str = "test0159";
4 std::regex re;
5 re = std::regex("^[a-z]+[0-9]+$",
6 std::regex_constants::extended | std::regex_constants::nosubs);
7 return std::regex_search(str, re) ? 0 : -1;
8 }
0 #include
1
2 int main() {
3 typedef std::chrono::steady_clock Clock;
4 Clock::time_point tp = Clock::now();
5 ((void)tp);
6 }
0 #define HAVE_THREAD_SAFETY_ATTRIBUTES
1 #include "../src/mutex.h"
2
3 int main() {}
0 # Assembly Tests
1
2 The Benchmark library provides a number of functions whose primary
3 purpose in to affect assembly generation, including `DoNotOptimize`
4 and `ClobberMemory`. In addition there are other functions,
5 such as `KeepRunning`, for which generating good assembly is paramount.
6
7 For these functions it's important to have tests that verify the
8 correctness and quality of the implementation. This requires testing
9 the code generated by the compiler.
10
11 This document describes how the Benchmark library tests compiler output,
12 as well as how to properly write new tests.
13
14
15 ## Anatomy of a Test
16
17 Writing a test has two steps:
18
19 * Write the code you want to generate assembly for.
20 * Add `// CHECK` lines to match against the verified assembly.
21
22 Example:
23 ```c++
24
25 // CHECK-LABEL: test_add:
26 extern "C" int test_add() {
27 extern int ExternInt;
28 return ExternInt + 1;
29
30 // CHECK: movl ExternInt(%rip), %eax
31 // CHECK: addl %eax
32 // CHECK: ret
33 }
34
35 ```
36
37 #### LLVM Filecheck
38
39 [LLVM's Filecheck](https://llvm.org/docs/CommandGuide/FileCheck.html)
40 is used to test the generated assembly against the `// CHECK` lines
41 specified in the tests source file. Please see the documentation
42 linked above for information on how to write `CHECK` directives.
43
44 #### Tips and Tricks:
45
46 * Tests should match the minimal amount of output required to establish
47 correctness. `CHECK` directives don't have to match on the exact next line
48 after the previous match, so tests should omit checks for unimportant
49 bits of assembly. ([`CHECK-NEXT`](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-next-directive)
50 can be used to ensure a match occurs exactly after the previous match).
51
52 * The tests are compiled with `-O3 -g0`. So we're only testing the
53 optimized output.
54
55 * The assembly output is further cleaned up using `tools/strip_asm.py`.
56 This removes comments, assembler directives, and unused labels before
57 the test is run.
58
59 * The generated and stripped assembly file for a test is output under
60 `/test/.s`
61
62 * Filecheck supports using [`CHECK` prefixes](https://llvm.org/docs/CommandGuide/FileCheck.html#cmdoption-check-prefixes)
63 to specify lines that should only match in certain situations.
64 The Benchmark tests use `CHECK-CLANG` and `CHECK-GNU` for lines that
65 are only expected to match Clang or GCC's output respectively. Normal
66 `CHECK` lines match against all compilers. (Note: `CHECK-NOT` and
67 `CHECK-LABEL` are NOT prefixes. They are versions of non-prefixed
68 `CHECK` lines)
69
70 * Use `extern "C"` to disable name mangling for specific functions. This
71 makes them easier to name in the `CHECK` lines.
72
73
74 ## Problems Writing Portable Tests
75
76 Writing tests which check the code generated by a compiler are
77 inherently non-portable. Different compilers and even different compiler
78 versions may generate entirely different code. The Benchmark tests
79 must tolerate this.
80
81 LLVM Filecheck provides a number of mechanisms to help write
82 "more portable" tests; including [matching using regular expressions](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-pattern-matching-syntax),
83 allowing the creation of [named variables](https://llvm.org/docs/CommandGuide/FileCheck.html#filecheck-variables)
84 for later matching, and [checking non-sequential matches](https://llvm.org/docs/CommandGuide/FileCheck.html#the-check-dag-directive).
85
86 #### Capturing Variables
87
88 For example, say GCC stores a variable in a register but Clang stores
89 it in memory. To write a test that tolerates both cases we "capture"
90 the destination of the store, and then use the captured expression
91 to write the remainder of the test.
92
93 ```c++
94 // CHECK-LABEL: test_div_no_op_into_shr:
95 extern "C" void test_div_no_op_into_shr(int value) {
96 int divisor = 2;
97 benchmark::DoNotOptimize(divisor); // hide the value from the optimizer
98 return value / divisor;
99
100 // CHECK: movl $2, [[DEST:.*]]
101 // CHECK: idivl [[DEST]]
102 // CHECK: ret
103 }
104 ```
105
106 #### Using Regular Expressions to Match Differing Output
107
108 Often tests require testing assembly lines which may subtly differ
109 between compilers or compiler versions. A common example of this
110 is matching stack frame addresses. In this case regular expressions
111 can be used to match the differing bits of output. For example:
112
113 ```c++
114 int ExternInt;
115 struct Point { int x, y, z; };
116
117 // CHECK-LABEL: test_store_point:
118 extern "C" void test_store_point() {
119 Point p{ExternInt, ExternInt, ExternInt};
120 benchmark::DoNotOptimize(p);
121
122 // CHECK: movl ExternInt(%rip), %eax
123 // CHECK: movl %eax, -{{[0-9]+}}(%rsp)
124 // CHECK: movl %eax, -{{[0-9]+}}(%rsp)
125 // CHECK: movl %eax, -{{[0-9]+}}(%rsp)
126 // CHECK: ret
127 }
128 ```
129
130 ## Current Requirements and Limitations
131
132 The tests require Filecheck to be installed along the `PATH` of the
133 build machine. Otherwise the tests will be disabled.
134
135 Additionally, as mentioned in the previous section, codegen tests are
136 inherently non-portable. Currently the tests are limited to:
137
138 * x86_64 targets.
139 * Compiled with GCC or Clang
140
141 Further work could be done, at least on a limited basis, to extend the
142 tests to other architectures and compilers (using `CHECK` prefixes).
143
144 Furthermore, the tests fail for builds which specify additional flags
145 that modify code generation, including `--coverage` or `-fsanitize=`.
0 # Benchmark Tools
1
2 ## compare_bench.py
3
4 The `compare_bench.py` utility which can be used to compare the result of benchmarks.
5 The program is invoked like:
6
7 ``` bash
8 $ compare_bench.py [benchmark options]...
9 ```
10
11 Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.
12
13 `[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.
14
15 The sample output using the JSON test files under `Inputs/` gives:
16
17 ``` bash
18 $ ./compare_bench.py ./gbench/Inputs/test1_run1.json ./gbench/Inputs/test1_run2.json
19 Comparing ./gbench/Inputs/test1_run1.json to ./gbench/Inputs/test1_run2.json
20 Benchmark Time CPU Time Old Time New CPU Old CPU New
21 -------------------------------------------------------------------------------------------------------------
22 BM_SameTimes +0.0000 +0.0000 10 10 10 10
23 BM_2xFaster -0.5000 -0.5000 50 25 50 25
24 BM_2xSlower +1.0000 +1.0000 50 100 50 100
25 BM_1PercentFaster -0.0100 -0.0100 100 99 100 99
26 BM_1PercentSlower +0.0100 +0.0100 100 101 100 101
27 BM_10PercentFaster -0.1000 -0.1000 100 90 100 90
28 BM_10PercentSlower +0.1000 +0.1000 100 110 100 110
29 BM_100xSlower +99.0000 +99.0000 100 10000 100 10000
30 BM_100xFaster -0.9900 -0.9900 10000 100 10000 100
31 BM_10PercentCPUToTime +0.1000 -0.1000 100 110 100 90
32 BM_ThirdFaster -0.3333 -0.3334 100 67 100 67
33 BM_BadTimeUnit -0.9000 +0.2000 0 0 0 1
34 ```
35
36 As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.
37
38 When a benchmark executable is run, the raw output from the benchmark is printed in real time to stdout. The sample output using `benchmark/basic_test` for both arguments looks like:
39
40 ```
41 ./compare_bench.py test/basic_test test/basic_test --benchmark_filter=BM_empty.*
42 RUNNING: test/basic_test --benchmark_filter=BM_empty.* --benchmark_out=/tmp/tmpN7LF3a
43 Run on (8 X 4000 MHz CPU s)
44 2017-11-07 23:28:36
45 ---------------------------------------------------------------------
46 Benchmark Time CPU Iterations
47 ---------------------------------------------------------------------
48 BM_empty 4 ns 4 ns 170178757
49 BM_empty/threads:8 1 ns 7 ns 103868920
50 BM_empty_stop_start 0 ns 0 ns 1000000000
51 BM_empty_stop_start/threads:8 0 ns 0 ns 1403031720
52 RUNNING: /test/basic_test --benchmark_filter=BM_empty.* --benchmark_out=/tmp/tmplvrIp8
53 Run on (8 X 4000 MHz CPU s)
54 2017-11-07 23:28:38
55 ---------------------------------------------------------------------
56 Benchmark Time CPU Iterations
57 ---------------------------------------------------------------------
58 BM_empty 4 ns 4 ns 169534855
59 BM_empty/threads:8 1 ns 7 ns 104188776
60 BM_empty_stop_start 0 ns 0 ns 1000000000
61 BM_empty_stop_start/threads:8 0 ns 0 ns 1404159424
62 Comparing ../build/test/basic_test to ../build/test/basic_test
63 Benchmark Time CPU Time Old Time New CPU Old CPU New
64 ---------------------------------------------------------------------------------------------------------------------
65 BM_empty -0.0048 -0.0049 4 4 4 4
66 BM_empty/threads:8 -0.0123 -0.0054 1 1 7 7
67 BM_empty_stop_start -0.0000 -0.0000 0 0 0 0
68 BM_empty_stop_start/threads:8 -0.0029 +0.0001 0 0 0 0
69
70 ```
71
72 As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.
73 Obviously this example doesn't give any useful output, but it's intended to show the output format when 'compare_bench.py' needs to run benchmarks.
74
75 ## compare.py
76
77 The `compare.py` can be used to compare the result of benchmarks.
78 There are three modes of operation:
79
80 1. Just compare two benchmarks, what `compare_bench.py` did.
81 The program is invoked like:
82
83 ``` bash
84 $ compare.py benchmarks [benchmark options]...
85 ```
86 Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.
87
88 `[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.
89
90 Example output:
91 ```
92 $ ./compare.py benchmarks ./a.out ./a.out
93 RUNNING: ./a.out --benchmark_out=/tmp/tmprBT5nW
94 Run on (8 X 4000 MHz CPU s)
95 2017-11-07 21:16:44
96 ------------------------------------------------------
97 Benchmark Time CPU Iterations
98 ------------------------------------------------------
99 BM_memcpy/8 36 ns 36 ns 19101577 211.669MB/s
100 BM_memcpy/64 76 ns 76 ns 9412571 800.199MB/s
101 BM_memcpy/512 84 ns 84 ns 8249070 5.64771GB/s
102 BM_memcpy/1024 116 ns 116 ns 6181763 8.19505GB/s
103 BM_memcpy/8192 643 ns 643 ns 1062855 11.8636GB/s
104 BM_copy/8 222 ns 222 ns 3137987 34.3772MB/s
105 BM_copy/64 1608 ns 1608 ns 432758 37.9501MB/s
106 BM_copy/512 12589 ns 12589 ns 54806 38.7867MB/s
107 BM_copy/1024 25169 ns 25169 ns 27713 38.8003MB/s
108 BM_copy/8192 201165 ns 201112 ns 3486 38.8466MB/s
109 RUNNING: ./a.out --benchmark_out=/tmp/tmpt1wwG_
110 Run on (8 X 4000 MHz CPU s)
111 2017-11-07 21:16:53
112 ------------------------------------------------------
113 Benchmark Time CPU Iterations
114 ------------------------------------------------------
115 BM_memcpy/8 36 ns 36 ns 19397903 211.255MB/s
116 BM_memcpy/64 73 ns 73 ns 9691174 839.635MB/s
117 BM_memcpy/512 85 ns 85 ns 8312329 5.60101GB/s
118 BM_memcpy/1024 118 ns 118 ns 6438774 8.11608GB/s
119 BM_memcpy/8192 656 ns 656 ns 1068644 11.6277GB/s
120 BM_copy/8 223 ns 223 ns 3146977 34.2338MB/s
121 BM_copy/64 1611 ns 1611 ns 435340 37.8751MB/s
122 BM_copy/512 12622 ns 12622 ns 54818 38.6844MB/s
123 BM_copy/1024 25257 ns 25239 ns 27779 38.6927MB/s
124 BM_copy/8192 205013 ns 205010 ns 3479 38.108MB/s
125 Comparing ./a.out to ./a.out
126 Benchmark Time CPU Time Old Time New CPU Old CPU New
127 ------------------------------------------------------------------------------------------------------
128 BM_memcpy/8 +0.0020 +0.0020 36 36 36 36
129 BM_memcpy/64 -0.0468 -0.0470 76 73 76 73
130 BM_memcpy/512 +0.0081 +0.0083 84 85 84 85
131 BM_memcpy/1024 +0.0098 +0.0097 116 118 116 118
132 BM_memcpy/8192 +0.0200 +0.0203 643 656 643 656
133 BM_copy/8 +0.0046 +0.0042 222 223 222 223
134 BM_copy/64 +0.0020 +0.0020 1608 1611 1608 1611
135 BM_copy/512 +0.0027 +0.0026 12589 12622 12589 12622
136 BM_copy/1024 +0.0035 +0.0028 25169 25257 25169 25239
137 BM_copy/8192 +0.0191 +0.0194 201165 205013 201112 205010
138 ```
139
140 What it does is for the every benchmark from the first run it looks for the benchmark with exactly the same name in the second run, and then compares the results. If the names differ, the benchmark is omitted from the diff.
141 As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.
142
143 2. Compare two different filters of one benchmark
144 The program is invoked like:
145
146 ``` bash
147 $ compare.py filters [benchmark options]...
148 ```
149 Where `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.
150
151 Where `` and `` are the same regex filters that you would pass to the `[--benchmark_filter=]` parameter of the benchmark binary.
152
153 `[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.
154
155 Example output:
156 ```
157 $ ./compare.py filters ./a.out BM_memcpy BM_copy
158 RUNNING: ./a.out --benchmark_filter=BM_memcpy --benchmark_out=/tmp/tmpBWKk0k
159 Run on (8 X 4000 MHz CPU s)
160 2017-11-07 21:37:28
161 ------------------------------------------------------
162 Benchmark Time CPU Iterations
163 ------------------------------------------------------
164 BM_memcpy/8 36 ns 36 ns 17891491 211.215MB/s
165 BM_memcpy/64 74 ns 74 ns 9400999 825.646MB/s
166 BM_memcpy/512 87 ns 87 ns 8027453 5.46126GB/s
167 BM_memcpy/1024 111 ns 111 ns 6116853 8.5648GB/s
168 BM_memcpy/8192 657 ns 656 ns 1064679 11.6247GB/s
169 RUNNING: ./a.out --benchmark_filter=BM_copy --benchmark_out=/tmp/tmpAvWcOM
170 Run on (8 X 4000 MHz CPU s)
171 2017-11-07 21:37:33
172 ----------------------------------------------------
173 Benchmark Time CPU Iterations
174 ----------------------------------------------------
175 BM_copy/8 227 ns 227 ns 3038700 33.6264MB/s
176 BM_copy/64 1640 ns 1640 ns 426893 37.2154MB/s
177 BM_copy/512 12804 ns 12801 ns 55417 38.1444MB/s
178 BM_copy/1024 25409 ns 25407 ns 27516 38.4365MB/s
179 BM_copy/8192 202986 ns 202990 ns 3454 38.4871MB/s
180 Comparing BM_memcpy to BM_copy (from ./a.out)
181 Benchmark Time CPU Time Old Time New CPU Old CPU New
182 --------------------------------------------------------------------------------------------------------------------
183 [BM_memcpy vs. BM_copy]/8 +5.2829 +5.2812 36 227 36 227
184 [BM_memcpy vs. BM_copy]/64 +21.1719 +21.1856 74 1640 74 1640
185 [BM_memcpy vs. BM_copy]/512 +145.6487 +145.6097 87 12804 87 12801
186 [BM_memcpy vs. BM_copy]/1024 +227.1860 +227.1776 111 25409 111 25407
187 [BM_memcpy vs. BM_copy]/8192 +308.1664 +308.2898 657 202986 656 202990
188 ```
189
190 As you can see, it applies filter to the benchmarks, both when running the benchmark, and before doing the diff. And to make the diff work, the matches are replaced with some common string. Thus, you can compare two different benchmark families within one benchmark binary.
191 As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.
192
193 3. Compare filter one from benchmark one to filter two from benchmark two:
194 The program is invoked like:
195
196 ``` bash
197 $ compare.py filters [benchmark options]...
198 ```
199
200 Where `` and `` either specify a benchmark executable file, or a JSON output file. The type of the input file is automatically detected. If a benchmark executable is specified then the benchmark is run to obtain the results. Otherwise the results are simply loaded from the output file.
201
202 Where `` and `` are the same regex filters that you would pass to the `[--benchmark_filter=]` parameter of the benchmark binary.
203
204 `[benchmark options]` will be passed to the benchmarks invocations. They can be anything that binary accepts, be it either normal `--benchmark_*` parameters, or some custom parameters your binary takes.
205
206 Example output:
207 ```
208 $ ./compare.py benchmarksfiltered ./a.out BM_memcpy ./a.out BM_copy
209 RUNNING: ./a.out --benchmark_filter=BM_memcpy --benchmark_out=/tmp/tmp_FvbYg
210 Run on (8 X 4000 MHz CPU s)
211 2017-11-07 21:38:27
212 ------------------------------------------------------
213 Benchmark Time CPU Iterations
214 ------------------------------------------------------
215 BM_memcpy/8 37 ns 37 ns 18953482 204.118MB/s
216 BM_memcpy/64 74 ns 74 ns 9206578 828.245MB/s
217 BM_memcpy/512 91 ns 91 ns 8086195 5.25476GB/s
218 BM_memcpy/1024 120 ns 120 ns 5804513 7.95662GB/s
219 BM_memcpy/8192 664 ns 664 ns 1028363 11.4948GB/s
220 RUNNING: ./a.out --benchmark_filter=BM_copy --benchmark_out=/tmp/tmpDfL5iE
221 Run on (8 X 4000 MHz CPU s)
222 2017-11-07 21:38:32
223 ----------------------------------------------------
224 Benchmark Time CPU Iterations
225 ----------------------------------------------------
226 BM_copy/8 230 ns 230 ns 2985909 33.1161MB/s
227 BM_copy/64 1654 ns 1653 ns 419408 36.9137MB/s
228 BM_copy/512 13122 ns 13120 ns 53403 37.2156MB/s
229 BM_copy/1024 26679 ns 26666 ns 26575 36.6218MB/s
230 BM_copy/8192 215068 ns 215053 ns 3221 36.3283MB/s
231 Comparing BM_memcpy (from ./a.out) to BM_copy (from ./a.out)
232 Benchmark Time CPU Time Old Time New CPU Old CPU New
233 --------------------------------------------------------------------------------------------------------------------
234 [BM_memcpy vs. BM_copy]/8 +5.1649 +5.1637 37 230 37 230
235 [BM_memcpy vs. BM_copy]/64 +21.4352 +21.4374 74 1654 74 1653
236 [BM_memcpy vs. BM_copy]/512 +143.6022 +143.5865 91 13122 91 13120
237 [BM_memcpy vs. BM_copy]/1024 +221.5903 +221.4790 120 26679 120 26666
238 [BM_memcpy vs. BM_copy]/8192 +322.9059 +323.0096 664 215068 664 215053
239 ```
240 This is a mix of the previous two modes, two (potentially different) benchmark binaries are run, and a different filter is applied to each one.
241 As you can note, the values in `Time` and `CPU` columns are calculated as `(new - old) / |old|`.
0 // Copyright 2015 Google Inc. All rights reserved.
1 //
2 // Licensed under the Apache License, Version 2.0 (the "License");
3 // you may not use this file except in compliance with the License.
4 // You may obtain a copy of the License at
5 //
6 // http://www.apache.org/licenses/LICENSE-2.0
7 //
8 // Unless required by applicable law or agreed to in writing, software
9 // distributed under the License is distributed on an "AS IS" BASIS,
10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 // See the License for the specific language governing permissions and
12 // limitations under the License.
13
14 // Support for registering benchmarks for functions.
15
16 /* Example usage:
17 // Define a function that executes the code to be measured a
18 // specified number of times:
19 static void BM_StringCreation(benchmark::State& state) {
20 for (auto _ : state)
21 std::string empty_string;
22 }
23
24 // Register the function as a benchmark
25 BENCHMARK(BM_StringCreation);
26
27 // Define another benchmark
28 static void BM_StringCopy(benchmark::State& state) {
29 std::string x = "hello";
30 for (auto _ : state)
31 std::string copy(x);
32 }
33 BENCHMARK(BM_StringCopy);
34
35 // Augment the main() program to invoke benchmarks if specified
36 // via the --benchmarks command line flag. E.g.,
37 // my_unittest --benchmark_filter=all
38 // my_unittest --benchmark_filter=BM_StringCreation
39 // my_unittest --benchmark_filter=String
40 // my_unittest --benchmark_filter='Copy|Creation'
41 int main(int argc, char** argv) {
42 benchmark::Initialize(&argc, argv);
43 benchmark::RunSpecifiedBenchmarks();
44 return 0;
45 }
46
47 // Sometimes a family of microbenchmarks can be implemented with
48 // just one routine that takes an extra argument to specify which
49 // one of the family of benchmarks to run. For example, the following
50 // code defines a family of microbenchmarks for measuring the speed
51 // of memcpy() calls of different lengths:
52
53 static void BM_memcpy(benchmark::State& state) {
54 char* src = new char[state.range(0)]; char* dst = new char[state.range(0)];
55 memset(src, 'x', state.range(0));
56 for (auto _ : state)
57 memcpy(dst, src, state.range(0));
58 state.SetBytesProcessed(int64_t(state.iterations()) *
59 int64_t(state.range(0)));
60 delete[] src; delete[] dst;
61 }
62 BENCHMARK(BM_memcpy)->Arg(8)->Arg(64)->Arg(512)->Arg(1<<10)->Arg(8<<10);
63
64 // The preceding code is quite repetitive, and can be replaced with the
65 // following short-hand. The following invocation will pick a few
66 // appropriate arguments in the specified range and will generate a
67 // microbenchmark for each such argument.
68 BENCHMARK(BM_memcpy)->Range(8, 8<<10);
69
70 // You might have a microbenchmark that depends on two inputs. For
71 // example, the following code defines a family of microbenchmarks for
72 // measuring the speed of set insertion.
73 static void BM_SetInsert(benchmark::State& state) {
74 set data;
75 for (auto _ : state) {
76 state.PauseTiming();
77 data = ConstructRandomSet(state.range(0));
78 state.ResumeTiming();
79 for (int j = 0; j < state.range(1); ++j)
80 data.insert(RandomNumber());
81 }
82 }
83 BENCHMARK(BM_SetInsert)
84 ->Args({1<<10, 128})
85 ->Args({2<<10, 128})
86 ->Args({4<<10, 128})
87 ->Args({8<<10, 128})
88 ->Args({1<<10, 512})
89 ->Args({2<<10, 512})
90 ->Args({4<<10, 512})
91 ->Args({8<<10, 512});
92
93 // The preceding code is quite repetitive, and can be replaced with
94 // the following short-hand. The following macro will pick a few
95 // appropriate arguments in the product of the two specified ranges
96 // and will generate a microbenchmark for each such pair.
97 BENCHMARK(BM_SetInsert)->Ranges({{1<<10, 8<<10}, {128, 512}});
98
99 // For more complex patterns of inputs, passing a custom function
100 // to Apply allows programmatic specification of an
101 // arbitrary set of arguments to run the microbenchmark on.
102 // The following example enumerates a dense range on
103 // one parameter, and a sparse range on the second.
104 static void CustomArguments(benchmark::internal::Benchmark* b) {
105 for (int i = 0; i <= 10; ++i)
106 for (int j = 32; j <= 1024*1024; j *= 8)
107 b->Args({i, j});
108 }
109 BENCHMARK(BM_SetInsert)->Apply(CustomArguments);
110
111 // Templated microbenchmarks work the same way:
112 // Produce then consume 'size' messages 'iters' times
113 // Measures throughput in the absence of multiprogramming.
114 template int BM_Sequential(benchmark::State& state) {
115 Q q;
116 typename Q::value_type v;
117 for (auto _ : state) {
118 for (int i = state.range(0); i--; )
119 q.push(v);
120 for (int e = state.range(0); e--; )
121 q.Wait(&v);
122 }
123 // actually messages, not bytes:
124 state.SetBytesProcessed(
125 static_cast(state.iterations())*state.range(0));
126 }
127 BENCHMARK_TEMPLATE(BM_Sequential, WaitQueue)->Range(1<<0, 1<<10);
128
129 Use `Benchmark::MinTime(double t)` to set the minimum time used to run the
130 benchmark. This option overrides the `benchmark_min_time` flag.
131
132 void BM_test(benchmark::State& state) {
133 ... body ...
134 }
135 BENCHMARK(BM_test)->MinTime(2.0); // Run for at least 2 seconds.
136
137 In a multithreaded test, it is guaranteed that none of the threads will start
138 until all have reached the loop start, and all will have finished before any
139 thread exits the loop body. As such, any global setup or teardown you want to
140 do can be wrapped in a check against the thread index:
141
142 static void BM_MultiThreaded(benchmark::State& state) {
143 if (state.thread_index == 0) {
144 // Setup code here.
145 }
146 for (auto _ : state) {
147 // Run the test as normal.
148 }
149 if (state.thread_index == 0) {
150 // Teardown code here.
151 }
152 }
153 BENCHMARK(BM_MultiThreaded)->Threads(4);
154
155
156 If a benchmark runs a few milliseconds it may be hard to visually compare the
157 measured times, since the output data is given in nanoseconds per default. In
158 order to manually set the time unit, you can specify it manually:
159
160 BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
161 */
162
163 #ifndef BENCHMARK_BENCHMARK_H_
164 #define BENCHMARK_BENCHMARK_H_
165
166
167 // The _MSVC_LANG check should detect Visual Studio 2015 Update 3 and newer.
168 #if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L)
169 #define BENCHMARK_HAS_CXX11
170 #endif
171
172 #include
173
174 #include
175 #include
176 #include
177 #include
178 #include
179 #include
180 #include
181 #include
182
183 #if defined(BENCHMARK_HAS_CXX11)
184 #include
185 #include
186 #include
187 #endif
188
189 #if defined(_MSC_VER)
190 #include // for _ReadWriteBarrier
191 #endif
192
193 #ifndef BENCHMARK_HAS_CXX11
194 #define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \
195 TypeName(const TypeName&); \
196 TypeName& operator=(const TypeName&)
197 #else
198 #define BENCHMARK_DISALLOW_COPY_AND_ASSIGN(TypeName) \
199 TypeName(const TypeName&) = delete; \
200 TypeName& operator=(const TypeName&) = delete
201 #endif
202
203 #if defined(__GNUC__)
204 #define BENCHMARK_UNUSED __attribute__((unused))
205 #define BENCHMARK_ALWAYS_INLINE __attribute__((always_inline))
206 #define BENCHMARK_NOEXCEPT noexcept
207 #define BENCHMARK_NOEXCEPT_OP(x) noexcept(x)
208 #elif defined(_MSC_VER) && !defined(__clang__)
209 #define BENCHMARK_UNUSED
210 #define BENCHMARK_ALWAYS_INLINE __forceinline
211 #if _MSC_VER >= 1900
212 #define BENCHMARK_NOEXCEPT noexcept
213 #define BENCHMARK_NOEXCEPT_OP(x) noexcept(x)
214 #else
215 #define BENCHMARK_NOEXCEPT
216 #define BENCHMARK_NOEXCEPT_OP(x)
217 #endif
218 #define __func__ __FUNCTION__
219 #else
220 #define BENCHMARK_UNUSED
221 #define BENCHMARK_ALWAYS_INLINE
222 #define BENCHMARK_NOEXCEPT
223 #define BENCHMARK_NOEXCEPT_OP(x)
224 #endif
225
226 #define BENCHMARK_INTERNAL_TOSTRING2(x) #x
227 #define BENCHMARK_INTERNAL_TOSTRING(x) BENCHMARK_INTERNAL_TOSTRING2(x)
228
229 #if defined(__GNUC__)
230 #define BENCHMARK_BUILTIN_EXPECT(x, y) __builtin_expect(x, y)
231 #define BENCHMARK_DEPRECATED_MSG(msg) __attribute__((deprecated(msg)))
232 #else
233 #define BENCHMARK_BUILTIN_EXPECT(x, y) x
234 #define BENCHMARK_DEPRECATED_MSG(msg)
235 #define BENCHMARK_WARNING_MSG(msg) __pragma(message(__FILE__ "(" BENCHMARK_INTERNAL_TOSTRING(__LINE__) ") : warning note: " msg))
236 #endif
237
238 #if defined(__GNUC__) && !defined(__clang__)
239 #define BENCHMARK_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
240 #endif
241
242 namespace benchmark {
243 class BenchmarkReporter;
244
245 void Initialize(int* argc, char** argv);
246
247 // Report to stdout all arguments in 'argv' as unrecognized except the first.
248 // Returns true there is at least on unrecognized argument (i.e. 'argc' > 1).
249 bool ReportUnrecognizedArguments(int argc, char** argv);
250
251 // Generate a list of benchmarks matching the specified --benchmark_filter flag
252 // and if --benchmark_list_tests is specified return after printing the name
253 // of each matching benchmark. Otherwise run each matching benchmark and
254 // report the results.
255 //
256 // The second and third overload use the specified 'console_reporter' and
257 // 'file_reporter' respectively. 'file_reporter' will write to the file
258 // specified
259 // by '--benchmark_output'. If '--benchmark_output' is not given the
260 // 'file_reporter' is ignored.
261 //
262 // RETURNS: The number of matching benchmarks.
263 size_t RunSpecifiedBenchmarks();
264 size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter);
265 size_t RunSpecifiedBenchmarks(BenchmarkReporter* console_reporter,
266 BenchmarkReporter* file_reporter);
267
268 // If this routine is called, peak memory allocation past this point in the
269 // benchmark is reported at the end of the benchmark report line. (It is
270 // computed by running the benchmark once with a single iteration and a memory
271 // tracer.)
272 // TODO(dominic)
273 // void MemoryUsage();
274
275 namespace internal {
276 class Benchmark;
277 class BenchmarkImp;
278 class BenchmarkFamilies;
279
280 void UseCharPointer(char const volatile*);
281
282 // Take ownership of the pointer and register the benchmark. Return the
283 // registered benchmark.
284 Benchmark* RegisterBenchmarkInternal(Benchmark*);
285
286 // Ensure that the standard streams are properly initialized in every TU.
287 int InitializeStreams();
288 BENCHMARK_UNUSED static int stream_init_anchor = InitializeStreams();
289
290 } // namespace internal
291
292
293 #if (!defined(__GNUC__) && !defined(__clang__)) || defined(__pnacl__) || \
294 defined(__EMSCRIPTEN__)
295 # define BENCHMARK_HAS_NO_INLINE_ASSEMBLY
296 #endif
297
298
299 // The DoNotOptimize(...) function can be used to prevent a value or
300 // expression from being optimized away by the compiler. This function is
301 // intended to add little to no overhead.
302 // See: https://youtu.be/nXaxk27zwlk?t=2441
303 #ifndef BENCHMARK_HAS_NO_INLINE_ASSEMBLY
304 template
305 inline BENCHMARK_ALWAYS_INLINE
306 void DoNotOptimize(Tp const& value) {
307 asm volatile("" : : "r,m"(value) : "memory");
308 }
309
310 template
311 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp& value) {
312 #if defined(__clang__)
313 asm volatile("" : "+r,m"(value) : : "memory");
314 #else
315 asm volatile("" : "+m,r"(value) : : "memory");
316 #endif
317 }
318
319 // Force the compiler to flush pending writes to global memory. Acts as an
320 // effective read/write barrier
321 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() {
322 asm volatile("" : : : "memory");
323 }
324 #elif defined(_MSC_VER)
325 template
326 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {
327 internal::UseCharPointer(&reinterpret_cast(value));
328 _ReadWriteBarrier();
329 }
330
331 inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() {
332 _ReadWriteBarrier();
333 }
334 #else
335 template
336 inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) {
337 internal::UseCharPointer(&reinterpret_cast(value));
338 }
339 // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers
340 #endif
341
342
343
344 // This class is used for user-defined counters.
345 class Counter {
346 public:
347
348 enum Flags {
349 kDefaults = 0,
350 // Mark the counter as a rate. It will be presented divided
351 // by the duration of the benchmark.
352 kIsRate = 1,
353 // Mark the counter as a thread-average quantity. It will be
354 // presented divided by the number of threads.
355 kAvgThreads = 2,
356 // Mark the counter as a thread-average rate. See above.
357 kAvgThreadsRate = kIsRate|kAvgThreads
358 };
359
360 double value;
361 Flags flags;
362
363 BENCHMARK_ALWAYS_INLINE
364 Counter(double v = 0., Flags f = kDefaults) : value(v), flags(f) {}
365
366 BENCHMARK_ALWAYS_INLINE operator double const& () const { return value; }
367 BENCHMARK_ALWAYS_INLINE operator double & () { return value; }
368
369 };
370
371 // This is the container for the user-defined counters.
372 typedef std::map UserCounters;
373
374
375 // TimeUnit is passed to a benchmark in order to specify the order of magnitude
376 // for the measured time.
377 enum TimeUnit { kNanosecond, kMicrosecond, kMillisecond };
378
379 // BigO is passed to a benchmark in order to specify the asymptotic
380 // computational
381 // complexity for the benchmark. In case oAuto is selected, complexity will be
382 // calculated automatically to the best fit.
383 enum BigO { oNone, o1, oN, oNSquared, oNCubed, oLogN, oNLogN, oAuto, oLambda };
384
385 // BigOFunc is passed to a benchmark in order to specify the asymptotic
386 // computational complexity for the benchmark.
387 typedef double(BigOFunc)(int64_t);
388
389 // StatisticsFunc is passed to a benchmark in order to compute some descriptive
390 // statistics over all the measurements of some type
391 typedef double(StatisticsFunc)(const std::vector&);
392
393 struct Statistics {
394 std::string name_;
395 StatisticsFunc* compute_;
396
397 Statistics(std::string name, StatisticsFunc* compute)
398 : name_(name), compute_(compute) {}
399 };
400
401 namespace internal {
402 class ThreadTimer;
403 class ThreadManager;
404
405 enum ReportMode
406 #if defined(BENCHMARK_HAS_CXX11)
407 : unsigned
408 #else
409 #endif
410 {
411 RM_Unspecified, // The mode has not been manually specified
412 RM_Default, // The mode is user-specified as default.
413 RM_ReportAggregatesOnly
414 };
415 } // namespace internal
416
417 // State is passed to a running Benchmark and contains state for the
418 // benchmark to use.
419 class State {
420 public:
421 struct StateIterator;
422 friend struct StateIterator;
423
424 // Returns iterators used to run each iteration of a benchmark using a
425 // C++11 ranged-based for loop. These functions should not be called directly.
426 //
427 // REQUIRES: The benchmark has not started running yet. Neither begin nor end
428 // have been called previously.
429 //
430 // NOTE: KeepRunning may not be used after calling either of these functions.
431 BENCHMARK_ALWAYS_INLINE StateIterator begin();
432 BENCHMARK_ALWAYS_INLINE StateIterator end();
433
434 // Returns true if the benchmark should continue through another iteration.
435 // NOTE: A benchmark may not return from the test until KeepRunning() has
436 // returned false.
437 bool KeepRunning();
438
439 // Returns true iff the benchmark should run n more iterations.
440 // REQUIRES: 'n' > 0.
441 // NOTE: A benchmark must not return from the test until KeepRunningBatch()
442 // has returned false.
443 // NOTE: KeepRunningBatch() may overshoot by up to 'n' iterations.
444 //
445 // Intended usage:
446 // while (state.KeepRunningBatch(1000)) {
447 // // process 1000 elements
448 // }
449 bool KeepRunningBatch(size_t n);
450
451 // REQUIRES: timer is running and 'SkipWithError(...)' has not been called
452 // by the current thread.
453 // Stop the benchmark timer. If not called, the timer will be
454 // automatically stopped after the last iteration of the benchmark loop.
455 //
456 // For threaded benchmarks the PauseTiming() function only pauses the timing
457 // for the current thread.
458 //
459 // NOTE: The "real time" measurement is per-thread. If different threads
460 // report different measurements the largest one is reported.
461 //
462 // NOTE: PauseTiming()/ResumeTiming() are relatively
463 // heavyweight, and so their use should generally be avoided
464 // within each benchmark iteration, if possible.
465 void PauseTiming();
466
467 // REQUIRES: timer is not running and 'SkipWithError(...)' has not been called
468 // by the current thread.
469 // Start the benchmark timer. The timer is NOT running on entrance to the
470 // benchmark function. It begins running after control flow enters the
471 // benchmark loop.
472 //
473 // NOTE: PauseTiming()/ResumeTiming() are relatively
474 // heavyweight, and so their use should generally be avoided
475 // within each benchmark iteration, if possible.
476 void ResumeTiming();
477
478 // REQUIRES: 'SkipWithError(...)' has not been called previously by the
479 // current thread.
480 // Report the benchmark as resulting in an error with the specified 'msg'.
481 // After this call the user may explicitly 'return' from the benchmark.
482 //
483 // If the ranged-for style of benchmark loop is used, the user must explicitly
484 // break from the loop, otherwise all future iterations will be run.
485 // If the 'KeepRunning()' loop is used the current thread will automatically
486 // exit the loop at the end of the current iteration.
487 //
488 // For threaded benchmarks only the current thread stops executing and future
489 // calls to `KeepRunning()` will block until all threads have completed
490 // the `KeepRunning()` loop. If multiple threads report an error only the
491 // first error message is used.
492 //
493 // NOTE: Calling 'SkipWithError(...)' does not cause the benchmark to exit
494 // the current scope immediately. If the function is called from within
495 // the 'KeepRunning()' loop the current iteration will finish. It is the users
496 // responsibility to exit the scope as needed.
497 void SkipWithError(const char* msg);
498
499 // REQUIRES: called exactly once per iteration of the benchmarking loop.
500 // Set the manually measured time for this benchmark iteration, which
501 // is used instead of automatically measured time if UseManualTime() was
502 // specified.
503 //
504 // For threaded benchmarks the final value will be set to the largest
505 // reported values.
506 void SetIterationTime(double seconds);
507
508 // Set the number of bytes processed by the current benchmark
509 // execution. This routine is typically called once at the end of a
510 // throughput oriented benchmark. If this routine is called with a
511 // value > 0, the report is printed in MB/sec instead of nanoseconds
512 // per iteration.
513 //
514 // REQUIRES: a benchmark has exited its benchmarking loop.
515 BENCHMARK_ALWAYS_INLINE
516 void SetBytesProcessed(int64_t bytes) { bytes_processed_ = bytes; }
517
518 BENCHMARK_ALWAYS_INLINE
519 int64_t bytes_processed() const { return bytes_processed_; }
520
521 // If this routine is called with complexity_n > 0 and complexity report is
522 // requested for the
523 // family benchmark, then current benchmark will be part of the computation
524 // and complexity_n will
525 // represent the length of N.
526 BENCHMARK_ALWAYS_INLINE
527 void SetComplexityN(int64_t complexity_n) { complexity_n_ = complexity_n; }
528
529 BENCHMARK_ALWAYS_INLINE
530 int64_t complexity_length_n() { return complexity_n_; }
531
532 // If this routine is called with items > 0, then an items/s
533 // label is printed on the benchmark report line for the currently
534 // executing benchmark. It is typically called at the end of a processing
535 // benchmark where a processing items/second output is desired.
536 //
537 // REQUIRES: a benchmark has exited its benchmarking loop.
538 BENCHMARK_ALWAYS_INLINE
539 void SetItemsProcessed(int64_t items) { items_processed_ = items; }
540
541 BENCHMARK_ALWAYS_INLINE
542 int64_t items_processed() const { return items_processed_; }
543
544 // If this routine is called, the specified label is printed at the
545 // end of the benchmark report line for the currently executing
546 // benchmark. Example:
547 // static void BM_Compress(benchmark::State& state) {
548 // ...
549 // double compress = input_size / output_size;
550 // state.SetLabel(StrFormat("compress:%.1f%%", 100.0*compression));
551 // }
552 // Produces output that looks like:
553 // BM_Compress 50 50 14115038 compress:27.3%
554 //
555 // REQUIRES: a benchmark has exited its benchmarking loop.
556 void SetLabel(const char* label);
557
558 void BENCHMARK_ALWAYS_INLINE SetLabel(const std::string& str) {
559 this->SetLabel(str.c_str());
560 }
561
562 // Range arguments for this run. CHECKs if the argument has been set.
563 BENCHMARK_ALWAYS_INLINE
564 int64_t range(std::size_t pos = 0) const {
565 assert(range_.size() > pos);
566 return range_[pos];
567 }
568
569 BENCHMARK_DEPRECATED_MSG("use 'range(0)' instead")
570 int64_t range_x() const { return range(0); }
571
572 BENCHMARK_DEPRECATED_MSG("use 'range(1)' instead")
573 int64_t range_y() const { return range(1); }
574
575 BENCHMARK_ALWAYS_INLINE
576 size_t iterations() const {
577 if (BENCHMARK_BUILTIN_EXPECT(!started_, false)) {
578 return 0;
579 }
580 return max_iterations - total_iterations_ + batch_leftover_;
581 }
582
583 private: // items we expect on the first cache line (ie 64 bytes of the struct)
584
585 // When total_iterations_ is 0, KeepRunning() and friends will return false.
586 // May be larger than max_iterations.
587 size_t total_iterations_;
588
589 // When using KeepRunningBatch(), batch_leftover_ holds the number of
590 // iterations beyond max_iters that were run. Used to track
591 // completed_iterations_ accurately.
592 size_t batch_leftover_;
593
594 public:
595 const size_t max_iterations;
596
597 private:
598 bool started_;
599 bool finished_;
600 bool error_occurred_;
601
602 private: // items we don't need on the first cache line
603 std::vector range_;
604
605 int64_t bytes_processed_;
606 int64_t items_processed_;
607
608 int64_t complexity_n_;
609
610 public:
611 // Container for user-defined counters.
612 UserCounters counters;
613 // Index of the executing thread. Values from [0, threads).
614 const int thread_index;
615 // Number of threads concurrently executing the benchmark.
616 const int threads;
617
618
619 // TODO(EricWF) make me private
620 State(size_t max_iters, const std::vector& ranges, int thread_i,
621 int n_threads, internal::ThreadTimer* timer,
622 internal::ThreadManager* manager);
623
624 private:
625 void StartKeepRunning();
626 // Implementation of KeepRunning() and KeepRunningBatch().
627 // is_batch must be true unless n is 1.
628 bool KeepRunningInternal(size_t n, bool is_batch);
629 void FinishKeepRunning();
630 internal::ThreadTimer* timer_;
631 internal::ThreadManager* manager_;
632 BENCHMARK_DISALLOW_COPY_AND_ASSIGN(State);
633 };
634
635 inline BENCHMARK_ALWAYS_INLINE
636 bool State::KeepRunning() {
637 return KeepRunningInternal(1, /*is_batch=*/ false);
638 }
639
640 inline BENCHMARK_ALWAYS_INLINE
641 bool State::KeepRunningBatch(size_t n) {
642 return KeepRunningInternal(n, /*is_batch=*/ true);
643 }
644
645 inline BENCHMARK_ALWAYS_INLINE
646 bool State::KeepRunningInternal(size_t n, bool is_batch) {
647 // total_iterations_ is set to 0 by the constructor, and always set to a
648 // nonzero value by StartKepRunning().
649 assert(n > 0);
650 // n must be 1 unless is_batch is true.
651 assert(is_batch || n == 1);
652 if (BENCHMARK_BUILTIN_EXPECT(total_iterations_ >= n, true)) {
653 total_iterations_ -= n;
654 return true;
655 }
656 if (!started_) {
657 StartKeepRunning();
658 if (!error_occurred_ && total_iterations_ >= n) {
659 total_iterations_-= n;
660 return true;
661 }
662 }
663 // For non-batch runs, total_iterations_ must be 0 by now.
664 if (is_batch && total_iterations_ != 0) {
665 batch_leftover_ = n - total_iterations_;
666 total_iterations_ = 0;
667 return true;
668 }
669 FinishKeepRunning();
670 return false;
671 }
672
673 struct State::StateIterator {
674 struct BENCHMARK_UNUSED Value {};
675 typedef std::forward_iterator_tag iterator_category;
676 typedef Value value_type;
677 typedef Value reference;
678 typedef Value pointer;
679 typedef std::ptrdiff_t difference_type;
680
681 private:
682 friend class State;
683 BENCHMARK_ALWAYS_INLINE
684 StateIterator() : cached_(0), parent_() {}
685
686 BENCHMARK_ALWAYS_INLINE
687 explicit StateIterator(State* st)
688 : cached_(st->error_occurred_ ? 0 : st->max_iterations), parent_(st) {}
689
690 public:
691 BENCHMARK_ALWAYS_INLINE
692 Value operator*() const { return Value(); }
693
694 BENCHMARK_ALWAYS_INLINE
695 StateIterator& operator++() {
696 assert(cached_ > 0);
697 --cached_;
698 return *this;
699 }
700
701 BENCHMARK_ALWAYS_INLINE
702 bool operator!=(StateIterator const&) const {
703 if (BENCHMARK_BUILTIN_EXPECT(cached_ != 0, true)) return true;
704 parent_->FinishKeepRunning();
705 return false;
706 }
707
708 private:
709 size_t cached_;
710 State* const parent_;
711 };
712
713 inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::begin() {
714 return StateIterator(this);
715 }
716 inline BENCHMARK_ALWAYS_INLINE State::StateIterator State::end() {
717 StartKeepRunning();
718 return StateIterator();
719 }
720
721 namespace internal {
722
723 typedef void(Function)(State&);
724
725 // ------------------------------------------------------
726 // Benchmark registration object. The BENCHMARK() macro expands
727 // into an internal::Benchmark* object. Various methods can
728 // be called on this object to change the properties of the benchmark.
729 // Each method returns "this" so that multiple method calls can
730 // chained into one expression.
731 class Benchmark {
732 public:
733 virtual ~Benchmark();
734
735 // Note: the following methods all return "this" so that multiple
736 // method calls can be chained together in one expression.
737
738 // Run this benchmark once with "x" as the extra argument passed
739 // to the function.
740 // REQUIRES: The function passed to the constructor must accept an arg1.
741 Benchmark* Arg(int64_t x);
742
743 // Run this benchmark with the given time unit for the generated output report
744 Benchmark* Unit(TimeUnit unit);
745
746 // Run this benchmark once for a number of values picked from the
747 // range [start..limit]. (start and limit are always picked.)
748 // REQUIRES: The function passed to the constructor must accept an arg1.
749 Benchmark* Range(int64_t start, int64_t limit);
750
751 // Run this benchmark once for all values in the range [start..limit] with
752 // specific step
753 // REQUIRES: The function passed to the constructor must accept an arg1.
754 Benchmark* DenseRange(int64_t start, int64_t limit, int step = 1);
755
756 // Run this benchmark once with "args" as the extra arguments passed
757 // to the function.
758 // REQUIRES: The function passed to the constructor must accept arg1, arg2 ...
759 Benchmark* Args(const std::vector& args);
760
761 // Equivalent to Args({x, y})
762 // NOTE: This is a legacy C++03 interface provided for compatibility only.
763 // New code should use 'Args'.
764 Benchmark* ArgPair(int64_t x, int64_t y) {
765 std::vector args;
766 args.push_back(x);
767 args.push_back(y);
768 return Args(args);
769 }
770
771 // Run this benchmark once for a number of values picked from the
772 // ranges [start..limit]. (starts and limits are always picked.)
773 // REQUIRES: The function passed to the constructor must accept arg1, arg2 ...
774 Benchmark* Ranges(const std::vector >& ranges);
775
776 // Equivalent to ArgNames({name})
777 Benchmark* ArgName(const std::string& name);
778
779 // Set the argument names to display in the benchmark name. If not called,
780 // only argument values will be shown.
781 Benchmark* ArgNames(const std::vector& names);
782
783 // Equivalent to Ranges({{lo1, hi1}, {lo2, hi2}}).
784 // NOTE: This is a legacy C++03 interface provided for compatibility only.
785 // New code should use 'Ranges'.
786 Benchmark* RangePair(int64_t lo1, int64_t hi1, int64_t lo2, int64_t hi2) {
787 std::vector > ranges;
788 ranges.push_back(std::make_pair(lo1, hi1));
789 ranges.push_back(std::make_pair(lo2, hi2));
790 return Ranges(ranges);
791 }
792
793 // Pass this benchmark object to *func, which can customize
794 // the benchmark by calling various methods like Arg, Args,
795 // Threads, etc.
796 Benchmark* Apply(void (*func)(Benchmark* benchmark));
797
798 // Set the range multiplier for non-dense range. If not called, the range
799 // multiplier kRangeMultiplier will be used.
800 Benchmark* RangeMultiplier(int multiplier);
801
802 // Set the minimum amount of time to use when running this benchmark. This
803 // option overrides the `benchmark_min_time` flag.
804 // REQUIRES: `t > 0` and `Iterations` has not been called on this benchmark.
805 Benchmark* MinTime(double t);
806
807 // Specify the amount of iterations that should be run by this benchmark.
808 // REQUIRES: 'n > 0' and `MinTime` has not been called on this benchmark.
809 //
810 // NOTE: This function should only be used when *exact* iteration control is
811 // needed and never to control or limit how long a benchmark runs, where
812 // `--benchmark_min_time=N` or `MinTime(...)` should be used instead.
813 Benchmark* Iterations(size_t n);
814
815 // Specify the amount of times to repeat this benchmark. This option overrides
816 // the `benchmark_repetitions` flag.
817 // REQUIRES: `n > 0`
818 Benchmark* Repetitions(int n);
819
820 // Specify if each repetition of the benchmark should be reported separately
821 // or if only the final statistics should be reported. If the benchmark
822 // is not repeated then the single result is always reported.
823 Benchmark* ReportAggregatesOnly(bool value = true);
824
825 // If a particular benchmark is I/O bound, runs multiple threads internally or
826 // if for some reason CPU timings are not representative, call this method. If
827 // called, the elapsed time will be used to control how many iterations are
828 // run, and in the printing of items/second or MB/seconds values. If not
829 // called, the cpu time used by the