Commit 7a767012 by Eric Fiselier

Adopt new benchmark timing internals.

This patch adopts a new internal structure for how timings are performed. Currently every iteration of a benchmark checks to see if it has been running for an appropriate amount of time. Checking the clock introduces noise into the timings and this can cause inconsistent output from each benchmark. Now every iteration of a benchmark only checks an iteration count to see if it should stop running. The iteration count is determined before hand by testing the benchmark on a series of increasing iteration counts until a suitable count is found. This increases the amount of time it takes to run the actual benchmarks but it also greatly increases the accuracy of the results. This patch introduces some breaking changes. The notable breaking changes are: 1. Benchmarks run on multiple threads no generate a report per thread. Instead only a single report is generated. 2. ::benchmark::UseRealTime() was removed and replaced with State::UseRealTime().
parent 7c6a7e30
...@@ -44,6 +44,10 @@ add_cxx_compiler_flag(-pedantic-errors) ...@@ -44,6 +44,10 @@ add_cxx_compiler_flag(-pedantic-errors)
add_cxx_compiler_flag(-fno-strict-aliasing RELEASE) add_cxx_compiler_flag(-fno-strict-aliasing RELEASE)
add_cxx_compiler_flag(-Wthread-safety) add_cxx_compiler_flag(-Wthread-safety)
if (HAVE_WTHREAD_SAFETY)
add_definitions(-DHAVE_WTHREAD_SAFETY)
cxx_feature_check(THREAD_SAFETY_ATTRIBUTES)
endif()
# C++ feature checks # C++ feature checks
cxx_feature_check(STD_REGEX) cxx_feature_check(STD_REGEX)
......
#define HAVE_THREAD_SAFETY_ATTRIBUTES
#include "../src/mutex.h"
int main() {}
...@@ -2,9 +2,8 @@ ...@@ -2,9 +2,8 @@
include_directories(${PROJECT_SOURCE_DIR}/src) include_directories(${PROJECT_SOURCE_DIR}/src)
# Define the source files # Define the source files
set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" "log.cc"
"log.cc" "sleep.cc" "string_util.cc" "sysinfo.cc" "sleep.cc" "string_util.cc" "sysinfo.cc" "walltime.cc")
"walltime.cc")
# Determine the correct regular expression engine to use # Determine the correct regular expression engine to use
if(HAVE_STD_REGEX) if(HAVE_STD_REGEX)
set(RE_FILES "re_std.cc") set(RE_FILES "re_std.cc")
......
#ifndef BENCHMARK_MUTEX_H_
#define BENCHMARK_MUTEX_H_
#include <mutex>
#include <condition_variable>
// Enable thread safety attributes only with clang.
// The attributes can be safely erased when compiling with other compilers.
#if defined(HAVE_THREAD_SAFETY_ATTRIBUTES)
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
#else
#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
#endif
#define CAPABILITY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(capability(x))
#define SCOPED_CAPABILITY \
THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
#define GUARDED_BY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))
#define PT_GUARDED_BY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))
#define ACQUIRED_BEFORE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))
#define ACQUIRED_AFTER(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))
#define REQUIRES(...) \
THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__))
#define REQUIRES_SHARED(...) \
THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__))
#define ACQUIRE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__))
#define ACQUIRE_SHARED(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__))
#define RELEASE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__))
#define RELEASE_SHARED(...) \
THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__))
#define TRY_ACQUIRE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__))
#define TRY_ACQUIRE_SHARED(...) \
THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__))
#define EXCLUDES(...) \
THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__))
#define ASSERT_CAPABILITY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x))
#define ASSERT_SHARED_CAPABILITY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x))
#define RETURN_CAPABILITY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x))
#define NO_THREAD_SAFETY_ANALYSIS \
THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis)
namespace benchmark {
typedef std::condition_variable Condition;
// NOTE: Wrappers for std::mutex and std::unique_lock are provided so that
// we can annotate them with thread safety attributes and use the
// -Wthread-safety warning with clang. The standard library types cannot be
// used directly because they do not provided the required annotations.
class CAPABILITY("mutex") Mutex
{
public:
Mutex() {}
void lock() ACQUIRE() { mut_.lock(); }
void unlock() RELEASE() { mut_.unlock(); }
std::mutex& native_handle() {
return mut_;
}
private:
std::mutex mut_;
};
class SCOPED_CAPABILITY MutexLock
{
typedef std::unique_lock<std::mutex> MutexLockImp;
public:
MutexLock(Mutex& m) ACQUIRE(m) : ml_(m.native_handle())
{ }
~MutexLock() RELEASE() {}
MutexLockImp& native_handle() { return ml_; }
private:
MutexLockImp ml_;
};
class Notification
{
public:
Notification() : notified_yet_(false) { }
void WaitForNotification() const EXCLUDES(mutex_) {
MutexLock m_lock(mutex_);
auto notified_fn = [this]() REQUIRES(mutex_) {
return this->HasBeenNotified();
};
cv_.wait(m_lock.native_handle(), notified_fn);
}
void Notify() EXCLUDES(mutex_) {
{
MutexLock lock(mutex_);
notified_yet_ = 1;
}
cv_.notify_all();
}
private:
bool HasBeenNotified() const REQUIRES(mutex_) {
return notified_yet_;
}
mutable Mutex mutex_;
mutable std::condition_variable cv_;
bool notified_yet_ GUARDED_BY(mutex_);
};
} // end namespace benchmark
#endif // BENCHMARK_MUTEX_H_
...@@ -24,13 +24,13 @@ static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits), ...@@ -24,13 +24,13 @@ static_assert(arraysize(kBigSIUnits) == arraysize(kBigIECUnits),
static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits), static_assert(arraysize(kSmallSIUnits) == arraysize(kBigSIUnits),
"Small SI and Big SI unit arrays must be the same size"); "Small SI and Big SI unit arrays must be the same size");
static const int kUnitsSize = arraysize(kBigSIUnits); static const int64_t kUnitsSize = arraysize(kBigSIUnits);
} // end anonymous namespace } // end anonymous namespace
void ToExponentAndMantissa(double val, double thresh, int precision, void ToExponentAndMantissa(double val, double thresh, int precision,
double one_k, std::string* mantissa, double one_k, std::string* mantissa,
int* exponent) { int64_t* exponent) {
std::stringstream mantissa_stream; std::stringstream mantissa_stream;
if (val < 0) { if (val < 0) {
...@@ -80,10 +80,10 @@ void ToExponentAndMantissa(double val, double thresh, int precision, ...@@ -80,10 +80,10 @@ void ToExponentAndMantissa(double val, double thresh, int precision,
*mantissa = mantissa_stream.str(); *mantissa = mantissa_stream.str();
} }
std::string ExponentToPrefix(int exponent, bool iec) { std::string ExponentToPrefix(int64_t exponent, bool iec) {
if (exponent == 0) return ""; if (exponent == 0) return "";
const int index = (exponent > 0 ? exponent - 1 : -exponent - 1); const int64_t index = (exponent > 0 ? exponent - 1 : -exponent - 1);
if (index >= kUnitsSize) return ""; if (index >= kUnitsSize) return "";
const char* array = const char* array =
...@@ -97,7 +97,7 @@ std::string ExponentToPrefix(int exponent, bool iec) { ...@@ -97,7 +97,7 @@ std::string ExponentToPrefix(int exponent, bool iec) {
std::string ToBinaryStringFullySpecified(double value, double threshold, std::string ToBinaryStringFullySpecified(double value, double threshold,
int precision) { int precision) {
std::string mantissa; std::string mantissa;
int exponent; int64_t exponent;
ToExponentAndMantissa(value, threshold, precision, 1024.0, &mantissa, ToExponentAndMantissa(value, threshold, precision, 1024.0, &mantissa,
&exponent); &exponent);
return mantissa + ExponentToPrefix(exponent, false); return mantissa + ExponentToPrefix(exponent, false);
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
#include "check.h" #include "check.h"
#include "cycleclock.h" #include "cycleclock.h"
#include "internal_macros.h" #include "internal_macros.h"
#include "log.h"
#include "sleep.h" #include "sleep.h"
namespace benchmark { namespace benchmark {
...@@ -322,7 +323,7 @@ double MyCPUUsage() { ...@@ -322,7 +323,7 @@ double MyCPUUsage() {
return value; return value;
} }
// Once MyCPUUsageCPUTimeNsLocked fails once fall back to getrusage(). // Once MyCPUUsageCPUTimeNsLocked fails once fall back to getrusage().
std::cout << "Reading /proc/self/cputime_ns failed. Using getrusage().\n"; VLOG(1) << "Reading /proc/self/cputime_ns failed. Using getrusage().\n";
use_cputime_ns = false; use_cputime_ns = false;
} }
} }
......
...@@ -20,3 +20,6 @@ add_test(filter_regex_none filter_test --benchmark_filter=monkey 0) ...@@ -20,3 +20,6 @@ add_test(filter_regex_none filter_test --benchmark_filter=monkey 0)
add_test(filter_regex_wildcard filter_test --benchmark_filter=.*Calculate.* 16) add_test(filter_regex_wildcard filter_test --benchmark_filter=.*Calculate.* 16)
add_test(filter_regex_begin filter_test --benchmark_filter=^BM_Calculate.* 16) add_test(filter_regex_begin filter_test --benchmark_filter=^BM_Calculate.* 16)
add_test(filter_regex_end filter_test --benchmark_filter=.*Pi$ 8) add_test(filter_regex_end filter_test --benchmark_filter=.*Pi$ 8)
compile_benchmark_test(basic_test)
add_test(basic basic_test)
#include <cstddef>
#include "benchmark/benchmark.h"
#define BASIC_BENCHMARK_TEST(x) \
BENCHMARK(x)->Arg(8)->Arg(512)->Arg(8192)
void BM_empty(benchmark::State& state) {
while (state.KeepRunning()) {
volatile std::size_t x = state.iterations();
((void)x);
}
}
BENCHMARK(BM_empty);
BENCHMARK(BM_empty)->ThreadPerCpu();
void BM_spin_empty(benchmark::State& state) {
while (state.KeepRunning()) {
for (int x = 0; x < state.range_x(); ++x) {
volatile int dummy = x;
((void)dummy);
}
}
}
BASIC_BENCHMARK_TEST(BM_spin_empty);
BASIC_BENCHMARK_TEST(BM_spin_empty)->ThreadPerCpu();
void BM_spin_pause_before(benchmark::State& state) {
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
while(state.KeepRunning()) {
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
}
BASIC_BENCHMARK_TEST(BM_spin_pause_before);
BASIC_BENCHMARK_TEST(BM_spin_pause_before)->ThreadPerCpu();
void BM_spin_pause_during(benchmark::State& state) {
while(state.KeepRunning()) {
state.PauseTiming();
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
state.ResumeTiming();
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
}
BASIC_BENCHMARK_TEST(BM_spin_pause_during);
BASIC_BENCHMARK_TEST(BM_spin_pause_during)->ThreadPerCpu();
void BM_spin_pause_after(benchmark::State& state) {
while(state.KeepRunning()) {
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
BASIC_BENCHMARK_TEST(BM_spin_pause_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_after)->ThreadPerCpu();
void BM_spin_pause_before_and_after(benchmark::State& state) {
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
while(state.KeepRunning()) {
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
for (int i = 0; i < state.range_x(); ++i) {
volatile int dummy = i;
((void)dummy);
}
}
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after);
BASIC_BENCHMARK_TEST(BM_spin_pause_before_and_after)->ThreadPerCpu();
void BM_empty_stop_start(benchmark::State& state) {
while (state.KeepRunning()) { }
}
BENCHMARK(BM_empty_stop_start);
BENCHMARK(BM_empty_stop_start)->ThreadPerCpu();
BENCHMARK_MAIN()
...@@ -53,18 +53,22 @@ static void BM_Factorial(benchmark::State& state) { ...@@ -53,18 +53,22 @@ static void BM_Factorial(benchmark::State& state) {
while (state.KeepRunning()) while (state.KeepRunning())
fac_42 = Factorial(8); fac_42 = Factorial(8);
// Prevent compiler optimizations // Prevent compiler optimizations
std::cout << fac_42; std::stringstream ss;
ss << fac_42;
state.SetLabel(ss.str());
} }
BENCHMARK(BM_Factorial); BENCHMARK(BM_Factorial);
static void BM_FactorialRealTime(benchmark::State& state) { static void BM_FactorialRealTime(benchmark::State& state) {
benchmark::UseRealTime(); state.UseRealTime();
int fac_42 = 0; int fac_42 = 0;
while (state.KeepRunning()) while (state.KeepRunning())
fac_42 = Factorial(8); fac_42 = Factorial(8);
// Prevent compiler optimizations // Prevent compiler optimizations
std::cout << fac_42; std::stringstream ss;
ss << fac_42;
state.SetLabel(ss.str());
} }
BENCHMARK(BM_FactorialRealTime); BENCHMARK(BM_FactorialRealTime);
...@@ -158,12 +162,5 @@ static void BM_LongTest(benchmark::State& state) { ...@@ -158,12 +162,5 @@ static void BM_LongTest(benchmark::State& state) {
} }
BENCHMARK(BM_LongTest)->Range(1<<16,1<<28); BENCHMARK(BM_LongTest)->Range(1<<16,1<<28);
int main(int argc, const char* argv[]) { BENCHMARK_MAIN()
benchmark::Initialize(&argc, argv);
assert(Factorial(8) == 40320);
assert(CalculatePi(1) == 0.0);
benchmark::RunSpecifiedBenchmarks();
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment