Try to reduce variance in angle_perftests.

scripts/perf_test_runner.py

@@ -6,8 +6,8 @@
 #
 # perf_test_runner.py:
 #   Helper script for running and analyzing perftest results. Runs the
-#   tests in an infinite batch, printing out the mean and standard
-#   deviation of the population continuously.
+#   tests in an infinite batch, printing out the mean and coefficient of
+#   variation of the population continuously.
 #
 
 import glob
@@ -36,20 +36,21 @@ def mean(data):
         raise ValueError('mean requires at least one data point')
     return float(sum(data))/float(n) # in Python 2 use sum(data)/float(n)
 
-def _ss(data):
+def sum_of_square_deviations(data, c):
     """Return sum of square deviations of sequence data."""
-    c = mean(data)
     ss = sum((float(x)-c)**2 for x in data)
     return ss
 
-def pstdev(data):
-    """Calculates the population standard deviation."""
+def coefficient_of_variation(data):
+    """Calculates the population coefficient of variation."""
     n = len(data)
     if n < 2:
         raise ValueError('variance requires at least two data points')
-    ss = _ss(data)
+    c = mean(data)
+    ss = sum_of_square_deviations(data, c)
     pvar = ss/n # the population variance
-    return pvar**0.5
+    stddev = (pvar**0.5) # population standard deviation
+    return stddev / c
 
 def truncated_list(data, n):
     """Compute a truncated list, n is truncation size"""
@@ -58,10 +59,12 @@ def truncated_list(data, n):
     return sorted(data)[n:-n]
 
 def truncated_mean(data, n):
+    """Compute a truncated mean, n is truncation size"""
     return mean(truncated_list(data, n))
 
-def truncated_stddev(data, n):
-    return pstdev(truncated_list(data, n))
+def truncated_cov(data, n):
+    """Compute a truncated coefficient of variation, n is truncation size"""
+    return coefficient_of_variation(truncated_list(data, n))
 
 # Find most recent binary
 newest_binary = None
@@ -123,14 +126,14 @@ while True:
     sys.stdout.write("score: " + str(score))
 
     scores.append(score)
-    sys.stdout.write(", mean: " + str(mean(scores)))
+    sys.stdout.write(", mean: %.2f" % mean(scores))
 
     if (len(scores) > 1):
-        sys.stdout.write(", stddev: " + str(pstdev(scores)))
+        sys.stdout.write(", variation: %.2f%%" % (coefficient_of_variation(scores) * 100.0))
 
     if (len(scores) > 7):
         trucation_n = len(scores) >> 3
-        sys.stdout.write(", truncated mean: " + str(truncated_mean(scores, trucation_n)))
-        sys.stdout.write(", stddev: " + str(truncated_stddev(scores, trucation_n)))
+        sys.stdout.write(", truncated mean: %.2f" % truncated_mean(scores, trucation_n))
+        sys.stdout.write(", variation: %.2f%%" % (truncated_cov(scores, trucation_n) * 100.0))
 
     print("")

src/tests/BUILD.gn

@@ -126,7 +126,6 @@ if (is_win || is_linux || is_mac || is_android) {
   test("angle_end2end_tests") {
     include_dirs = [
       "../../src/tests",
-      "../../util",
     ]
     defines = [
       "ANGLE_EGL_LIBRARY_NAME=\"libEGL${angle_libs_suffix}\"",
@@ -187,7 +186,6 @@ if (is_win || is_linux || is_mac || is_android) {
   test("angle_white_box_tests") {
     include_dirs = [
       "../../src/tests",
-      "../../util",
     ]
 
     if (is_android) {
@@ -240,7 +238,6 @@ if (is_win || is_linux || is_android || is_mac) {
   test("angle_perftests") {
     include_dirs = [
       "../../src/tests",
-      "../../util",
     ]
 
     sources = angle_perf_tests_sources
@@ -471,7 +468,6 @@ if (build_angle_gles1_conform_tests) {
 
     include_dirs = [
       "../../src/tests",
-      "../../util",
     ]
 
     sources = [

src/tests/perf_tests/ANGLEPerfTest.cpp

@@ -8,8 +8,10 @@
 //
 
 #include "ANGLEPerfTest.h"
+
 #include "third_party/perf/perf_test.h"
 #include "third_party/trace_event/trace_event.h"
+#include "system_utils.h"
 
 #include <cassert>
 #include <cmath>
@@ -21,6 +23,9 @@
 namespace
 {
 constexpr size_t kInitialTraceEventBufferSize = 50000;
+constexpr size_t kWarmupIterations            = 3;
+constexpr double kMicroSecondsPerSecond       = 1e6;
+constexpr double kNanoSecondsPerSecond        = 1e9;
 
 struct TraceCategory
 {
@@ -143,13 +148,16 @@ bool g_OnlyOneRunFrame = false;
 bool gEnableTrace      = false;
 const char *gTraceFile = "ANGLETrace.json";
 
-ANGLEPerfTest::ANGLEPerfTest(const std::string &name, const std::string &suffix)
+ANGLEPerfTest::ANGLEPerfTest(const std::string &name,
+                             const std::string &suffix,
+                             unsigned int iterationsPerStep)
     : mName(name),
       mSuffix(suffix),
       mTimer(CreateTimer()),
-      mRunTimeSeconds(5.0),
+      mRunTimeSeconds(2.0),
       mSkipTest(false),
       mNumStepsPerformed(0),
+      mIterationsPerStep(iterationsPerStep),
       mRunning(true)
 {
 }
@@ -203,7 +211,25 @@ void ANGLEPerfTest::TearDown()
     {
         return;
     }
-    double relativeScore = static_cast<double>(mNumStepsPerformed) / mTimer->getElapsedTime();
+
+    double elapsedTimeSeconds = mTimer->getElapsedTime();
+
+    double secondsPerStep      = elapsedTimeSeconds / static_cast<double>(mNumStepsPerformed);
+    double secondsPerIteration = secondsPerStep / static_cast<double>(mIterationsPerStep);
+
+    // Give the result a different name to ensure separate graphs if we transition.
+    if (secondsPerIteration > 1e-3)
+    {
+        double microSecondsPerIteration = secondsPerIteration * kMicroSecondsPerSecond;
+        printResult("microSecPerIteration", microSecondsPerIteration, "us", true);
+    }
+    else
+    {
+        double nanoSecPerIteration = secondsPerIteration * kNanoSecondsPerSecond;
+        printResult("nanoSecPerIteration", nanoSecPerIteration, "ns", true);
+    }
+
+    double relativeScore = static_cast<double>(mNumStepsPerformed) / elapsedTimeSeconds;
     printResult("score", static_cast<size_t>(std::round(relativeScore)), "score", true);
 }
 
@@ -235,7 +261,7 @@ std::string RenderTestParams::suffix() const
 }
 
 ANGLERenderTest::ANGLERenderTest(const std::string &name, const RenderTestParams &testParams)
-    : ANGLEPerfTest(name, testParams.suffix()),
+    : ANGLEPerfTest(name, testParams.suffix(), testParams.iterationsPerStep),
       mTestParams(testParams),
       mEGLWindow(createEGLWindow(testParams)),
       mOSWindow(nullptr)
@@ -259,6 +285,9 @@ void ANGLERenderTest::SetUp()
 {
     ANGLEPerfTest::SetUp();
 
+    // Set a consistent CPU core affinity and high priority.
+    angle::StabilizeCPUForBenchmarking();
+
     mOSWindow = CreateOSWindow();
     ASSERT(mEGLWindow != nullptr);
     mEGLWindow->setSwapInterval(0);
@@ -293,11 +322,23 @@ void ANGLERenderTest::SetUp()
 
     if (mSkipTest)
     {
-        std::cout << "Test skipped due to missing extension." << std::endl;
         return;
     }
 
     initializeBenchmark();
+
+    if (mTestParams.iterationsPerStep == 0)
+    {
+        FAIL() << "Please initialize 'iterationsPerStep'.";
+        abortTest();
+        return;
+    }
+
+    // Warm up the benchmark to reduce variance.
+    for (size_t iteration = 0; iteration < kWarmupIterations; ++iteration)
+    {
+        drawBenchmark();
+    }
 }
 
 void ANGLERenderTest::TearDown()

src/tests/perf_tests/ANGLEPerfTest.h

@@ -58,7 +58,9 @@ struct TraceEvent final
 class ANGLEPerfTest : public testing::Test, angle::NonCopyable
 {
   public:
-    ANGLEPerfTest(const std::string &name, const std::string &suffix);
+    ANGLEPerfTest(const std::string &name,
+                  const std::string &suffix,
+                  unsigned int iterationsPerStep);
     virtual ~ANGLEPerfTest();
 
     virtual void step() = 0;
@@ -91,6 +93,7 @@ class ANGLEPerfTest : public testing::Test, angle::NonCopyable
 
   private:
     unsigned int mNumStepsPerformed;
+    unsigned int mIterationsPerStep;
     bool mRunning;
 };
 
@@ -100,6 +103,7 @@ struct RenderTestParams : public angle::PlatformParameters
 
     EGLint windowWidth  = 64;
     EGLint windowHeight = 64;
+    unsigned int iterationsPerStep = 0;
 };
 
 class ANGLERenderTest : public ANGLEPerfTest

src/tests/perf_tests/BindingPerf.cpp

@@ -17,6 +17,7 @@
 
 namespace angle
 {
+constexpr unsigned int kIterationsPerStep = 128;
 
 enum AllocationStyle
 {
@@ -33,18 +34,15 @@ struct BindingsParams final : public RenderTestParams
         minorVersion = 0;
         windowWidth  = 720;
         windowHeight = 720;
-        iterations   = 128;
 
         numObjects      = 100;
         allocationStyle = EVERY_ITERATION;
+        iterationsPerStep = kIterationsPerStep;
     }
 
     std::string suffix() const override;
     size_t numObjects;
     AllocationStyle allocationStyle;
-
-    // static parameters
-    size_t iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const BindingsParams &params)
@@ -98,7 +96,6 @@ BindingsBenchmark::BindingsBenchmark() : ANGLERenderTest("Bindings", GetParam())
 void BindingsBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
-    ASSERT_GT(params.iterations, 0u);
 
     mBuffers.resize(params.numObjects, 0);
     if (params.allocationStyle == AT_INITIALIZATION)
@@ -144,7 +141,7 @@ void BindingsBenchmark::drawBenchmark()
 {
     const auto &params = GetParam();
 
-    for (size_t it = 0; it < params.iterations; ++it)
+    for (unsigned int it = 0; it < params.iterationsPerStep; ++it)
     {
         // Generate a buffer (if needed) and bind it to a "random" binding point
         if (params.allocationStyle == EVERY_ITERATION)

src/tests/perf_tests/BitSetIteratorPerf.cpp

@@ -31,7 +31,7 @@ class BitSetIteratorPerfTest : public ANGLEPerfTest
 };
 
 template <typename T>
-BitSetIteratorPerfTest<T>::BitSetIteratorPerfTest() : ANGLEPerfTest("BitSetIteratorPerf", "_run")
+BitSetIteratorPerfTest<T>::BitSetIteratorPerfTest() : ANGLEPerfTest("BitSetIteratorPerf", "_run", 1)
 {
 }
 

src/tests/perf_tests/BlitFramebufferPerf.cpp

@@ -12,6 +12,7 @@
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 5;
 
 enum class BufferType
 {
@@ -93,6 +94,7 @@ struct BlitFramebufferParams final : public RenderTestParams
 {
     BlitFramebufferParams()
     {
+        iterationsPerStep = kIterationsPerStep;
         majorVersion = 3;
         minorVersion = 0;
         windowWidth  = 256;
@@ -114,7 +116,6 @@ struct BlitFramebufferParams final : public RenderTestParams
     BufferType type              = BufferType::COLOR;
     unsigned int framebufferSize = 512;
     unsigned int samples         = 0;
-    unsigned int iterations      = 5;
 };
 
 std::ostream &operator<<(std::ostream &os, const BlitFramebufferParams &params)
@@ -216,7 +217,7 @@ void BlitFramebufferPerf::drawBenchmark()
             break;
     }
 
-    for (unsigned int iteration = 0; iteration < param.iterations; ++iteration)
+    for (unsigned int iteration = 0; iteration < param.iterationsPerStep; ++iteration)
     {
         glBlitFramebuffer(0, 0, size, size, 0, 0, size, size, mask, GL_NEAREST);
     }

src/tests/perf_tests/BufferSubData.cpp

@@ -16,6 +16,7 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 4;
 
 struct BufferSubDataParams final : public RenderTestParams
 {
@@ -28,7 +29,7 @@ struct BufferSubDataParams final : public RenderTestParams
         windowHeight = 512;
         updateSize = 3000;
         bufferSize = 40000000;
-        iterations   = 4;
+        iterationsPerStep = kIterationsPerStep;
         updateRate = 1;
     }
 
@@ -42,7 +43,6 @@ struct BufferSubDataParams final : public RenderTestParams
     // static parameters
     GLsizeiptr updateSize;
     GLsizeiptr bufferSize;
-    unsigned int iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const BufferSubDataParams &params)
@@ -267,7 +267,6 @@ void BufferSubDataBenchmark::initializeBenchmark()
     const auto &params = GetParam();
 
     ASSERT_LT(1, params.vertexComponentCount);
-    ASSERT_LT(0u, params.iterations);
 
     mProgram = SetupSimpleScaleAndOffsetProgram();
     ASSERT_NE(0u, mProgram);
@@ -335,7 +334,7 @@ void BufferSubDataBenchmark::drawBenchmark()
 
     const auto &params = GetParam();
 
-    for (unsigned int it = 0; it < params.iterations; it++)
+    for (unsigned int it = 0; it < params.iterationsPerStep; it++)
     {
         if (params.updateSize > 0 && ((getNumStepsPerformed() % params.updateRate) == 0))
         {

src/tests/perf_tests/CompilerPerf.cpp

@@ -162,6 +162,8 @@ void main()
 
 const char *kTrickyESSL300Id = "TrickyESSL300";
 
+constexpr int kNumIterationsPerStep = 10;
+
 struct CompilerPerfParameters final : public angle::CompilerParameters
 {
     CompilerPerfParameters(ShShaderOutput output,
@@ -206,7 +208,8 @@ class CompilerPerfTest : public ANGLEPerfTest,
     sh::TCompiler *mTranslator;
 };
 
-CompilerPerfTest::CompilerPerfTest() : ANGLEPerfTest("CompilerPerf", GetParam().testId)
+CompilerPerfTest::CompilerPerfTest()
+    : ANGLEPerfTest("CompilerPerf", GetParam().testId, kNumIterationsPerStep)
 {
 }
 
@@ -250,8 +253,6 @@ void CompilerPerfTest::step()
     ShCompileOptions compileOptions = SH_OBJECT_CODE | SH_VARIABLES |
                                       SH_INITIALIZE_UNINITIALIZED_LOCALS | SH_INIT_OUTPUT_VARIABLES;
 
-    const int kNumIterationsPerStep = 10;
-
 #if !defined(NDEBUG)
     // Make sure that compilation succeeds and print the info log if it doesn't in debug mode.
     if (!mTranslator->compile(shaderStrings, 1, compileOptions))

src/tests/perf_tests/DispatchComputePerf.cpp

@@ -12,18 +12,19 @@
 
 namespace
 {
+unsigned int kIterationsPerStep = 50;
 
 struct DispatchComputePerfParams final : public RenderTestParams
 {
     DispatchComputePerfParams()
     {
+        iterationsPerStep = kIterationsPerStep;
         majorVersion = 3;
         minorVersion = 1;
     }
 
     std::string suffix() const override;
 
-    unsigned int iterations    = 50;
     unsigned int localSizeX    = 16;
     unsigned int localSizeY    = 16;
     unsigned int textureWidth  = 32;
@@ -77,7 +78,6 @@ DispatchComputePerfBenchmark::DispatchComputePerfBenchmark()
 void DispatchComputePerfBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
-    ASSERT_LT(0u, params.iterations);
 
     initComputeShader();
     initTextures();
@@ -149,7 +149,7 @@ void DispatchComputePerfBenchmark::destroyBenchmark()
 void DispatchComputePerfBenchmark::drawBenchmark()
 {
     const auto &params = GetParam();
-    for (unsigned int it = 0; it < params.iterations; it++)
+    for (unsigned int it = 0; it < params.iterationsPerStep; it++)
     {
         glDispatchCompute(mDispatchX, mDispatchY, 1);
         glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT);

src/tests/perf_tests/DrawCallPerf.cpp

@@ -117,8 +117,6 @@ void DrawCallPerfBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
-
     if (params.stateChange == StateChange::Texture)
     {
         mProgram = SetupSimpleTextureProgram();
@@ -225,21 +223,21 @@ void DrawCallPerfBenchmark::drawBenchmark()
     switch (params.stateChange)
     {
         case StateChange::VertexBuffer:
-            ChangeVerticesThenDraw(params.iterations, numElements, mBuffer1, mBuffer2);
+            ChangeVerticesThenDraw(params.iterationsPerStep, numElements, mBuffer1, mBuffer2);
             break;
         case StateChange::Texture:
-            ChangeTextureThenDraw(params.iterations, numElements, mTexture1, mTexture2);
+            ChangeTextureThenDraw(params.iterationsPerStep, numElements, mTexture1, mTexture2);
             break;
         case StateChange::NoChange:
             if (eglParams.deviceType != EGL_PLATFORM_ANGLE_DEVICE_TYPE_NULL_ANGLE ||
                 (eglParams.renderer != EGL_PLATFORM_ANGLE_TYPE_OPENGL_ANGLE &&
                  eglParams.renderer != EGL_PLATFORM_ANGLE_TYPE_OPENGLES_ANGLE))
             {
-                ClearThenDraw(params.iterations, numElements);
+                ClearThenDraw(params.iterationsPerStep, numElements);
             }
             else
             {
-                JustDraw(params.iterations, numElements);
+                JustDraw(params.iterationsPerStep, numElements);
             }
             break;
     }

src/tests/perf_tests/DrawCallPerfParams.cpp

@@ -15,10 +15,15 @@ DrawCallPerfParams::DrawCallPerfParams()
 {
     majorVersion = 2;
     minorVersion = 0;
-    windowWidth  = 256;
-    windowHeight = 256;
-
-    iterations     = 50;
+    windowWidth  = 64;
+    windowHeight = 64;
+
+// Lower the iteration count in debug.
+#if !defined(NDEBUG)
+    iterationsPerStep = 100;
+#else
+    iterationsPerStep = 20000;
+#endif
     runTimeSeconds = 10.0;
     numTris        = 1;
     useFBO         = false;
@@ -80,7 +85,6 @@ DrawCallPerfParams DrawCallPerfValidationOnly()
 {
     DrawCallPerfParams params;
     params.eglParameters  = DEFAULT();
-    params.iterations     = 10000;
     params.numTris        = 0;
     params.runTimeSeconds = 5.0;
     return params;

src/tests/perf_tests/DrawCallPerfParams.h

@@ -22,7 +22,6 @@ struct DrawCallPerfParams : public RenderTestParams
 
     std::string suffix() const override;
 
-    unsigned int iterations;
     double runTimeSeconds;
     int numTris;
     bool useFBO;

src/tests/perf_tests/DrawElementsPerf.cpp

@@ -102,8 +102,6 @@ void DrawElementsPerfBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
-
     mProgram = SetupSimpleDrawProgram();
     ASSERT_NE(0u, mProgram);
 
@@ -175,7 +173,7 @@ void DrawElementsPerfBenchmark::drawBenchmark()
         const void *bufferData = (params.type == GL_UNSIGNED_INT)
                                      ? static_cast<GLvoid *>(mIntIndexData.data())
                                      : static_cast<GLvoid *>(mShortIndexData.data());
-        for (unsigned int it = 0; it < params.iterations; it++)
+        for (unsigned int it = 0; it < params.iterationsPerStep; it++)
         {
             glBufferSubData(GL_ELEMENT_ARRAY_BUFFER, 0, mBufferSize, bufferData);
             glDrawElements(GL_TRIANGLES, static_cast<GLsizei>(mCount), params.type, 0);
@@ -183,7 +181,7 @@ void DrawElementsPerfBenchmark::drawBenchmark()
     }
     else
     {
-        for (unsigned int it = 0; it < params.iterations; it++)
+        for (unsigned int it = 0; it < params.iterationsPerStep; it++)
         {
             glDrawElements(GL_TRIANGLES, static_cast<GLsizei>(mCount), params.type, 0);
         }

src/tests/perf_tests/DynamicPromotionPerfTest.cpp

@@ -18,20 +18,17 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 4;
 
 struct DynamicPromotionParams final : public RenderTestParams
 {
-    DynamicPromotionParams();
+    DynamicPromotionParams() { iterationsPerStep = kIterationsPerStep; }
+
     std::string suffix() const override;
 
-    size_t vertexCount;
-    unsigned int iterations;
+    size_t vertexCount = 1024;
 };
 
-DynamicPromotionParams::DynamicPromotionParams() : vertexCount(1024), iterations(4)
-{
-}
-
 std::string DynamicPromotionParams::suffix() const
 {
     return RenderTestParams::suffix();
@@ -151,7 +148,7 @@ void DynamicPromotionPerfTest::destroyBenchmark()
 
 void DynamicPromotionPerfTest::drawBenchmark()
 {
-    unsigned int iterations = GetParam().iterations;
+    unsigned int iterations = GetParam().iterationsPerStep;
     size_t vertexCount      = GetParam().vertexCount;
 
     glClear(GL_COLOR_BUFFER_BIT);

src/tests/perf_tests/EGLInitializePerf.cpp

@@ -75,9 +75,7 @@ class EGLInitializePerfTest : public ANGLEPerfTest,
 };
 
 EGLInitializePerfTest::EGLInitializePerfTest()
-    : ANGLEPerfTest("EGLInitialize", "_run"),
-      mOSWindow(nullptr),
-      mDisplay(EGL_NO_DISPLAY)
+    : ANGLEPerfTest("EGLInitialize", "_run", 1), mOSWindow(nullptr), mDisplay(EGL_NO_DISPLAY)
 {
     auto platform = GetParam().eglParameters;
 

src/tests/perf_tests/EGLMakeCurrentPerf.cpp

@@ -37,7 +37,7 @@ class EGLMakeCurrentPerfTest : public ANGLEPerfTest,
 };
 
 EGLMakeCurrentPerfTest::EGLMakeCurrentPerfTest()
-    : ANGLEPerfTest("EGLMakeCurrent", "_run"),
+    : ANGLEPerfTest("EGLMakeCurrent", "_run", 1),
       mOSWindow(nullptr),
       mDisplay(EGL_NO_DISPLAY),
       mSurface(EGL_NO_SURFACE),

src/tests/perf_tests/IndexConversionPerf.cpp

@@ -16,7 +16,6 @@ using namespace angle;
 
 namespace
 {
-
 struct IndexConversionPerfParams final : public RenderTestParams
 {
     std::string suffix() const override
@@ -33,7 +32,6 @@ struct IndexConversionPerfParams final : public RenderTestParams
         return strstr.str();
     }
 
-    unsigned int iterations;
     unsigned int numIndexTris;
 
     // A second test, which covers using index ranges with an offset.
@@ -81,7 +79,7 @@ void IndexConversionPerfTest::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
+    ASSERT_LT(0u, params.iterationsPerStep);
     ASSERT_LT(0u, params.numIndexTris);
 
     mProgram = SetupSimpleScaleAndOffsetProgram();
@@ -157,7 +155,7 @@ void IndexConversionPerfTest::drawConversion()
     // Trigger an update to ensure we convert once a frame
     updateBufferData();
 
-    for (unsigned int it = 0; it < params.iterations; it++)
+    for (unsigned int it = 0; it < params.iterationsPerStep; it++)
     {
         glDrawElements(GL_TRIANGLES, static_cast<GLsizei>(params.numIndexTris * 3 - 1),
                        GL_UNSIGNED_SHORT, reinterpret_cast<void *>(0));
@@ -178,7 +176,7 @@ void IndexConversionPerfTest::drawIndexRange()
     // This test increments an offset each step. Drawing repeatedly may cause the system memory
     // to release. Then, using a fresh offset will require index range validation, which pages
     // it back in. The performance should be good even if the data is was used quite a bit.
-    for (unsigned int it = 0; it < params.iterations; it++)
+    for (unsigned int it = 0; it < params.iterationsPerStep; it++)
     {
         glDrawElements(GL_TRIANGLES, static_cast<GLsizei>(indexCount), GL_UNSIGNED_SHORT,
                        reinterpret_cast<void *>(offset));
@@ -195,7 +193,7 @@ IndexConversionPerfParams IndexConversionPerfD3D11Params()
     params.minorVersion     = 0;
     params.windowWidth      = 256;
     params.windowHeight     = 256;
-    params.iterations       = 225;
+    params.iterationsPerStep = 225;
     params.numIndexTris     = 3000;
     params.indexRangeOffset = 0;
     return params;
@@ -209,7 +207,7 @@ IndexConversionPerfParams IndexRangeOffsetPerfD3D11Params()
     params.minorVersion     = 0;
     params.windowWidth      = 256;
     params.windowHeight     = 256;
-    params.iterations       = 16;
+    params.iterationsPerStep = 16;
     params.numIndexTris     = 50000;
     params.indexRangeOffset = 64;
     return params;

src/tests/perf_tests/IndexDataManagerTest.cpp

@@ -20,6 +20,7 @@ using namespace testing;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 100;
 
 class MockIndexBuffer : public rx::IndexBuffer
 {
@@ -158,7 +159,7 @@ class IndexDataManagerPerfTest : public ANGLEPerfTest
 };
 
 IndexDataManagerPerfTest::IndexDataManagerPerfTest()
-    : ANGLEPerfTest("IndexDataManger", "_run"),
+    : ANGLEPerfTest("IndexDataManger", "_run", kIterationsPerStep),
       mIndexDataManager(&mMockBufferFactory),
       mIndexCount(4000),
       mBufferSize(mIndexCount * sizeof(GLushort)),
@@ -181,7 +182,7 @@ void IndexDataManagerPerfTest::step()
 {
     rx::TranslatedIndexData translatedIndexData;
     gl::IndexRange indexRange;
-    for (unsigned int iteration = 0; iteration < 100; ++iteration)
+    for (unsigned int iteration = 0; iteration < kIterationsPerStep; ++iteration)
     {
         (void)mIndexBuffer.getIndexRange(nullptr, GL_UNSIGNED_SHORT, 0, mIndexCount, false,
                                          &indexRange);

src/tests/perf_tests/InstancingPerf.cpp

@@ -49,7 +49,7 @@ struct InstancingPerfParams final : public RenderTestParams
         minorVersion      = 0;
         windowWidth       = 256;
         windowHeight      = 256;
-        iterations        = 1;
+        iterationsPerStep = 1;
         runTimeSeconds    = 10.0;
         animationEnabled  = false;
         instancingEnabled = true;
@@ -69,7 +69,6 @@ struct InstancingPerfParams final : public RenderTestParams
         return strstr.str();
     }
 
-    unsigned int iterations;
     double runTimeSeconds;
     bool animationEnabled;
     bool instancingEnabled;
@@ -111,8 +110,6 @@ void InstancingPerfBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
-
     const std::string vs =
         "attribute vec2 aPosition;\n"
         "attribute vec3 aTranslate;\n"
@@ -314,14 +311,14 @@ void InstancingPerfBenchmark::drawBenchmark()
     // Render the instances/billboards.
     if (params.instancingEnabled)
     {
-        for (unsigned int it = 0; it < params.iterations; it++)
+        for (unsigned int it = 0; it < params.iterationsPerStep; it++)
         {
             glDrawElementsInstancedANGLE(GL_TRIANGLES, 6, GL_UNSIGNED_INT, nullptr, mNumPoints);
         }
     }
     else
     {
-        for (unsigned int it = 0; it < params.iterations; it++)
+        for (unsigned int it = 0; it < params.iterationsPerStep; it++)
         {
             glDrawElements(GL_TRIANGLES, 6 * mNumPoints, GL_UNSIGNED_INT, nullptr);
         }

src/tests/perf_tests/InterleavedAttributeData.cpp

@@ -21,6 +21,8 @@ struct InterleavedAttributeDataParams final : public RenderTestParams
 {
     InterleavedAttributeDataParams()
     {
+        iterationsPerStep = 1;
+
         // Common default values
         majorVersion = 2;
         minorVersion = 0;

src/tests/perf_tests/LinkProgramPerfTest.cpp

@@ -31,6 +31,8 @@ struct LinkProgramParams final : public RenderTestParams
 {
     LinkProgramParams(LinkProgramOption optionIn)
     {
+        iterationsPerStep = 1;
+
         majorVersion = 2;
         minorVersion = 0;
         windowWidth  = 256;

src/tests/perf_tests/MultiviewPerf.cpp

@@ -64,6 +64,7 @@ struct MultiviewPerfParams final : public RenderTestParams
                         const MultiviewPerfWorkload &workloadIn,
                         MultiviewOption multiviewOptionIn)
     {
+        iterationsPerStep = 1;
         majorVersion    = 3;
         minorVersion    = 0;
         eglParameters   = platformParametersIn;

src/tests/perf_tests/PointSprites.cpp

@@ -19,17 +19,19 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 100;
 
 struct PointSpritesParams final : public RenderTestParams
 {
     PointSpritesParams()
     {
+        iterationsPerStep = kIterationsPerStep;
+
         // Common default params
         majorVersion = 2;
         minorVersion = 0;
         windowWidth = 1280;
         windowHeight = 720;
-        iterations   = 100;
         count = 10;
         size = 3.0f;
         numVaryings = 3;
@@ -40,9 +42,6 @@ struct PointSpritesParams final : public RenderTestParams
     unsigned int count;
     float size;
     unsigned int numVaryings;
-
-    // static parameters
-    unsigned int iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const PointSpritesParams &params)
@@ -87,8 +86,6 @@ void PointSpritesBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
-
     std::stringstream vstrstr;
 
     // Verify "numVaryings" is within MAX_VARYINGS limit
@@ -189,7 +186,7 @@ void PointSpritesBenchmark::drawBenchmark()
 
     const auto &params = GetParam();
 
-    for (unsigned int it = 0; it < params.iterations; it++)
+    for (unsigned int it = 0; it < params.iterationsPerStep; it++)
     {
         //TODO(jmadill): Indexed point rendering. ANGLE is bad at this.
         glDrawArrays(GL_POINTS, 0, params.count);

src/tests/perf_tests/ResultPerf.cpp

@@ -14,7 +14,7 @@ volatile int gThing = 0;
 
 namespace
 {
-constexpr int kIterations = 1000;
+constexpr int kIterationsPerStep = 1000;
 
 class ResultPerfTest : public ANGLEPerfTest
 {
@@ -23,7 +23,7 @@ class ResultPerfTest : public ANGLEPerfTest
     void step() override;
 };
 
-ResultPerfTest::ResultPerfTest() : ANGLEPerfTest("ResultPerf", "_run")
+ResultPerfTest::ResultPerfTest() : ANGLEPerfTest("ResultPerf", "_run", kIterationsPerStep)
 {
 }
 
@@ -56,7 +56,7 @@ gl::Error CallReturningResult(int depth)
 
 void ResultPerfTest::step()
 {
-    for (int i = 0; i < kIterations; i++)
+    for (int i = 0; i < kIterationsPerStep; i++)
     {
         (void)CallReturningResult(0);
         (void)CallReturningResult(0);

src/tests/perf_tests/TexSubImage.cpp

@@ -16,11 +16,14 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 9;
 
 struct TexSubImageParams final : public RenderTestParams
 {
     TexSubImageParams()
     {
+        iterationsPerStep = kIterationsPerStep;
+
         // Common default parameters
         majorVersion = 2;
         minorVersion = 0;
@@ -31,7 +34,6 @@ struct TexSubImageParams final : public RenderTestParams
         imageHeight    = 1024;
         subImageWidth  = 64;
         subImageHeight = 64;
-        iterations     = 9;
     }
 
     std::string suffix() const override;
@@ -41,7 +43,6 @@ struct TexSubImageParams final : public RenderTestParams
     int imageHeight;
     int subImageWidth;
     int subImageHeight;
-    unsigned int iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const TexSubImageParams &params)
@@ -107,8 +108,6 @@ GLuint TexSubImageBenchmark::createTexture()
 {
     const auto &params = GetParam();
 
-    assert(params.iterations > 0);
-
     // Use tightly packed data
     glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
@@ -249,7 +248,7 @@ void TexSubImageBenchmark::drawBenchmark()
 
     const auto &params = GetParam();
 
-    for (unsigned int iteration = 0; iteration < params.iterations; ++iteration)
+    for (unsigned int iteration = 0; iteration < params.iterationsPerStep; ++iteration)
     {
         glTexSubImage2D(GL_TEXTURE_2D, 0, rand() % (params.imageWidth - params.subImageWidth),
                         rand() % (params.imageHeight - params.subImageHeight), params.subImageWidth,

src/tests/perf_tests/TextureSampling.cpp

@@ -22,17 +22,19 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 4;
 
 struct TextureSamplingParams final : public RenderTestParams
 {
     TextureSamplingParams()
     {
+        iterationsPerStep = kIterationsPerStep;
+
         // Common default params
         majorVersion = 2;
         minorVersion = 0;
         windowWidth  = 720;
         windowHeight = 720;
-        iterations   = 4;
 
         numSamplers = 2;
         textureSize = 32;
@@ -43,9 +45,6 @@ struct TextureSamplingParams final : public RenderTestParams
     unsigned int numSamplers;
     unsigned int textureSize;
     unsigned int kernelSize;
-
-    // static parameters
-    unsigned int iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const TextureSamplingParams &params)
@@ -92,8 +91,6 @@ void TextureSamplingBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_LT(0u, params.iterations);
-
     // Verify "numSamplers" is within MAX_TEXTURE_IMAGE_UNITS limit
     GLint maxTextureImageUnits;
     glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits);
@@ -249,7 +246,7 @@ void TextureSamplingBenchmark::drawBenchmark()
 
     const auto &params = GetParam();
 
-    for (unsigned int it = 0; it < params.iterations; ++it)
+    for (unsigned int it = 0; it < params.iterationsPerStep; ++it)
     {
         glDrawArrays(GL_TRIANGLES, 0, 6);
     }

src/tests/perf_tests/TexturesPerf.cpp

@@ -17,17 +17,19 @@
 
 namespace angle
 {
+constexpr unsigned int kIterationsPerStep = 256;
 
 struct TexturesParams final : public RenderTestParams
 {
     TexturesParams()
     {
+        iterationsPerStep = kIterationsPerStep;
+
         // Common default params
         majorVersion = 2;
         minorVersion = 0;
         windowWidth  = 720;
         windowHeight = 720;
-        iterations   = 256;
 
         numTextures                 = 8;
         textureRebindFrequency      = 5;
@@ -44,9 +46,6 @@ struct TexturesParams final : public RenderTestParams
     size_t textureMipCount;
 
     bool webgl;
-
-    // static parameters
-    size_t iterations;
 };
 
 std::ostream &operator<<(std::ostream &os, const TexturesParams &params)
@@ -103,8 +102,6 @@ void TexturesBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_GT(params.iterations, 0u);
-
     // Verify the uniform counts are within the limits
     GLint maxTextureUnits;
     glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureUnits);
@@ -210,7 +207,7 @@ void TexturesBenchmark::drawBenchmark()
 {
     const auto &params = GetParam();
 
-    for (size_t it = 0; it < params.iterations; ++it)
+    for (size_t it = 0; it < params.iterationsPerStep; ++it)
     {
         if (it % params.textureRebindFrequency == 0)
         {

src/tests/perf_tests/UniformsPerf.cpp

@@ -21,6 +21,7 @@ using namespace angle;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 4;
 
 // Controls when we call glUniform, if the data is the same as last frame.
 enum DataMode
@@ -48,6 +49,8 @@ struct UniformsParams final : public RenderTestParams
 {
     UniformsParams()
     {
+        iterationsPerStep = kIterationsPerStep;
+
         // Common default params
         majorVersion = 2;
         minorVersion = 0;
@@ -62,9 +65,6 @@ struct UniformsParams final : public RenderTestParams
     DataType dataType = DataType::VEC4;
     DataMode dataMode = DataMode::REPEAT;
     ProgramMode programMode = ProgramMode::SINGLE;
-
-    // static parameters
-    size_t iterations = 4;
 };
 
 std::ostream &operator<<(std::ostream &os, const UniformsParams &params)
@@ -160,8 +160,6 @@ void UniformsBenchmark::initializeBenchmark()
 {
     const auto &params = GetParam();
 
-    ASSERT_GT(params.iterations, 0u);
-
     // Verify the uniform counts are within the limits
     GLint maxVertexUniformVectors, maxFragmentUniformVectors;
     glGetIntegerv(GL_MAX_VERTEX_UNIFORM_VECTORS, &maxVertexUniformVectors);
@@ -323,7 +321,7 @@ void UniformsBenchmark::drawLoop(const SetUniformFunc &setUniformsFunc)
 
     size_t frameIndex = 0;
 
-    for (size_t it = 0; it < params.iterations; ++it, frameIndex = (frameIndex == 0 ? 1 : 0))
+    for (size_t it = 0; it < params.iterationsPerStep; ++it, frameIndex = (frameIndex == 0 ? 1 : 0))
     {
         if (MultiProgram)
         {

src/tests/perf_tests/VulkanCommandBufferPerf.cpp

@@ -72,7 +72,7 @@ class VulkanCommandBufferPerfTest : public ANGLEPerfTest,
 };
 
 VulkanCommandBufferPerfTest::VulkanCommandBufferPerfTest()
-    : ANGLEPerfTest("VulkanCommandBufferPerfTest", GetParam().suffix)
+    : ANGLEPerfTest("VulkanCommandBufferPerfTest", GetParam().suffix, GetParam().frames)
 {
     mInfo             = {};
     mSampleTitle      = "Draw Textured Cube";

src/tests/perf_tests/VulkanPipelineCachePerf.cpp

@@ -15,6 +15,8 @@ using namespace rx;
 
 namespace
 {
+constexpr unsigned int kIterationsPerStep = 100;
+
 class VulkanPipelineCachePerfTest : public ANGLEPerfTest
 {
   public:
@@ -36,7 +38,7 @@ class VulkanPipelineCachePerfTest : public ANGLEPerfTest
 };
 
 VulkanPipelineCachePerfTest::VulkanPipelineCachePerfTest()
-    : ANGLEPerfTest("VulkanPipelineCachePerf", "")
+    : ANGLEPerfTest("VulkanPipelineCachePerf", "", kIterationsPerStep)
 {
 }
 
@@ -85,7 +87,7 @@ void VulkanPipelineCachePerfTest::step()
     vk::PipelineAndSerial *result = nullptr;
     gl::AttributesMask am;
 
-    for (int iteration = 0; iteration < 100; ++iteration)
+    for (unsigned int iteration = 0; iteration < kIterationsPerStep; ++iteration)
     {
         for (const auto &hit : mCacheHits)
         {

util/posix/Posix_system_utils.cpp

@@ -47,4 +47,10 @@ void WriteDebugMessage(const char *format, ...)
     // TODO(jmadill): Implement this
 }
 
+bool StabilizeCPUForBenchmarking()
+{
+    // TODO(jmadill): Implement this. http://anglebug.com/2923
+    return true;
+}
+
 } // namespace angle

util/system_utils.h

@@ -27,6 +27,9 @@ ANGLE_EXPORT void SetLowPriorityProcess();
 // Write a debug message, either to a standard output or Debug window.
 ANGLE_EXPORT void WriteDebugMessage(const char *format, ...);
 
+// Set thread affinity and priority.
+ANGLE_EXPORT bool StabilizeCPUForBenchmarking();
+
 } // namespace angle
 
 #endif  // UTIL_SYSTEM_UTILS_H_

util/windows/win32/Win32_system_utils.cpp

@@ -19,4 +19,18 @@ void SetLowPriorityProcess()
     SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS);
 }
 
+bool StabilizeCPUForBenchmarking()
+{
+    if (SetThreadAffinityMask(GetCurrentThread(), 1) == 0)
+    {
+        return false;
+    }
+    if (SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST) == FALSE)
+    {
+        return false;
+    }
+
+    return true;
+}
+
 }  // namespace angle

util/windows/winrt/WinRT_system_utils.cpp

@@ -19,4 +19,10 @@ void SetLowPriorityProcess()
     // No equivalent to this in WinRT
 }
 
+bool StabilizeCPUForBenchmarking()
+{
+    // No equivalent to this in WinRT
+    return true;
+}
+
 }  // namespace angle