Skip to content

A
angle
Commit c65517d6 by Manh Nguyen Committed by Commit Bot
Options

Add tests' expectations to capture replay tests

Skips failing tests so that they do not affect the results of passing tests. Bug: angleproject:4817 Change-Id: I80678cb62b0b81d6a1aa811908019022efbe1268 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2351394Reviewed-by: 's avatarCody Northrop <cnorthrop@google.com> Reviewed-by: 's avatarJamie Madill <jmadill@chromium.org> Commit-Queue: Manh Nguyen <nguyenmh@google.com>
parent 07ae53f5
Showing with 255 additions and 23 deletions
src/tests/capture_replay_tests.py
...@@ -36,6 +36,7 @@ Script testing capture_replay with angle_end2end_tests ...@@ -36,6 +36,7 @@ Script testing capture_replay with angle_end2end_tests
import argparse import argparse
import distutils.util import distutils.util
import fnmatch
import math import math
import multiprocessing import multiprocessing
import os import os
...@@ -338,16 +339,30 @@ def GetTestNamesAndParamsCommand(test_exec_path, filter="*"): ...@@ -338,16 +339,30 @@ def GetTestNamesAndParamsCommand(test_exec_path, filter="*"):
def ProcessGetTestNamesAndParamsCommandOutput(output): def ProcessGetTestNamesAndParamsCommandOutput(output):
def SkipTest(skipped_test_patterns, full_test_name):
for skipped_test_pattern in skipped_test_patterns:
if fnmatch.fnmatch(full_test_name, skipped_test_pattern):
return True
return False
output_lines = output.splitlines() output_lines = output.splitlines()
tests = [] tests = []
last_testcase_name = "" last_testcase_name = ""
test_name_splitter = "# GetParam() =" test_name_splitter = "# GetParam() ="
test_expectations_filename = "capture_replay_expectations.txt"
skipped_test_patterns = [
line.rstrip() for line in \
open(os.path.join(REPLAY_SAMPLE_FOLDER, test_expectations_filename), "r") \
if (not line.startswith("#") and not line.isspace())
]
for line in output_lines: for line in output_lines:
if test_name_splitter in line: if test_name_splitter in line:
# must be a test name line # must be a test name line
test_name_and_params = line.split(test_name_splitter) test_name_and_params = line.split(test_name_splitter)
tests.append((last_testcase_name + test_name_and_params[0].strip(), \ full_test_name = last_testcase_name + test_name_and_params[0].strip()
test_name_and_params[1].strip())) if not SkipTest(skipped_test_patterns, full_test_name):
tests.append((full_test_name, test_name_and_params[1].strip()))
else: else:
# gtest_list_tests returns the test in this format # gtest_list_tests returns the test in this format
# test case # test case
...@@ -495,29 +510,33 @@ class Test(): ...@@ -495,29 +510,33 @@ class Test():
return self.full_test_name.replace(".", "_").replace("/", "_") return self.full_test_name.replace(".", "_").replace("/", "_")
def CanRunReplay(self, trace_folder_path): def CanRunReplay(self, trace_folder_path):
test_files = []
label = self.GetLabel() label = self.GetLabel()
assert (self.context_id > 0) assert (self.context_id == 0)
trace_file_suffix = TRACE_FILE_SUFFIX + str(self.context_id)
required_trace_files = {
label + trace_file_suffix + ".h", label + trace_file_suffix + ".cpp",
label + trace_file_suffix + "_files.txt"
}
required_trace_files_count = 0
frame_files_count = 0
for f in os.listdir(trace_folder_path): for f in os.listdir(trace_folder_path):
if not os.path.isfile(os.path.join(trace_folder_path, f)): if os.path.isfile(os.path.join(trace_folder_path, f)) and f.startswith(label):
continue test_files.append(f)
if f in required_trace_files: frame_files_count = 0
required_trace_files_count += 1 context_header_count = 0
elif f.startswith(label + trace_file_suffix + "_frame"): context_source_count = 0
source_txt_count = 0
context_id = 0
for f in test_files:
if "_frame" in f:
frame_files_count += 1 frame_files_count += 1
elif f.startswith(label + elif f.endswith(".txt"):
trace_file_suffix[:-1]) and not f.startswith(label + source_txt_count += 1
trace_file_suffix): elif f.endswith(".h"):
# if trace_files of another context exists, then the test creates multiple contexts context_header_count += 1
return False context_id = int(f.split("capture_context")[1][:-2])
# angle_capture_context1.angledata.gz can be missing elif f.endswith(".cpp"):
return required_trace_files_count == len(required_trace_files) and frame_files_count >= 1 context_source_count += 1
can_run_replay = frame_files_count >= 1 and context_header_count == 1 \
and context_source_count == 1 and source_txt_count == 1
if not can_run_replay:
return False
self.context_id = context_id
return True
class TestBatch(): class TestBatch():
...@@ -642,7 +661,6 @@ class TestBatch(): ...@@ -642,7 +661,6 @@ class TestBatch():
assert len(self.tests) <= self.batch_count assert len(self.tests) <= self.batch_count
test.index = len(self.tests) test.index = len(self.tests)
self.tests.append(test) self.tests.append(test)
test.context_id = len(self.tests)
# gni file, which holds all the sources for a replay application # gni file, which holds all the sources for a replay application
def CreateGNIFile(self, trace_folder_path, composite_file_id, tests): def CreateGNIFile(self, trace_folder_path, composite_file_id, tests):
......
src/tests/capture_replay_tests/capture_replay_expectations.txt 0 → 100644
#
# Copyright 2020 The ANGLE Project Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# capture_replay_expectations:
# Skip these tests because they are expected to fail
#
# Failed tests
BindGeneratesResourceTest.*
BindUniformLocationTest.Basic/ES2_Vulkan
BindUniformLocationTest.Compositor/ES2_Vulkan
BindUniformLocationTest.SameLocationForUsedAndUnusedUniform/ES2_Vulkan
BindUniformLocationTest.SamplerLocation/ES2_Vulkan
BindUniformLocationTest.UnusedUniformUpdate/ES2_Vulkan
BindUniformLocationTest.UnusedUniformUpdateComplex/ES2_Vulkan
BindUniformLocationTest.UseSamplerWhenUnusedUniforms/ES2_Vulkan
VulkanClearTest.*
ClientArraysTest.*
CopyTexImageTest.*
CopyTextureTestWebGL.*
DifferentStencilMasksTest.*
EGLContextASANTest.*
EGLContextSharingTest.TextureLifetime/ES2_Vulkan
FramebufferTest.MultiContextNoRenderingFeedbackLoops/ES2_Vulkan
MaxTextureSizeTest.*
MultisampledRenderToTextureTest.FramebufferCompleteness/ES2_Vulkan
MultisampledRenderToTextureTest.TextureCubeMapParameterCheck/ES2_Vulkan
PbufferTest.BindTexImage/ES2_Vulkan
PbufferTest.Clearing/ES2_Vulkan
PbufferTest.TextureSizeReset/ES2_Vulkan
RobustBufferAccessBehaviorTest.DynamicBuffer/ES2_Vulkan
TextureCubeTest.*
Texture2DDepthTest.DepthTextureES2Compatibility/ES2_Vulkan
WebGL2ReadOutsideFramebufferTest.*
WebGLDrawElementsTest.*
WebGLGLSLTest.*
WebGLReadOutsideFramebufferTest.*
D3DImageFormatConversionTest.*
# Crashed tests
BlendMinMaxTest.*
BlitFramebufferANGLETest.*
BufferDataTest.*
ClipDistanceTest.*
ColorMaskTest.*
DebugMarkerTest.*
DebugTest.*
DepthStencilTest.*
DiscardFramebufferEXTTest.*
DrawBuffersTest.*
EGLStreamTest.*
ExternalWrapTest.*
FenceNVTest.*
GLSLTest.ArrayOfStructContainingArrayOfSamplers/ES2_Vulkan
GLSLTest.MixedShaderLengths/ES2_Vulkan
GLSLTest.ZeroShaderLength/ES2_Vulkan
GetImageTestNoExtensions.EntryPointsInactive/ES2_Vulkan
ImageTest.*
IndexedPointsTestUByte.*
IndexedPointsTestUInt.*
IndexedPointsTestUShort.*
OcclusionQueriesTest.*
PBOExtensionTest.*
SRGBFramebufferTest.Validation/ES2_Vulkan
SemaphoreTest.SemaphoreShouldBeSemaphore/ES2_Vulkan
SyncQueriesTest.*
TimerQueriesTest.*
UniformTest.Sampler/ES2_Vulkan
UnpackAlignmentTest.*
UnpackRowLengthTest.*
VertexAttributeCachingTest.*
VertexAttributeOORTest.*
VertexAttributeTest.*
ViewportTest.*
WebGLCompatibilityTest.*
WebGLFramebufferTest.*
# Skipped tests
EGLProgramCacheControlTest.NegativeAPI/ES2_Vulkan
EGLProgramCacheControlTest.SaveAndReload/ES2_Vulkan
LineLoopTest.DISABLED_DrawArraysWithLargeCount/ES2_Vulkan
MipmapTest.DISABLED_ThreeLevelsInitData/ES2_Vulkan
# Compile failed tests
GetImageTest.*
InstancingTest.IndexedAttrib0PointBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0PointBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib0PointMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0PointMemoryExt/ES2_Vulkan
InstancingTest.IndexedAttrib0QuadBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0QuadBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib0QuadMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0QuadMemoryExt/ES2_Vulkan
InstancingTest.IndexedAttrib0TriFanBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0TriFanBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib0TriFanMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib0TriFanMemoryExt/ES2_Vulkan
InstancingTest.IndexedAttrib1PointBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1PointBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib1PointMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1PointMemoryExt/ES2_Vulkan
InstancingTest.IndexedAttrib1QuadBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1QuadBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib1QuadMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1QuadMemoryExt/ES2_Vulkan
InstancingTest.IndexedAttrib1TriFanBufferAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1TriFanBufferExt/ES2_Vulkan
InstancingTest.IndexedAttrib1TriFanMemoryAngle/ES2_Vulkan
InstancingTest.IndexedAttrib1TriFanMemoryExt/ES2_Vulkan
InstancingTest.Instances10Divisor1/ES2_Vulkan
InstancingTest.Instances11Divisor1/ES2_Vulkan
InstancingTest.Instances11Divisor2/ES2_Vulkan
InstancingTest.Instances12Divisor1/ES2_Vulkan
InstancingTest.Instances12Divisor11/ES2_Vulkan
InstancingTest.Instances13Divisor1/ES2_Vulkan
InstancingTest.Instances13Divisor2/ES2_Vulkan
InstancingTest.Instances14Divisor1/ES2_Vulkan
InstancingTest.Instances15Divisor1/ES2_Vulkan
InstancingTest.Instances15Divisor2/ES2_Vulkan
InstancingTest.Instances16Divisor1/ES2_Vulkan
InstancingTest.Instances16Divisor3/ES2_Vulkan
InstancingTest.Instances16Divisor7/ES2_Vulkan
InstancingTest.Instances17Divisor2/ES2_Vulkan
InstancingTest.Instances1Divisor1/ES2_Vulkan
InstancingTest.Instances1Divisor2/ES2_Vulkan
InstancingTest.Instances20Divisor2/ES2_Vulkan
InstancingTest.Instances21Divisor2/ES2_Vulkan
InstancingTest.Instances23Divisor2/ES2_Vulkan
InstancingTest.Instances25Divisor33/ES2_Vulkan
InstancingTest.Instances25Divisor5/ES2_Vulkan
InstancingTest.Instances26Divisor2/ES2_Vulkan
InstancingTest.Instances26Divisor3/ES2_Vulkan
InstancingTest.Instances27Divisor2/ES2_Vulkan
InstancingTest.Instances27Divisor4/ES2_Vulkan
InstancingTest.Instances28Divisor3/ES2_Vulkan
InstancingTest.Instances29Divisor11/ES2_Vulkan
InstancingTest.Instances29Divisor2/ES2_Vulkan
InstancingTest.Instances2Divisor1/ES2_Vulkan
InstancingTest.Instances30Divisor4/ES2_Vulkan
InstancingTest.Instances31Divisor6/ES2_Vulkan
InstancingTest.Instances32Divisor2/ES2_Vulkan
InstancingTest.Instances32Divisor3/ES2_Vulkan
InstancingTest.Instances32Divisor8/ES2_Vulkan
InstancingTest.Instances34Divisor3/ES2_Vulkan
InstancingTest.Instances34Divisor30/ES2_Vulkan
InstancingTest.Instances3Divisor1/ES2_Vulkan
InstancingTest.Instances3Divisor2/ES2_Vulkan
InstancingTest.Instances4Divisor1/ES2_Vulkan
InstancingTest.Instances5Divisor1/ES2_Vulkan
InstancingTest.Instances5Divisor2/ES2_Vulkan
InstancingTest.Instances6Divisor1/ES2_Vulkan
InstancingTest.Instances6Divisor2/ES2_Vulkan
InstancingTest.Instances7Divisor1/ES2_Vulkan
InstancingTest.Instances7Divisor2/ES2_Vulkan
InstancingTest.Instances8Divisor1/ES2_Vulkan
InstancingTest.Instances8Divisor2/ES2_Vulkan
InstancingTest.Instances8Divisor4/ES2_Vulkan
InstancingTest.Instances9Divisor1/ES2_Vulkan
InstancingTest.Instances9Divisor2/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointBufferAngleOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointBufferExtOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointMemoryAngleOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0PointMemoryExtOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadBufferAngleOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadBufferExtOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadMemoryAngleOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0QuadMemoryExtOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanBufferAngleOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanBufferExtOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanMemoryAngleOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib0TriFanMemoryExtOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointBufferAngleOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointBufferExtOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointMemoryAngleOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1PointMemoryExtOffset2/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadBufferAngleOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadBufferExtOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadMemoryAngleOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1QuadMemoryExtOffset4/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanBufferAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanBufferAngleOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanBufferExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanBufferExtOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanMemoryAngleOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanMemoryAngleOffset8/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanMemoryExtOffset0/ES2_Vulkan
InstancingTest.NonIndexedAttrib1TriFanMemoryExtOffset8/ES2_Vulkan
MemoryObjectTest.MemoryObjectQueries/ES2_Vulkan
MemoryObjectTest.MemoryObjectShouldBeMemoryObject/ES2_Vulkan
RobustClientMemoryTest.*
RobustResourceInitTest.*
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment