Skip to content

A
angle
Commit a72f400c by Jamie Madill Committed by Commit Bot
Options

run_code_generation: Compare hashes instead of mtime.

Using mtime is fundamentally flawed when working with git. Replace these flaky checks with hash comparisons. The hashes are stored in an autogenerated json file that will be stored in the repository. This makes the run_code_generation script robust against any and all input changes. It also removes the need to track script outputs as dependencies. Bug: angleproject:2697 Change-Id: I60f2a87a8680b1f775ad678b05112f5b16c7dde7 Reviewed-on: https://chromium-review.googlesource.com/1120159 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarYuly Novikov <ynovikov@chromium.org>
parent 33e72d99
Showing with 133 additions and 83 deletions
scripts/run_code_generation.py
...@@ -7,9 +7,14 @@ ...@@ -7,9 +7,14 @@
# run_code_reneration.py: # run_code_reneration.py:
# Runs ANGLE format table and other script run_code_renerationgeneration. # Runs ANGLE format table and other script run_code_renerationgeneration.
import os, subprocess, sys import hashlib
import json
import os
import subprocess
import sys
root_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) script_dir = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
root_dir = os.path.abspath(os.path.join(script_dir, '..'))
# auto_script is a standard way for scripts to return their inputs and outputs. # auto_script is a standard way for scripts to return their inputs and outputs.
...@@ -24,9 +29,10 @@ def auto_script(script): ...@@ -24,9 +29,10 @@ def auto_script(script):
return { return {
'script': script, 'script': script,
'inputs': grab_from_script(base_script, 'inputs'), 'inputs': grab_from_script(base_script, 'inputs'),
'outputs': grab_from_script(base_script, 'outputs'),
} }
hash_fname = "run_code_generation_hashes.json"
# TODO(jmadill): Convert everyting to auto-script. # TODO(jmadill): Convert everyting to auto-script.
generators = { generators = {
'ANGLE format': { 'ANGLE format': {
...@@ -35,19 +41,12 @@ generators = { ...@@ -35,19 +41,12 @@ generators = {
'src/libANGLE/renderer/angle_format_data.json', 'src/libANGLE/renderer/angle_format_data.json',
'src/libANGLE/renderer/angle_format_map.json', 'src/libANGLE/renderer/angle_format_map.json',
], ],
'outputs': [
'src/libANGLE/renderer/Format_table_autogen.cpp',
'src/libANGLE/renderer/Format_ID_autogen.inl',
],
'script': 'src/libANGLE/renderer/gen_angle_format_table.py', 'script': 'src/libANGLE/renderer/gen_angle_format_table.py',
}, },
'ANGLE load functions table': { 'ANGLE load functions table': {
'inputs': [ 'inputs': [
'src/libANGLE/renderer/load_functions_data.json', 'src/libANGLE/renderer/load_functions_data.json',
], ],
'outputs': [
'src/libANGLE/renderer/load_functions_table_autogen.cpp',
],
'script': 'src/libANGLE/renderer/gen_load_functions_table.py', 'script': 'src/libANGLE/renderer/gen_load_functions_table.py',
}, },
'D3D11 format': { 'D3D11 format': {
...@@ -56,9 +55,6 @@ generators = { ...@@ -56,9 +55,6 @@ generators = {
'src/libANGLE/renderer/d3d/d3d11/texture_format_data.json', 'src/libANGLE/renderer/d3d/d3d11/texture_format_data.json',
'src/libANGLE/renderer/d3d/d3d11/texture_format_map.json', 'src/libANGLE/renderer/d3d/d3d11/texture_format_map.json',
], ],
'outputs': [
'src/libANGLE/renderer/d3d/d3d11/texture_format_table_autogen.cpp',
],
'script': 'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py', 'script': 'src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py',
}, },
'DXGI format': { 'DXGI format': {
...@@ -67,27 +63,18 @@ generators = { ...@@ -67,27 +63,18 @@ generators = {
'src/libANGLE/renderer/angle_format_map.json', 'src/libANGLE/renderer/angle_format_map.json',
'src/libANGLE/renderer/d3d/d3d11/dxgi_format_data.json', 'src/libANGLE/renderer/d3d/d3d11/dxgi_format_data.json',
], ],
'outputs': [
'src/libANGLE/renderer/d3d/d3d11/dxgi_format_map_autogen.cpp',
],
'script': 'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_format_table.py', 'script': 'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_format_table.py',
}, },
'DXGI format support': { 'DXGI format support': {
'inputs': [ 'inputs': [
'src/libANGLE/renderer/d3d/d3d11/dxgi_support_data.json', 'src/libANGLE/renderer/d3d/d3d11/dxgi_support_data.json',
], ],
'outputs': [
'src/libANGLE/renderer/d3d/d3d11/dxgi_support_table.cpp',
],
'script': 'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_support_tables.py', 'script': 'src/libANGLE/renderer/d3d/d3d11/gen_dxgi_support_tables.py',
}, },
'GL copy conversion table': { 'GL copy conversion table': {
'inputs': [ 'inputs': [
'src/libANGLE/es3_copy_conversion_formats.json', 'src/libANGLE/es3_copy_conversion_formats.json',
], ],
'outputs': [
'src/libANGLE/es3_copy_conversion_table_autogen.cpp',
],
'script': 'src/libANGLE/gen_copy_conversion_table.py', 'script': 'src/libANGLE/gen_copy_conversion_table.py',
}, },
'GL entry point': { 'GL entry point': {
...@@ -95,12 +82,6 @@ generators = { ...@@ -95,12 +82,6 @@ generators = {
'scripts/entry_point_packed_gl_enums.json', 'scripts/entry_point_packed_gl_enums.json',
'scripts/gl.xml', 'scripts/gl.xml',
], ],
'outputs': [
'src/libGLESv2/entry_points_gles_2_0_autogen.cpp',
'src/libGLESv2/entry_points_gles_2_0_autogen.h',
'src/libGLESv2/entry_points_gles_3_0_autogen.cpp',
'src/libGLESv2/entry_points_gles_3_0_autogen.h',
],
'script': 'scripts/generate_entry_points.py', 'script': 'scripts/generate_entry_points.py',
}, },
'GL format map': { 'GL format map': {
...@@ -108,28 +89,16 @@ generators = { ...@@ -108,28 +89,16 @@ generators = {
'src/libANGLE/es3_format_type_combinations.json', 'src/libANGLE/es3_format_type_combinations.json',
'src/libANGLE/format_map_data.json', 'src/libANGLE/format_map_data.json',
], ],
'outputs': [
'src/libANGLE/format_map_autogen.cpp',
],
'script': 'src/libANGLE/gen_format_map.py', 'script': 'src/libANGLE/gen_format_map.py',
}, },
'uniform type': { 'uniform type': {
'inputs': [], 'inputs': [],
'outputs': [
'src/common/uniform_type_info_autogen.cpp',
],
'script': 'src/common/gen_uniform_type_table.py', 'script': 'src/common/gen_uniform_type_table.py',
}, },
'OpenGL dispatch table': { 'OpenGL dispatch table': {
'inputs': [ 'inputs': [
'scripts/gl.xml', 'scripts/gl.xml',
], ],
'outputs': [
'src/libANGLE/renderer/gl/DispatchTableGL_autogen.cpp',
'src/libANGLE/renderer/gl/DispatchTableGL_autogen.h',
'src/libANGLE/renderer/gl/null_functions.h',
'src/libANGLE/renderer/gl/null_functions.cpp',
],
'script': 'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py', 'script': 'src/libANGLE/renderer/gl/generate_gl_dispatch_table.py',
}, },
'packed enum': { 'packed enum': {
...@@ -137,21 +106,12 @@ generators = { ...@@ -137,21 +106,12 @@ generators = {
'src/common/packed_gl_enums.json', 'src/common/packed_gl_enums.json',
'src/common/packed_egl_enums.json', 'src/common/packed_egl_enums.json',
], ],
'outputs': [
'src/common/PackedEGLEnums_autogen.cpp',
'src/common/PackedEGLEnums_autogen.h',
'src/common/PackedGLEnums_autogen.cpp',
'src/common/PackedGLEnums_autogen.h',
],
'script': 'src/common/gen_packed_gl_enums.py', 'script': 'src/common/gen_packed_gl_enums.py',
}, },
'proc table': { 'proc table': {
'inputs': [ 'inputs': [
'src/libGLESv2/proc_table_data.json', 'src/libGLESv2/proc_table_data.json',
], ],
'outputs': [
'src/libGLESv2/proc_table_autogen.cpp',
],
'script': 'src/libGLESv2/gen_proc_table.py', 'script': 'src/libGLESv2/gen_proc_table.py',
}, },
'Vulkan format': { 'Vulkan format': {
...@@ -160,9 +120,6 @@ generators = { ...@@ -160,9 +120,6 @@ generators = {
'src/libANGLE/renderer/angle_format_map.json', 'src/libANGLE/renderer/angle_format_map.json',
'src/libANGLE/renderer/vulkan/vk_format_map.json', 'src/libANGLE/renderer/vulkan/vk_format_map.json',
], ],
'outputs': [
'src/libANGLE/renderer/vulkan/vk_format_table_autogen.cpp',
],
'script': 'src/libANGLE/renderer/vulkan/gen_vk_format_table.py', 'script': 'src/libANGLE/renderer/vulkan/gen_vk_format_table.py',
}, },
'Vulkan mandatory format support table': { 'Vulkan mandatory format support table': {
...@@ -171,9 +128,6 @@ generators = { ...@@ -171,9 +128,6 @@ generators = {
'third_party/vulkan-headers/src/registry/vk.xml', 'third_party/vulkan-headers/src/registry/vk.xml',
'src/libANGLE/renderer/vulkan/vk_mandatory_format_support_data.json', 'src/libANGLE/renderer/vulkan/vk_mandatory_format_support_data.json',
], ],
'outputs': [
'src/libANGLE/renderer/vulkan/vk_mandatory_format_support_table_autogen.cpp',
],
'script': 'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py', 'script': 'src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py',
}, },
'Vulkan internal shader programs': 'Vulkan internal shader programs':
...@@ -182,9 +136,6 @@ generators = { ...@@ -182,9 +136,6 @@ generators = {
'inputs': [ 'inputs': [
'src/compiler/translator/emulated_builtin_function_data_hlsl.json' 'src/compiler/translator/emulated_builtin_function_data_hlsl.json'
], ],
'outputs': [
'src/compiler/translator/emulated_builtin_functions_hlsl_autogen.cpp'
],
'script': 'src/compiler/translator/gen_emulated_builtin_function_tables.py' 'script': 'src/compiler/translator/gen_emulated_builtin_function_tables.py'
}, },
'ESSL static builtins': { 'ESSL static builtins': {
...@@ -192,41 +143,37 @@ generators = { ...@@ -192,41 +143,37 @@ generators = {
'src/compiler/translator/builtin_function_declarations.txt', 'src/compiler/translator/builtin_function_declarations.txt',
'src/compiler/translator/builtin_variables.json', 'src/compiler/translator/builtin_variables.json',
], ],
'outputs': [
'src/compiler/translator/tree_util/BuiltIn_autogen.h',
'src/compiler/translator/builtin_symbols_hash_autogen.txt',
'src/compiler/translator/ParseContext_autogen.h',
'src/compiler/translator/SymbolTable_autogen.cpp',
'src/compiler/translator/SymbolTable_autogen.h',
'src/tests/compiler_tests/ImmutableString_test_autogen.cpp',
],
'script': 'src/compiler/translator/gen_builtin_symbols.py', 'script': 'src/compiler/translator/gen_builtin_symbols.py',
}, },
} }
def md5(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def any_input_dirty(name, inputs):
for finput in inputs:
key = name + ":" + finput
new_hashes[key] = md5(finput)
if (not key in old_hashes) or (old_hashes[key] != new_hashes[key]):
return True
return False
os.chdir(script_dir)
old_hashes = json.load(open(hash_fname))
new_hashes = {}
any_dirty = False any_dirty = False
for name, info in sorted(generators.iteritems()): for name, info in sorted(generators.iteritems()):
# Set the CWD to the root ANGLE directory. # Reset the CWD to the root ANGLE directory.
os.chdir(root_dir) os.chdir(root_dir)
script = info['script'] script = info['script']
dirty = False
if any_input_dirty(name, info['inputs'] + [script]):
for finput in info['inputs'] + [script]:
input_mtime = os.path.getmtime(finput)
for foutput in info['outputs']:
if not os.path.exists(foutput):
print('Output ' + foutput + ' not found for ' + name + ' table')
dirty = True
else:
output_mtime = os.path.getmtime(foutput)
# Use a fuzzy comparison to avoid tiny time delta errors.
if input_mtime - output_mtime > 0.1:
dirty = True
if dirty:
any_dirty = True any_dirty = True
# Set the CWD to the script directory. # Set the CWD to the script directory.
...@@ -247,3 +194,7 @@ if any_dirty: ...@@ -247,3 +194,7 @@ if any_dirty:
args += ['cl', 'format', '--full'] args += ['cl', 'format', '--full']
print('Calling git cl format') print('Calling git cl format')
subprocess.call(args) subprocess.call(args)
os.chdir(script_dir)
json.dump(new_hashes, open(hash_fname, "w"), indent=2, sort_keys=True,
separators=(',', ':\n '))
scripts/run_code_generation_hashes.json 0 → 100644
{
"ANGLE format:src/libANGLE/renderer/angle_format.py":
"ddf4dae07bfad22bbf13f4b14e43d8f4",
"ANGLE format:src/libANGLE/renderer/angle_format_data.json":
"3513511d9f0c7afd9dbc33d1d0de5987",
"ANGLE format:src/libANGLE/renderer/angle_format_map.json":
"28aeb36a9cdb5262702d9f1e8b81b8ed",
"ANGLE format:src/libANGLE/renderer/gen_angle_format_table.py":
"aa268de9ae7424a83633656b97d8161d",
"ANGLE load functions table:src/libANGLE/renderer/gen_load_functions_table.py":
"de1532286bb6720c309f673a9630a132",
"ANGLE load functions table:src/libANGLE/renderer/load_functions_data.json":
"976cac5f0dcb9b2a782b1b257819b71c",
"D3D11 format:src/libANGLE/renderer/angle_format.py":
"ddf4dae07bfad22bbf13f4b14e43d8f4",
"D3D11 format:src/libANGLE/renderer/d3d/d3d11/gen_texture_format_table.py":
"2a6afa5c7381b3cba46bf4e19146511c",
"D3D11 format:src/libANGLE/renderer/d3d/d3d11/texture_format_data.json":
"27c0b8debb6d5f610ad4f868d0b57aee",
"D3D11 format:src/libANGLE/renderer/d3d/d3d11/texture_format_map.json":
"c60bd3863f0964ea6c3eee7d62d7fab2",
"DXGI format support:src/libANGLE/renderer/d3d/d3d11/dxgi_support_data.json":
"ed2631ebf8535890c03433a264bfe9b7",
"DXGI format support:src/libANGLE/renderer/d3d/d3d11/gen_dxgi_support_tables.py":
"e5073223e0338d625a3ec3adb743fc1e",
"DXGI format:src/libANGLE/renderer/angle_format.py":
"ddf4dae07bfad22bbf13f4b14e43d8f4",
"DXGI format:src/libANGLE/renderer/angle_format_map.json":
"28aeb36a9cdb5262702d9f1e8b81b8ed",
"DXGI format:src/libANGLE/renderer/d3d/d3d11/dxgi_format_data.json":
"3607db87d772bddcfe8f7e4a247046fc",
"DXGI format:src/libANGLE/renderer/d3d/d3d11/gen_dxgi_format_table.py":
"abe278e3e7d68b53ea44eabad7360954",
"ESSL static builtins:src/compiler/translator/builtin_function_declarations.txt":
"b8b75e150cea623fdb7503e196fc67ef",
"ESSL static builtins:src/compiler/translator/builtin_variables.json":
"fd8612ea296fe89f9a0c132018569753",
"ESSL static builtins:src/compiler/translator/gen_builtin_symbols.py":
"7f75b265cc4aeef2f9585348aa1f1267",
"Emulated HLSL functions:src/compiler/translator/emulated_builtin_function_data_hlsl.json":
"6c908980bdfe28569d8f148326e748a4",
"Emulated HLSL functions:src/compiler/translator/gen_emulated_builtin_function_tables.py":
"b43ad00aaca29f278b78986df2705f53",
"GL copy conversion table:src/libANGLE/es3_copy_conversion_formats.json":
"9ebd7404a10e4069485258db8fe3dffc",
"GL copy conversion table:src/libANGLE/gen_copy_conversion_table.py":
"5218d44a5c02b7ac98599a7262375cc7",
"GL entry point:scripts/entry_point_packed_gl_enums.json":
"d409cb2e59a037e1d5698c6b00829c18",
"GL entry point:scripts/generate_entry_points.py":
"57bea24a1a17f8076ed290c335a3208f",
"GL entry point:scripts/gl.xml":
"521a7e2337f60ba5aa5ae906f5d1a5c4",
"GL format map:src/libANGLE/es3_format_type_combinations.json":
"0ae351748cdcd5fd2e7447416bac64b5",
"GL format map:src/libANGLE/format_map_data.json":
"9a76bd8b82d2417dcd5c1ee2052f77b5",
"GL format map:src/libANGLE/gen_format_map.py":
"87226f5d2224743f941026ec1c9489e5",
"OpenGL dispatch table:scripts/gl.xml":
"521a7e2337f60ba5aa5ae906f5d1a5c4",
"OpenGL dispatch table:src/libANGLE/renderer/gl/generate_gl_dispatch_table.py":
"98855aea9916706b384e8843fc8cc145",
"Vulkan format:src/libANGLE/renderer/angle_format.py":
"ddf4dae07bfad22bbf13f4b14e43d8f4",
"Vulkan format:src/libANGLE/renderer/angle_format_map.json":
"28aeb36a9cdb5262702d9f1e8b81b8ed",
"Vulkan format:src/libANGLE/renderer/vulkan/gen_vk_format_table.py":
"be1ca0bd596cf826d8f38d110fba35b0",
"Vulkan format:src/libANGLE/renderer/vulkan/vk_format_map.json":
"8b02327195f831c782b0a30b36b589d6",
"Vulkan internal shader programs:C:\\src\\angle\\src\\libANGLE\\renderer\\vulkan\\shaders\\src\\FullScreenQuad.vert":
"ab897aa9eb45f83f66080ee082e7d51e",
"Vulkan internal shader programs:C:\\src\\angle\\src\\libANGLE\\renderer\\vulkan\\shaders\\src\\PushConstantColor.frag":
"3b931f51bfee8310f3d6497d5c9ed8a1",
"Vulkan internal shader programs:src/libANGLE/renderer/vulkan/gen_vk_internal_shaders.py":
"d651e28ab62477029093152e37ed49c9",
"Vulkan mandatory format support table:src/libANGLE/renderer/angle_format.py":
"ddf4dae07bfad22bbf13f4b14e43d8f4",
"Vulkan mandatory format support table:src/libANGLE/renderer/vulkan/gen_vk_mandatory_format_support_table.py":
"9f8beddd1c293787c851bb9dc42552cb",
"Vulkan mandatory format support table:src/libANGLE/renderer/vulkan/vk_mandatory_format_support_data.json":
"f9faa72f7d2b926e4eca5cc0ffb73feb",
"Vulkan mandatory format support table:third_party/vulkan-headers/src/registry/vk.xml":
"147f8153623cd74b26a0ceba90afbc86",
"packed enum:src/common/gen_packed_gl_enums.py":
"878d244fb50d4163adcbfbce8a8c660c",
"packed enum:src/common/packed_egl_enums.json":
"4921ce9fc375dee01fe839e5f25b4a06",
"packed enum:src/common/packed_gl_enums.json":
"44b4eb27ff237a096246e5fb31c64958",
"proc table:src/libGLESv2/gen_proc_table.py":
"69518f0b2e34331f8c911e2602fe049d",
"proc table:src/libGLESv2/proc_table_data.json":
"52e9cae4a47cb00ee68633275603c6b4",
"uniform type:src/common/gen_uniform_type_table.py":
"e892048b2185e3e661b8ba9c541834cc"
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment