Infrastructure for uploading flex/bison binaries

.gitignore

@@ -54,6 +54,12 @@
 /third_party/yasm
 /third_party/zlib
 /tools/clang
+/tools/flex-bison/linux/bison
+/tools/flex-bison/linux/flex
+/tools/flex-bison/windows/bison.exe
+/tools/flex-bison/windows/flex.exe
+/tools/flex-bison/windows/m4.exe
+/tools/flex-bison/windows/msys*.dll
 /tools/glslang/glslang_validator
 /tools/glslang/glslang_validator.exe
 /tools/md_browser

tools/angle_tools.py

+#!/usr/bin/python2
+#
+# Copyright 2019 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# angle_tools.py:
+#   Common functionality to scripts in angle/tools directory.
+
+import os
+import platform
+import subprocess
+
+is_windows = platform.system() == 'Windows'
+is_linux = platform.system() == 'Linux'
+
+
+def find_file_in_path(filename):
+    """ Finds |filename| by searching the environment paths """
+    path_delimiter = ';' if is_windows else ':'
+    for env_path in os.environ['PATH'].split(path_delimiter):
+        full_path = os.path.join(env_path, filename)
+        if os.path.isfile(full_path):
+            return full_path
+    raise Exception('Cannot find %s in environment' % filename)
+
+
+def get_exe_name(file_name, windows_extension):
+    exe_name = file_name
+    if is_windows:
+        exe_name += windows_extension
+    return exe_name
+
+
+def upload_to_google_storage(bucket, files):
+    upload_script = find_file_in_path('upload_to_google_storage.py')
+    upload_args = ['python', upload_script, '-b', bucket] + files
+    return subprocess.call(upload_args) == 0
+
+
+def stage_google_storage_sha1(files):
+    git_exe = get_exe_name('git', '.bat')
+    git_exe = find_file_in_path(git_exe)
+
+    sha1_files = [f + '.sha1' for f in files]
+    return subprocess.call([git_exe, 'add'] + sha1_files) == 0

tools/flex-bison/README.md

+# flex and bison binaries
+
+This folder contains the flex and bison binaries. We use these binaries to
+generate the ANGLE translator's lexer and parser.
+
+Use the script [`update_flex_bison_binaries.py`](update_flex_bison_binaries.py)
+to update the versions of these binaries in cloud storage. It must be run on
+Linux or Windows. It will update the SHAs for your platform. After running the
+script run `git commit` and then `git cl upload` to code review using the normal
+review process. You will also want to run
+[`scripts/run_code_generation.py`](../../scripts/run_code_generation.py) to
+update the generated files.
+
+Please update both Windows and Linux binaries at the same time. Use two CLs. One
+for each platform. Note that we don't currently support Mac for generating the
+lexer and parser files. If we do we should add a flex/bison download for Mac as
+well.
+
+Contact jmadill or syoussefi for any help with updating the binaries.
+
+## Updating flex and bison binaries
+
+This is expected to be a rare operation, and is currently done based on the
+following instructions.  Note: get the binaries first on windows, as only a
+single option is available, then build the binaries for the same version on
+Linux.
+
+### On Windows
+
+Install MSys2 (x86_64) from http://www.msys2.org/ on Windows.  `flex` should
+already be installed.  Install bison:
+
+```
+$ pacman -S bison
+```
+
+Note the versions of flex and bison so the same versions can be build on Linux.
+For example:
+
+```
+$ flex --version
+flex 2.6.4
+
+$ bison --version
+bison (GNU Bison) 3.3.2
+```
+
+The only dependencies from MSys2 should be the following:
+
+```
+msys-intl-8.dll
+msys-iconv-2.dll
+msys-2.0.dll
+```
+
+This can be verified with:
+
+```
+$ ldd /usr/bin/flex
+$ ldd /usr/bin/bison
+```
+
+Additionally, we need the binary for m4 at `/usr/bin/m4`.
+
+Copy all these 5 files to this directory:
+
+```
+$ cd angle/
+$ cp /usr/bin/flex.exe \
+     /usr/bin/bison.exe \
+     /usr/bin/m4.exe \
+     /usr/bin/msys-intl-8.dll \
+     /usr/bin/msys-iconv-2.dll \
+     /usr/bin/msys-2.0.dll \
+     tools/flex-bison/windows/
+```
+
+Upload the binaries:
+
+```
+$ cd angle/
+$ python tools/flex-bison/update_flex_bison_binaries.py
+```
+
+### On Linux
+
+```
+# Get the source of flex
+$ git clone https://github.com/westes/flex.git
+$ cd flex/
+# Checkout the same version as msys2 on windows
+$ git checkout v2.6.4
+# Build
+$ autoreconf -i
+$ mkdir build && cd build
+$ ../configure CFLAGS="-O2 -D_GNU_SOURCE"
+$ make -j
+```
+
+```
+# Get the source of bison
+$ curl http://ftp.gnu.org/gnu/bison/bison-3.3.2.tar.xz | tar -xJ
+$ cd bison-3.3.2
+# Build
+$ mkdir build && cd build
+$ ../configure CFLAGS="-O2"
+$ make -j
+```
+
+Note: Bison's [home page][Bison] lists ftp server and other mirrors.  If the
+above link is broken, replace with a mirror.
+
+Copy the 2 executables to this directory:
+
+```
+$ cd angle/
+$ cp /path/to/flex/build/src/flex \
+     /path/to/bison/build/src/bison \
+     tools/flex-bison/linux/
+```
+
+Upload the binaries:
+
+```
+$ cd angle/
+$ python tools/flex-bison/update_flex_bison_binaries.py
+```
+
+[Bison]: https://www.gnu.org/software/bison/

tools/flex-bison/update_flex_bison_binaries.py

+#!/usr/bin/python2
+#
+# Copyright 2019 The ANGLE Project Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# update_flex_bison_binaries.py:
+#   Helper script to update the version of flex and bison in cloud storage.
+#   These binaries are used to generate the ANGLE translator's lexer and parser.
+#   This script relies on flex and bison binaries to be externally built which
+#   is expected to be a rare operation. It currently only works on Windows and
+#   Linux. It also will update the hashes stored in the tree. For more info see
+#   README.md in this folder.
+
+import os
+import platform
+import shutil
+import sys
+
+sys.path.append('tools/')
+import angle_tools
+
+
+def main():
+    if angle_tools.is_linux:
+        subdir = 'linux'
+        files = ['flex', 'bison']
+    elif angle_tools.is_windows:
+        subdir = 'windows'
+        files = [
+            'flex.exe', 'bison.exe', 'm4.exe', 'msys-2.0.dll', 'msys-iconv-2.dll',
+            'msys-intl-8.dll'
+        ]
+    else:
+        print('Script must be run on Linux or Windows.')
+        return 1
+
+    files = [os.path.join(sys.path[0], subdir, f) for f in files]
+
+    # Step 1: Upload to cloud storage
+    if not angle_tools.upload_to_google_storage('angle-flex-bison', files):
+        print('Error upload to cloud storage')
+        return 1
+
+    # Step 2: Stage new SHA to git
+    if not angle_tools.stage_google_storage_sha1(files):
+        print('Error running git add')
+        return 2
+
+    print('')
+    print('The updated SHA has been staged for commit. Please commit and upload.')
+    print('Suggested commit message (please indicate flex/bison versions):')
+    print('----------------------------')
+    print('')
+    print('Update flex and bison binaries for %s.' % platform.system())
+    print('')
+    print('This binary was updated using %s.' % os.path.basename(__file__))
+    print('Please see instructions in tools/flex-bison/README.md.')
+    print('')
+    print('flex is at version TODO.')
+    print('bison is at version TODO.')
+    print('')
+    print('Bug: None')
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/glslang/README.md

@@ -17,4 +17,4 @@ Please update both Windows and Linux binaries at the same time. Use two CLs. One
 for each platform. Note that we don't currently support Mac on Vulkan. If we do
 we should add a glslang download for Mac as well.
 
-Contact jmadill or syouseffi for any help with the validator or updating the binaries.
+Contact jmadill or syoussefi for any help with the validator or updating the binaries.

tools/glslang/update_glslang_binary.py

@@ -13,31 +13,20 @@
 
 import os
 import platform
-import re
 import shutil
 import subprocess
 import sys
 
+sys.path.append('tools/')
+import angle_tools
+
 gn_args = """is_clang = true
 is_debug = false
 angle_enable_vulkan = true"""
 
-is_windows = platform.system() == 'Windows'
-is_linux = platform.system() == 'Linux'
-
-
-def find_file_in_path(filename):
-    """ Finds |filename| by searching the environment paths """
-    path_delimiter = ';' if is_windows else ':'
-    for env_path in os.environ['PATH'].split(path_delimiter):
-        full_path = os.path.join(env_path, filename)
-        if os.path.isfile(full_path):
-            return full_path
-    raise Exception('Cannot find %s in environment' % filename)
-
 
 def main():
-    if not is_windows and not is_linux:
+    if not angle_tools.is_windows and not angle_tools.is_linux:
         print('Script must be run on Linux or Windows.')
         return 1
 
@@ -53,9 +42,7 @@ def main():
             f.write(gn_args)
             f.close()
 
-    gn_exe = 'gn'
-    if is_windows:
-        gn_exe += '.bat'
+    gn_exe = angle_tools.get_exe_name('gn', '.bat')
 
     # Step 2: Generate the ninja build files in the output directory
     if subprocess.call([gn_exe, 'gen', out_dir]) != 0:
@@ -68,9 +55,7 @@ def main():
         return 3
 
     # Step 4: Copy glslang_validator to the tools/glslang directory
-    glslang_exe = 'glslang_validator'
-    if is_windows:
-        glslang_exe += '.exe'
+    glslang_exe = angle_tools.get_exe_name('glslang_validator', '.exe')
 
     glslang_src = os.path.join(out_dir, glslang_exe)
     glslang_dst = os.path.join(sys.path[0], glslang_exe)
@@ -81,19 +66,12 @@ def main():
     shutil.rmtree(out_dir)
 
     # Step 6: Upload to cloud storage
-    upload_script = find_file_in_path('upload_to_google_storage.py')
-    upload_args = ['python', upload_script, '-b', 'angle-glslang-validator', glslang_dst]
-    if subprocess.call(upload_args) != 0:
+    if not angle_tools.upload_to_google_storage('angle-glslang-validator', [glslang_dst]):
         print('Error upload to cloud storage')
         return 4
 
     # Step 7: Stage new SHA to git
-    git_exe = 'git'
-    if is_windows:
-        git_exe += '.bat'
-    git_exe = find_file_in_path(git_exe)
-
-    if subprocess.call([git_exe, 'add', glslang_dst + '.sha1']) != 0:
+    if not angle_tools.stage_google_storage_sha1([glslang_dst]):
         print('Error running git add')
         return 5