Squashed 'third_party/marl/' changes from d3b8558ce..59068ee4c

CMakeLists.txt

@@ -21,14 +21,15 @@ project(Marl C CXX ASM)
 ###########################################################
 # Options
 ###########################################################
+option(MARL_WARNINGS_AS_ERRORS "Treat warnings as errors" OFF)
 option(MARL_BUILD_EXAMPLES "Build example applications" OFF)
-option(MARL_BUILD_TESTS "Build tests" ON)
+option(MARL_BUILD_TESTS "Build tests" OFF)
 option(MARL_ASAN "Build marl with address sanitizer" OFF)
 option(MARL_TSAN "Build marl with thread sanitizer" OFF)
 
-if (MARL_ASAN AND MARL_TSAN)
+if(MARL_ASAN AND MARL_TSAN)
     message(FATAL_ERROR "MARL_ASAN and MARL_TSAN are mutually exclusive")
-endif (MARL_ASAN AND MARL_TSAN)
+endif(MARL_ASAN AND MARL_TSAN)
 
 ###########################################################
 # Directories
@@ -52,21 +53,40 @@ endif(MARL_BUILD_TESTS)
 ###########################################################
 # File lists
 ###########################################################
-file(GLOB MARL_FULL_LIST
-    ${MARL_SRC_DIR}/*.cpp
-    ${MARL_SRC_DIR}/*.h
-    ${MARL_SRC_DIR}/*.c
+file(GLOB MARL_LIST
+    ${MARL_SRC_DIR}/debug.cpp
+    ${MARL_SRC_DIR}/scheduler.cpp
+    ${MARL_SRC_DIR}/thread.cpp
+    ${MARL_SRC_DIR}/trace.cpp
+    ${MARL_SRC_DIR}/osfiber_aarch64.c
+    ${MARL_SRC_DIR}/osfiber_arm.c
+    ${MARL_SRC_DIR}/osfiber_ppc64.c
+    ${MARL_SRC_DIR}/osfiber_x64.c
+    ${MARL_SRC_DIR}/osfiber_x86.c
 )
-
-if (NOT MSVC)
-    file(GLOB MARL_ASSEMBLY_LIST ${MARL_SRC_DIR}/*.S)
-    list(APPEND MARL_FULL_LIST ${MARL_ASSEMBLY_LIST})
+if(NOT MSVC)
+    list(APPEND MARL_LIST
+        ${MARL_SRC_DIR}/osfiber_asm_aarch64.S
+        ${MARL_SRC_DIR}/osfiber_asm_arm.S
+        ${MARL_SRC_DIR}/osfiber_asm_ppc64.S
+        ${MARL_SRC_DIR}/osfiber_asm_x64.S
+        ${MARL_SRC_DIR}/osfiber_asm_x86.S
+    )
 endif(NOT MSVC)
 
-set(MARL_LIST ${MARL_FULL_LIST})
-set(MARL_TEST_LIST ${MARL_FULL_LIST})
-list(FILTER MARL_LIST EXCLUDE REGEX ".*_test\\..*")
-list(FILTER MARL_TEST_LIST INCLUDE REGEX ".*_test\\..*")
+file(GLOB MARL_TEST_LIST
+    ${MARL_SRC_DIR}/blockingcall_test.cpp
+    ${MARL_SRC_DIR}/conditionvariable_test.cpp
+    ${MARL_SRC_DIR}/containers_test.cpp
+    ${MARL_SRC_DIR}/defer_test.cpp
+    ${MARL_SRC_DIR}/marl_test.cpp
+    ${MARL_SRC_DIR}/marl_test.h
+    ${MARL_SRC_DIR}/osfiber_test.cpp
+    ${MARL_SRC_DIR}/pool_test.cpp
+    ${MARL_SRC_DIR}/scheduler_test.cpp
+    ${MARL_SRC_DIR}/ticket_test.cpp
+    ${MARL_SRC_DIR}/waitgroup_test.cpp
+)
 
 ###########################################################
 # OS libraries
@@ -80,23 +100,51 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
 endif()
 
 ###########################################################
+# Functions
+###########################################################
+
+function(marl_set_target_options target)
+    # Enable all warnings
+    if(NOT MSVC)
+        target_compile_options(${target} PRIVATE "-Wall")
+    endif()
+
+    # Disable specific, pedantic warnings
+    if(MSVC)
+        target_compile_options(${target} PRIVATE "-D_CRT_SECURE_NO_WARNINGS")
+    endif()
+
+    # Treat all warnings as errors
+    if(MARL_WARNINGS_AS_ERRORS)
+        if(MSVC)
+            target_compile_options(${target} PRIVATE "/WX")
+        else()
+            target_compile_options(${target} PRIVATE "-Werror")
+        endif()
+    endif(MARL_WARNINGS_AS_ERRORS)
+
+    if(MARL_ASAN)
+        target_compile_options(${target} PUBLIC "-fsanitize=address")
+        target_link_libraries(${target} "-fsanitize=address")
+    elseif(MARL_MSAN)
+        target_compile_options(${target} PUBLIC "-fsanitize=memory")
+        target_link_libraries(${target} "-fsanitize=memory")
+    endif()
+
+    target_include_directories(${target} PRIVATE ${MARL_INCLUDE_DIR})
+endfunction(marl_set_target_options)
+
+###########################################################
 # Targets
 ###########################################################
 
 # marl
 add_library(marl STATIC ${MARL_LIST})
 set_target_properties(marl PROPERTIES
-    INCLUDE_DIRECTORIES "${MARL_INCLUDE_DIR}"
     POSITION_INDEPENDENT_CODE 1
 )
 
-if (MARL_ASAN)
-    target_compile_options(marl PUBLIC "-fsanitize=address")
-    target_link_libraries(marl "-fsanitize=address")
-elseif (MARL_MSAN)
-    target_compile_options(marl PUBLIC "-fsanitize=memory")
-    target_link_libraries(marl "-fsanitize=memory")
-endif ()
+marl_set_target_options(marl)
 
 target_link_libraries(marl "${MARL_OS_LIBS}")
 
@@ -111,7 +159,6 @@ if(MARL_BUILD_TESTS)
         ${GOOGLETEST_DIR}/googletest/include/
         ${GOOGLETEST_DIR}/googlemock/include/
         ${GOOGLETEST_DIR}/googletest/
-        ${CMAKE_CURRENT_SOURCE_DIR}/include
     )
 
     add_executable(marl-unittests ${MARL_TEST_LIST})
@@ -120,20 +167,24 @@ if(MARL_BUILD_TESTS)
         INCLUDE_DIRECTORIES "${MARL_TEST_INCLUDE_DIR}"
         FOLDER "Tests"
     )
+
+    marl_set_target_options(marl-unittests)
+
     target_link_libraries(marl-unittests marl "${MARL_OS_LIBS}")
 endif(MARL_BUILD_TESTS)
 
 # examples
 if(MARL_BUILD_EXAMPLES)
-    function(BUILD_EXAMPLE name)
-        add_executable(${name} "${CMAKE_CURRENT_SOURCE_DIR}/examples/${name}.cpp")
-        set_target_properties(${name} PROPERTIES
-            INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    function(build_example target)
+        add_executable(${target} "${CMAKE_CURRENT_SOURCE_DIR}/examples/${target}.cpp")
+        set_target_properties(${target} PROPERTIES
             FOLDER "Examples"
         )
-        target_link_libraries(${name} marl "${MARL_OS_LIBS}")
-    endfunction(BUILD_EXAMPLE)
+        marl_set_target_options(${target})
+        target_link_libraries(${target} marl "${MARL_OS_LIBS}")
+    endfunction(build_example)
 
-    BUILD_EXAMPLE(fractal)
+    build_example(fractal)
+    build_example(primes)
 
 endif(MARL_BUILD_EXAMPLES)

README.md

@@ -10,12 +10,59 @@ Marl uses a combination of fibers and threads to allow efficient execution of ta
 
 Marl supports Windows, macOS, Linux, Fuchsia and Android (arm, aarch64, ppc64 (ELFv2), x86 and x64).
 
-Marl has no dependencies on other libraries (with exception on googletest fo building the optional unit tests).
+Marl has no dependencies on other libraries (with exception on googletest for building the optional unit tests).
 
 Marl is in early development and will have breaking API changes.
 
-
 **More documentation and examples coming soon.**
 
-
 Note: This is not an officially supported Google product
+
+## Building
+
+Marl contains a number of unit tests and examples which can be built using CMake.
+
+Unit tests require fetching the `googletest` external project, which can be done by typing the following in your terminal:
+
+```bash
+cd <path-to-marl>
+git submodule update --init
+```
+
+### Linux and macOS
+
+To build the unit tests and examples, type the following in your terminal:
+
+```bash
+cd <path-to-marl>
+mkdir build
+cd build
+cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1
+make
+```
+
+The resulting binaries will be found in `<path-to-marl>/build`
+
+### Windows
+
+Marl can be build using [Visual Studio 2019's CMake integration](https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=vs-2019).
+
+### Using Marl in your CMake project
+
+You can build and link Marl using `add_subdirectory()` in your project's `CMakeLists.txt` file:
+```cmake
+set(MARL_DIR <path-to-marl>) # example <path-to-marl>: "${CMAKE_CURRENT_SOURCE_DIR}/third_party/marl"
+add_subdirectory(${MARL_DIR})
+```
+
+This will define the `marl` library target, which you can pass to `target_link_libraries()`:
+
+```cmake
+target_link_libraries(<target> marl) # replace <target> with the name of your project's target
+```
+
+You will also want to add the `marl` public headers to your project's include search paths so you can `#include` the marl headers:
+
+```cmake
+target_include_directories($<target> PRIVATE "${MARL_DIR}/include") # replace <target> with the name of your project's target
+```

examples/fractal.cpp

@@ -12,13 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// This is an example application that uses Marl to parallelize the calculation of
-// a Julia fractal.
+// This is an example application that uses Marl to parallelize the calculation
+// of a Julia fractal.
 
-#include <marl/defer.h>
-#include <marl/scheduler.h>
-#include <marl/thread.h>
-#include <marl/waitgroup.h>
+#include "marl/defer.h"
+#include "marl/scheduler.h"
+#include "marl/thread.h"
+#include "marl/waitgroup.h"
 
 #include <fstream>
 
@@ -64,10 +64,9 @@ inline float lerp(float x, float min, float max) {
 // julia calculates the Julia-set fractal value for the given coordinate and
 // constant. See https://en.wikipedia.org/wiki/Julia_set for more information.
 Color<float> julia(float x, float y, float cx, float cy) {
-  int iteration = 0;
   for (int i = 0; i < 1000; i++) {
     if (x * x + y * y > 4) {
-      return colorize(sqrt(i));
+      return colorize(sqrtf(static_cast<float>(i)));
     }
 
     auto xtemp = x * x - y * y;
@@ -99,7 +98,6 @@ bool writeBMP(const Color<uint8_t>* texels,
   const uint32_t padding = -(3 * width) & 3U;   // in bytes
   const uint32_t stride = 3 * width + padding;  // in bytes
   const uint32_t offset = 54;
-  const uint32_t size = offset + stride * height * 3;
 
   // Bitmap file header
   put1('B');  // header field
@@ -128,7 +126,7 @@ bool writeBMP(const Color<uint8_t>* texels,
       put1(texel.g);
       put1(texel.r);
     }
-    for (int i = 0; i < padding; i++) {
+    for (uint32_t i = 0; i < padding; i++) {
       put1(0);
     }
   }
@@ -139,7 +137,8 @@ bool writeBMP(const Color<uint8_t>* texels,
 // Constants used for rendering the fractal.
 constexpr uint32_t imageWidth = 2048;
 constexpr uint32_t imageHeight = 2048;
-constexpr int samplesPerPixel = 8;
+constexpr int samplesPerPixelW = 3;
+constexpr int samplesPerPixelH = 3;
 constexpr float windowMinX = -0.5f;
 constexpr float windowMaxX = +0.5f;
 constexpr float windowMinY = -0.5f;
@@ -165,7 +164,7 @@ int main(int argc, const char** argv) {
   marl::WaitGroup wg(imageHeight);
 
   // For each line of the image...
-  for (int y = 0; y < imageHeight; y++) {
+  for (uint32_t y = 0; y < imageHeight; y++) {
     // Schedule a task to calculate the image for this line.
     // These may run concurrently across hardware threads.
     marl::schedule([=] {
@@ -173,18 +172,20 @@ int main(int argc, const char** argv) {
       // This is used to indicate that the task is done.
       defer(wg.done());
 
-      for (int x = 0; x < imageWidth; x++) {
+      for (uint32_t x = 0; x < imageWidth; x++) {
         // Calculate the fractal pixel color.
         Color<float> color = {};
-        for (int sample = 0; sample < samplesPerPixel; sample++) {
-          auto fx = float(x) + (rand() / float(RAND_MAX));
-          auto fy = float(y) + (rand() / float(RAND_MAX));
-          auto dx = float(fx) / float(imageWidth);
-          auto dy = float(fy) / float(imageHeight);
-          color += julia(lerp(dx, windowMinX, windowMaxX),
-                         lerp(dy, windowMinY, windowMaxY), cx, cy);
+        for (int sy = 0; sy < samplesPerPixelH; sy++) {
+          auto fy = float(y) + (sy / float(samplesPerPixelH));
+          for (int sx = 0; sx < samplesPerPixelW; sx++) {
+            auto fx = float(x) + (sx / float(samplesPerPixelW));
+            auto dx = float(fx) / float(imageWidth);
+            auto dy = float(fy) / float(imageHeight);
+            color += julia(lerp(dx, windowMinX, windowMaxX),
+                           lerp(dy, windowMinY, windowMaxY), cx, cy);
+          }
         }
-        color /= samplesPerPixel;
+        color /= samplesPerPixelW * samplesPerPixelH;
         pixels[x + y * imageWidth] = {static_cast<uint8_t>(color.r * 255),
                                       static_cast<uint8_t>(color.g * 255),
                                       static_cast<uint8_t>(color.b * 255)};

examples/primes.cpp

+// Copyright 2019 The Marl Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is an example application that uses Marl to find and print all the prime
+// numbers in the range 1 to 10000000.
+
+#include "marl/defer.h"
+#include "marl/scheduler.h"
+#include "marl/thread.h"
+#include "marl/ticket.h"
+
+// searchMax defines the upper limit on primes to find.
+constexpr int searchMax = 10000000;
+
+// searchChunkSize is the number of numbers searched, per task, for primes.
+constexpr int searchChunkSize = 10000;
+
+// isPrime returns true if i is prime.
+bool isPrime(int i) {
+  auto c = static_cast<int>(sqrt(i));
+  for (int j = 2; j <= c; j++) {
+    if (i % j == 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+int main(int argc, const char** argv) {
+  // Create a marl scheduler using the full number of logical cpus.
+  // Bind this scheduler to the main thread so we can call marl::schedule()
+  marl::Scheduler scheduler;
+  scheduler.setWorkerThreadCount(marl::Thread::numLogicalCPUs());
+  scheduler.bind();
+  defer(scheduler.unbind());  // unbind before destructing the scheduler.
+
+  // Create a ticket queue. This will be used to ensure that the primes are
+  // printed in ascending order.
+  marl::Ticket::Queue queue;
+
+  // Iterate over the range [1, searchMax] in steps of searchChunkSize.
+  for (int searchBase = 1; searchBase <= searchMax;
+       searchBase += searchChunkSize) {
+    // Take a ticket from the ticket queue for this task.
+    auto ticket = queue.take();
+
+    // Schedule the task.
+    marl::schedule([=] {
+      // Find all the primes in [searchBase, searchBase+searchChunkSize-1].
+      // Note this is run in parallel with the other scheduled tasks.
+      std::vector<int> primes;
+      for (int i = searchBase; i < searchBase + searchChunkSize; i++) {
+        if (isPrime(i)) {
+          primes.push_back(i);
+        }
+      }
+
+      // Wait until the ticket is called. This ensures that the primes are
+      // printed in ascending order. This may cause this task to yield and allow
+      // other tasks to be executed while waiting for this ticket to be called.
+      ticket.wait();
+
+      // Print the primes.
+      for (auto prime : primes) {
+        printf("%d is prime\n", prime);
+      }
+
+      // Call the next ticket so that those primes can be printed.
+      ticket.done();
+    });
+  }
+
+  // take a ticket and wait on it to ensure that all the primes have been
+  // calculated and printed.
+  queue.take().wait();
+
+  return 0;
+}

kokoro/linux/presubmit.sh

@@ -11,7 +11,7 @@ mkdir build
 cd build
 
 build_and_run() {
-    cmake .. -DMARL_BUILD_EXAMPLES=1 $1
+    cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_WARNINGS_AS_ERRORS=1 $1
     make --jobs=$(nproc)
 
     ./marl-unittests

kokoro/macos/presubmit.sh

@@ -9,7 +9,7 @@ git submodule update --init
 
 mkdir build
 cd build
-cmake .. -DMARL_BUILD_EXAMPLES=1
+cmake .. -DMARL_BUILD_EXAMPLES=1 -DMARL_BUILD_TESTS=1 -DMARL_WARNINGS_AS_ERRORS=1
 make -j$(sysctl -n hw.logicalcpu)
 
 ./marl-unittests

kokoro/windows/presubmit.bat

@@ -18,7 +18,7 @@ mkdir %SRC%\build
 cd %SRC%\build
 if !ERRORLEVEL! neq 0 exit /b !ERRORLEVEL!
 
-cmake .. -G "Visual Studio 15 2017 Win64" -Thost=x64 "-DMARL_BUILD_EXAMPLES=1"
+cmake .. -G "Visual Studio 15 2017 Win64" -Thost=x64 "-DMARL_BUILD_TESTS=1" "-DMARL_BUILD_EXAMPLES=1" "-DMARL_WARNINGS_AS_ERRORS=1"
 if !ERRORLEVEL! neq 0 exit /b !ERRORLEVEL!
 
 %MSBUILD% /p:Configuration=%CONFIG% Marl.sln

src/scheduler.cpp

@@ -476,7 +476,7 @@ _Requires_lock_held_(lock) void Scheduler::Worker::runUntilIdle(
 }
 
 Scheduler::Fiber* Scheduler::Worker::createWorkerFiber() {
-  auto id = workerFibers.size() + 1;
+  auto id = static_cast<int32_t>(workerFibers.size() + 1);
   auto fiber = Fiber::create(id, FiberStackSize, [&] { run(); });
   workerFibers.push_back(std::unique_ptr<Fiber>(fiber));
   return fiber;