Commit 7d53c60b by jchen10 Committed by Commit Bot

ParallelCompile: use the native extensions

This enhances to use the native parallel compile extensions if available. Bug: 873724 Change-Id: I0aaed314accd75e1bfa596b322225b56d729d3a6 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1475234 Commit-Queue: Jie A Chen <jie.a.chen@intel.com> Reviewed-by: 's avatarJamie Madill <jmadill@chromium.org>
parent 0e2c39f2
......@@ -170,17 +170,17 @@
"OpenGL dispatch table:src/libANGLE/renderer/angle_format.py":
"b18ca0fe4835114a4a2f54977b19e798",
"OpenGL dispatch table:src/libANGLE/renderer/gl/DispatchTableGL_autogen.cpp":
"6556e48f03112aaf0e4d0fa2949727b4",
"2ed6500b69d5db151348ddccf6d9b2b8",
"OpenGL dispatch table:src/libANGLE/renderer/gl/DispatchTableGL_autogen.h":
"caea949141a8c6b8692f1c021fb4fa42",
"4211cf4afa7d4a0f8a95de39db45a55e",
"OpenGL dispatch table:src/libANGLE/renderer/gl/generate_gl_dispatch_table.py":
"86a66ba63f6dceac553d8af6c132b6fb",
"7571edb9e610891ed0c95dc496120cff",
"OpenGL dispatch table:src/libANGLE/renderer/gl/gl_bindings_data.json":
"71079f089335ce1f67835d67a6d49d1a",
"6cd07767ecf12d24d2d3572194c3323b",
"OpenGL dispatch table:src/libANGLE/renderer/gl/null_functions.cpp":
"774c21cf434656bb40735cc67fb4fa40",
"f10d2f520f2ceae32ad8e65322fdb0a5",
"OpenGL dispatch table:src/libANGLE/renderer/gl/null_functions.h":
"594f92ec5ffaaf21409579009f579cd7",
"638724bfa3c1e926e93fb97a84b7617f",
"Vulkan format:src/libANGLE/renderer/angle_format.py":
"b18ca0fe4835114a4a2f54977b19e798",
"Vulkan format:src/libANGLE/renderer/angle_format_map.json":
......
......@@ -7954,6 +7954,7 @@ void Context::maxShaderCompilerThreads(GLuint count)
mThreadPool = angle::WorkerThreadPool::Create(count > 0);
}
mThreadPool->setMaxThreads(count);
mImplementation->setMaxShaderCompilerThreads(count);
}
bool Context::isGLES1() const
......
......@@ -20,7 +20,6 @@
#include "libANGLE/Constants.h"
#include "libANGLE/Context.h"
#include "libANGLE/ResourceManager.h"
#include "libANGLE/WorkerThread.h"
#include "libANGLE/renderer/GLImplFactory.h"
#include "libANGLE/renderer/ShaderImpl.h"
......@@ -108,47 +107,10 @@ class ScopedExit final : angle::NonCopyable
std::function<void()> mExit;
};
using CompileImplFunctor = std::function<void(const std::string &, std::string &)>;
class CompileTask : public angle::Closure
struct Shader::CompilingState
{
public:
CompileTask(ShHandle handle,
std::string &&sourcePath,
std::string &&source,
ShCompileOptions options,
CompileImplFunctor &&functor)
: mHandle(handle),
mSourcePath(sourcePath),
mSource(source),
mOptions(options),
mCompileImplFunctor(functor),
mResult(false)
{}
void operator()() override
{
std::vector<const char *> srcStrings;
if (!mSourcePath.empty())
{
srcStrings.push_back(mSourcePath.c_str());
}
srcStrings.push_back(mSource.c_str());
mResult = sh::Compile(mHandle, &srcStrings[0], srcStrings.size(), mOptions);
if (mResult)
{
mCompileImplFunctor(sh::GetObjectCode(mHandle), mInfoLog);
}
}
bool getResult() { return mResult; }
const std::string &getInfoLog() { return mInfoLog; }
private:
ShHandle mHandle;
std::string mSourcePath;
std::string mSource;
ShCompileOptions mOptions;
CompileImplFunctor mCompileImplFunctor;
bool mResult;
std::string mInfoLog;
std::shared_ptr<rx::WaitableCompileEvent> compileEvent;
ShCompilerInstance shCompilerInstance;
};
ShaderState::ShaderState(ShaderType shaderType)
......@@ -359,17 +321,10 @@ void Shader::compile(const Context *context)
mState.mCompileStatus = CompileStatus::COMPILE_REQUESTED;
mBoundCompiler.set(context, context->getCompiler());
// Cache the compile source and options for compilation. Must be done now, since the source
// can change before the link call or another call that resolves the compile.
std::stringstream sourceStream;
std::string sourcePath;
ShCompileOptions options =
mImplementation->prepareSourceAndReturnOptions(context, &sourceStream, &sourcePath);
options |= (SH_OBJECT_CODE | SH_VARIABLES | SH_EMULATE_GL_DRAW_ID);
auto source = sourceStream.str();
ShCompileOptions options = (SH_OBJECT_CODE | SH_VARIABLES | SH_EMULATE_GL_DRAW_ID);
// Add default options to WebGL shaders to prevent unexpected behavior during compilation.
// Add default options to WebGL shaders to prevent unexpected behavior during
// compilation.
if (context->getExtensions().webglCompatibility)
{
options |= SH_INIT_GL_POSITION;
......@@ -379,9 +334,9 @@ void Shader::compile(const Context *context)
options |= SH_INIT_SHARED_VARIABLES;
}
// Some targets (eg D3D11 Feature Level 9_3 and below) do not support non-constant loop indexes
// in fragment shaders. Shader compilation will fail. To provide a better error message we can
// instruct the compiler to pre-validate.
// Some targets (eg D3D11 Feature Level 9_3 and below) do not support non-constant loop
// indexes in fragment shaders. Shader compilation will fail. To provide a better error
// message we can instruct the compiler to pre-validate.
if (mRendererLimitations.shadersRequireIndexedLoopValidation)
{
options |= SH_VALIDATE_LOOP_INDEXING;
......@@ -390,27 +345,15 @@ void Shader::compile(const Context *context)
mCurrentMaxComputeWorkGroupInvocations = context->getCaps().maxComputeWorkGroupInvocations;
ASSERT(mBoundCompiler.get());
mShCompilerInstance = mBoundCompiler->getInstance(mState.mShaderType);
ShHandle compilerHandle = mShCompilerInstance.getHandle();
ShCompilerInstance compilerInstance = mBoundCompiler->getInstance(mState.mShaderType);
ShHandle compilerHandle = compilerInstance.getHandle();
ASSERT(compilerHandle);
mCompilerResourcesString = mShCompilerInstance.getBuiltinResourcesString();
mCompilerResourcesString = compilerInstance.getBuiltinResourcesString();
mWorkerPool = context->getWorkerThreadPool();
std::function<void(const std::string &, std::string &)> compileImplFunctor;
if (mWorkerPool->isAsync())
{
compileImplFunctor = [this](const std::string &source, std::string &infoLog) {
mImplementation->compileAsync(source, infoLog);
};
}
else
{
compileImplFunctor = [](const std::string &source, std::string &infoLog) {};
}
mCompileTask =
std::make_shared<CompileTask>(compilerHandle, std::move(sourcePath), std::move(source),
options, std::move(compileImplFunctor));
mCompileEvent = mWorkerPool->postWorkerTask(mCompileTask);
mCompilingState.reset(new CompilingState());
mCompilingState->shCompilerInstance = std::move(compilerInstance);
mCompilingState->compileEvent =
mImplementation->compile(context, &(mCompilingState->shCompilerInstance), options);
}
void Shader::resolveCompile()
......@@ -420,22 +363,20 @@ void Shader::resolveCompile()
return;
}
ASSERT(mCompileEvent.get());
ASSERT(mCompileTask.get());
mCompileEvent->wait();
ASSERT(mCompilingState.get());
mCompileEvent.reset();
mWorkerPool.reset();
mCompilingState->compileEvent->wait();
bool compiled = mCompileTask->getResult();
mInfoLog += mCompileTask->getInfoLog();
mCompileTask.reset();
mInfoLog += mCompilingState->compileEvent->getInfoLog();
ScopedExit exit([this]() { mBoundCompiler->putInstance(std::move(mShCompilerInstance)); });
ScopedExit exit([this]() {
mBoundCompiler->putInstance(std::move(mCompilingState->shCompilerInstance));
mCompilingState->compileEvent.reset();
mCompilingState.reset();
});
ShHandle compilerHandle = mShCompilerInstance.getHandle();
if (!compiled)
ShHandle compilerHandle = mCompilingState->shCompilerInstance.getHandle();
if (!mCompilingState->compileEvent->getResult())
{
mInfoLog += sh::GetInfoLog(compilerHandle);
WARN() << std::endl << mInfoLog;
......@@ -552,7 +493,7 @@ void Shader::resolveCompile()
ASSERT(!mState.mTranslatedSource.empty());
bool success = mImplementation->postTranslateCompile(&mShCompilerInstance, &mInfoLog);
bool success = mCompilingState->compileEvent->postTranslate(&mInfoLog);
mState.mCompileStatus = success ? CompileStatus::COMPILED : CompileStatus::NOT_COMPILED;
}
......@@ -594,7 +535,7 @@ bool Shader::isCompiled()
bool Shader::isCompleted()
{
return (!mState.compilePending() || mCompileEvent->isReady());
return (!mState.compilePending() || mCompilingState->compileEvent->isReady());
}
int Shader::getShaderVersion()
......
......@@ -31,6 +31,7 @@ namespace rx
class GLImplFactory;
class ShaderImpl;
class ShaderSh;
class WaitableCompileEvent;
} // namespace rx
namespace angle
......@@ -189,6 +190,8 @@ class Shader final : angle::NonCopyable, public LabeledObject
const std::string &getCompilerResourcesString() const;
private:
struct CompilingState;
~Shader() override;
static void GetSourceImpl(const std::string &source,
GLsizei bufSize,
......@@ -208,10 +211,7 @@ class Shader final : angle::NonCopyable, public LabeledObject
// We keep a reference to the translator in order to defer compiles while preserving settings.
BindingPointer<Compiler> mBoundCompiler;
ShCompilerInstance mShCompilerInstance;
std::shared_ptr<CompileTask> mCompileTask;
std::shared_ptr<angle::WorkerThreadPool> mWorkerPool;
std::shared_ptr<angle::WaitableEvent> mCompileEvent;
std::unique_ptr<CompilingState> mCompilingState;
std::string mCompilerResourcesString;
ShaderProgramManager *mResourceManager;
......
......@@ -24,6 +24,13 @@ namespace angle
WaitableEvent::WaitableEvent() = default;
WaitableEvent::~WaitableEvent() = default;
void WaitableEventDone::wait() {}
bool WaitableEventDone::isReady()
{
return true;
}
WorkerThreadPool::WorkerThreadPool() = default;
WorkerThreadPool::~WorkerThreadPool() = default;
......@@ -219,4 +226,17 @@ std::shared_ptr<WorkerThreadPool> WorkerThreadPool::Create(bool multithreaded)
return pool;
}
// static
std::shared_ptr<WaitableEvent> WorkerThreadPool::PostWorkerTask(
std::shared_ptr<WorkerThreadPool> pool,
std::shared_ptr<Closure> task)
{
std::shared_ptr<WaitableEvent> event = pool->postWorkerTask(task);
if (event.get())
{
event->setWorkerThreadPool(pool);
}
return event;
}
} // namespace angle
......@@ -21,6 +21,8 @@
namespace angle
{
class WorkerThreadPool;
// A callback function with no return value and no arguments.
class Closure
{
......@@ -41,6 +43,7 @@ class WaitableEvent : angle::NonCopyable
// Peeks whether the event is ready. If ready, wait() will not block.
virtual bool isReady() = 0;
void setWorkerThreadPool(std::shared_ptr<WorkerThreadPool> pool) { mPool = pool; }
template <size_t Count>
static void WaitMany(std::array<std::shared_ptr<WaitableEvent>, Count> *waitables)
......@@ -51,6 +54,17 @@ class WaitableEvent : angle::NonCopyable
(*waitables)[index]->wait();
}
}
private:
std::shared_ptr<WorkerThreadPool> mPool;
};
// A dummy waitable event.
class WaitableEventDone final : public WaitableEvent
{
public:
void wait() override;
bool isReady() override;
};
// Request WorkerThreads from the WorkerThreadPool. Each pool can keep worker threads around so
......@@ -62,14 +76,17 @@ class WorkerThreadPool : angle::NonCopyable
virtual ~WorkerThreadPool();
static std::shared_ptr<WorkerThreadPool> Create(bool multithreaded);
// Returns an event to wait on for the task to finish.
// If the pool fails to create the task, returns null.
virtual std::shared_ptr<WaitableEvent> postWorkerTask(std::shared_ptr<Closure> task) = 0;
static std::shared_ptr<WaitableEvent> PostWorkerTask(std::shared_ptr<WorkerThreadPool> pool,
std::shared_ptr<Closure> task);
virtual void setMaxThreads(size_t maxThreads) = 0;
virtual bool isAsync() = 0;
private:
// Returns an event to wait on for the task to finish.
// If the pool fails to create the task, returns null.
virtual std::shared_ptr<WaitableEvent> postWorkerTask(std::shared_ptr<Closure> task) = 0;
};
} // namespace angle
......
......@@ -35,8 +35,10 @@ TEST(WorkerPoolTest, SimpleTask)
{std::make_shared<TestTask>(), std::make_shared<TestTask>(),
std::make_shared<TestTask>(), std::make_shared<TestTask>()}};
std::array<std::shared_ptr<WaitableEvent>, 4> waitables = {
{pool->postWorkerTask(tasks[0]), pool->postWorkerTask(tasks[1]),
pool->postWorkerTask(tasks[2]), pool->postWorkerTask(tasks[3])}};
{WorkerThreadPool::PostWorkerTask(pool, tasks[0]),
WorkerThreadPool::PostWorkerTask(pool, tasks[1]),
WorkerThreadPool::PostWorkerTask(pool, tasks[2]),
WorkerThreadPool::PostWorkerTask(pool, tasks[3])}};
WaitableEvent::WaitMany(&waitables);
......
......@@ -124,4 +124,5 @@ void ContextImpl::handleError(GLenum errorCode,
errorStream << "Internal error: " << gl::FmtHex(errorCode) << ": " << message;
mErrors->handleError(errorCode, errorStream.str().c_str(), file, function, line);
}
} // namespace rx
......@@ -140,6 +140,9 @@ class ContextImpl : public GLImplFactory
virtual void pushDebugGroup(GLenum source, GLuint id, const std::string &message) = 0;
virtual void popDebugGroup() = 0;
// KHR_parallel_shader_compile
virtual void setMaxShaderCompilerThreads(GLuint count) {}
// State sync with dirty bits.
virtual angle::Result syncState(const gl::Context *context,
const gl::State::DirtyBits &dirtyBits,
......
//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// ShaderImpl.cpp: Implementation methods of ShaderImpl
#include "libANGLE/renderer/ShaderImpl.h"
#include "libANGLE/Context.h"
namespace rx
{
WaitableCompileEvent::WaitableCompileEvent(std::shared_ptr<angle::WaitableEvent> waitableEvent)
: mWaitableEvent(waitableEvent)
{}
WaitableCompileEvent::~WaitableCompileEvent()
{
mWaitableEvent.reset();
}
void WaitableCompileEvent::wait()
{
mWaitableEvent->wait();
}
bool WaitableCompileEvent::isReady()
{
return mWaitableEvent->isReady();
}
const std::string &WaitableCompileEvent::getInfoLog()
{
return mInfoLog;
}
class TranslateTask : public angle::Closure
{
public:
TranslateTask(ShHandle handle, ShCompileOptions options, const std::string &source)
: mHandle(handle), mOptions(options), mSource(source), mResult(false)
{}
void operator()() override
{
const char *source = mSource.c_str();
mResult = sh::Compile(mHandle, &source, 1, mOptions);
}
bool getResult() { return mResult; }
ShHandle getHandle() { return mHandle; }
private:
ShHandle mHandle;
ShCompileOptions mOptions;
std::string mSource;
bool mResult;
};
class WaitableCompileEventImpl final : public WaitableCompileEvent
{
public:
WaitableCompileEventImpl(std::shared_ptr<angle::WaitableEvent> waitableEvent,
std::shared_ptr<TranslateTask> translateTask)
: WaitableCompileEvent(waitableEvent), mTranslateTask(translateTask)
{}
bool getResult() override { return mTranslateTask->getResult(); }
bool postTranslate(std::string *infoLog) override { return true; }
private:
std::shared_ptr<TranslateTask> mTranslateTask;
};
std::shared_ptr<WaitableCompileEvent> ShaderImpl::compileImpl(
const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
const std::string &source,
ShCompileOptions compileOptions)
{
auto workerThreadPool = context->getWorkerThreadPool();
auto translateTask =
std::make_shared<TranslateTask>(compilerInstance->getHandle(), compileOptions, source);
return std::make_shared<WaitableCompileEventImpl>(
angle::WorkerThreadPool::PostWorkerTask(workerThreadPool, translateTask), translateTask);
}
} // namespace rx
......@@ -9,12 +9,42 @@
#ifndef LIBANGLE_RENDERER_SHADERIMPL_H_
#define LIBANGLE_RENDERER_SHADERIMPL_H_
#include <functional>
#include "common/angleutils.h"
#include "libANGLE/Shader.h"
#include "libANGLE/WorkerThread.h"
namespace gl
{
class ShCompilerInstance;
} // namespace gl
namespace rx
{
using UpdateShaderStateFunctor = std::function<void(bool compiled, ShHandle handle)>;
class WaitableCompileEvent : public angle::WaitableEvent
{
public:
WaitableCompileEvent(std::shared_ptr<angle::WaitableEvent> waitableEvent);
~WaitableCompileEvent() override;
void wait() override;
bool isReady() override;
virtual bool getResult() = 0;
virtual bool postTranslate(std::string *infoLog) = 0;
const std::string &getInfoLog();
protected:
std::shared_ptr<angle::WaitableEvent> mWaitableEvent;
std::string mInfoLog;
};
class ShaderImpl : angle::NonCopyable
{
public:
......@@ -23,22 +53,20 @@ class ShaderImpl : angle::NonCopyable
virtual void destroy() {}
// Returns additional sh::Compile options.
virtual ShCompileOptions prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath) = 0;
// Uses the GL driver to compile the shader source in a worker thread.
virtual void compileAsync(const std::string &source, std::string &infoLog) {}
// Returns success for compiling on the driver. Returns success.
virtual bool postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog) = 0;
virtual std::shared_ptr<WaitableCompileEvent> compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options) = 0;
virtual std::string getDebugInfo() const = 0;
const gl::ShaderState &getData() const { return mData; }
protected:
std::shared_ptr<WaitableCompileEvent> compileImpl(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
const std::string &source,
ShCompileOptions compileOptions);
const gl::ShaderState &mData;
};
......
......@@ -955,7 +955,7 @@ class ProgramD3D::LoadBinaryLinkEvent final : public LinkEvent
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
: mTask(std::make_shared<ProgramD3D::LoadBinaryTask>(context, program, stream, infoLog)),
mWaitableEvent(workerPool->postWorkerTask(mTask))
mWaitableEvent(angle::WorkerThreadPool::PostWorkerTask(workerPool, mTask))
{}
angle::Result wait(const gl::Context *context) override
......@@ -1771,14 +1771,15 @@ class ProgramD3D::GraphicsProgramLinkEvent final : public LinkEvent
const ShaderD3D *vertexShader,
const ShaderD3D *fragmentShader)
: mInfoLog(infoLog),
mWorkerPool(workerPool),
mVertexTask(vertexTask),
mPixelTask(pixelTask),
mGeometryTask(geometryTask),
mWaitEvents(
{{std::shared_ptr<WaitableEvent>(workerPool->postWorkerTask(mVertexTask)),
std::shared_ptr<WaitableEvent>(workerPool->postWorkerTask(mPixelTask)),
std::shared_ptr<WaitableEvent>(workerPool->postWorkerTask(mGeometryTask))}}),
mWaitEvents({{std::shared_ptr<WaitableEvent>(
angle::WorkerThreadPool::PostWorkerTask(workerPool, mVertexTask)),
std::shared_ptr<WaitableEvent>(
angle::WorkerThreadPool::PostWorkerTask(workerPool, mPixelTask)),
std::shared_ptr<WaitableEvent>(
angle::WorkerThreadPool::PostWorkerTask(workerPool, mGeometryTask))}}),
mUseGS(useGS),
mVertexShader(vertexShader),
mFragmentShader(fragmentShader)
......@@ -1863,7 +1864,6 @@ class ProgramD3D::GraphicsProgramLinkEvent final : public LinkEvent
}
gl::InfoLog &mInfoLog;
std::shared_ptr<WorkerThreadPool> mWorkerPool;
std::shared_ptr<ProgramD3D::GetVertexExecutableTask> mVertexTask;
std::shared_ptr<ProgramD3D::GetPixelExecutableTask> mPixelTask;
std::shared_ptr<ProgramD3D::GetGeometryExecutableTask> mGeometryTask;
......
......@@ -11,6 +11,7 @@
#include "common/utilities.h"
#include "libANGLE/Caps.h"
#include "libANGLE/Compiler.h"
#include "libANGLE/Context.h"
#include "libANGLE/Shader.h"
#include "libANGLE/features.h"
#include "libANGLE/renderer/d3d/ProgramD3D.h"
......@@ -19,6 +20,71 @@
namespace rx
{
class TranslateTaskD3D : public angle::Closure
{
public:
TranslateTaskD3D(ShHandle handle,
ShCompileOptions options,
const std::string &source,
const std::string &sourcePath)
: mHandle(handle),
mOptions(options),
mSource(source),
mSourcePath(sourcePath),
mResult(false)
{}
void operator()() override
{
std::vector<const char *> srcStrings;
if (!mSourcePath.empty())
{
srcStrings.push_back(mSourcePath.c_str());
}
srcStrings.push_back(mSource.c_str());
mResult = sh::Compile(mHandle, &srcStrings[0], srcStrings.size(), mOptions);
}
bool getResult() { return mResult; }
private:
ShHandle mHandle;
ShCompileOptions mOptions;
std::string mSource;
std::string mSourcePath;
bool mResult;
};
using PostTranslateFunctor =
std::function<bool(gl::ShCompilerInstance *compiler, std::string *infoLog)>;
class WaitableCompileEventD3D final : public WaitableCompileEvent
{
public:
WaitableCompileEventD3D(std::shared_ptr<angle::WaitableEvent> waitableEvent,
gl::ShCompilerInstance *compilerInstance,
PostTranslateFunctor &&postTranslateFunctor,
std::shared_ptr<TranslateTaskD3D> translateTask)
: WaitableCompileEvent(waitableEvent),
mCompilerInstance(compilerInstance),
mPostTranslateFunctor(std::move(postTranslateFunctor)),
mTranslateTask(translateTask)
{}
bool getResult() override { return mTranslateTask->getResult(); }
bool postTranslate(std::string *infoLog) override
{
return mPostTranslateFunctor(mCompilerInstance, infoLog);
}
private:
gl::ShCompilerInstance *mCompilerInstance;
PostTranslateFunctor mPostTranslateFunctor;
std::shared_ptr<TranslateTaskD3D> mTranslateTask;
};
ShaderD3D::ShaderD3D(const gl::ShaderState &data,
const angle::WorkaroundsD3D &workarounds,
const gl::Extensions &extensions)
......@@ -155,10 +221,25 @@ bool ShaderD3D::useImage2DFunction(const std::string &functionName) const
return mUsedImage2DFunctionNames.find(functionName) != mUsedImage2DFunctionNames.end();
}
ShCompileOptions ShaderD3D::prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *shaderSourceStream,
std::string *sourcePath)
const std::map<std::string, unsigned int> &GetUniformRegisterMap(
const std::map<std::string, unsigned int> *uniformRegisterMap)
{
ASSERT(uniformRegisterMap);
return *uniformRegisterMap;
}
const std::set<std::string> &GetUsedImage2DFunctionNames(
const std::set<std::string> *usedImage2DFunctionNames)
{
ASSERT(usedImage2DFunctionNames);
return *usedImage2DFunctionNames;
}
std::shared_ptr<WaitableCompileEvent> ShaderD3D::compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options)
{
std::string sourcePath;
uncompile();
ShCompileOptions additionalOptions = 0;
......@@ -168,39 +249,17 @@ ShCompileOptions ShaderD3D::prepareSourceAndReturnOptions(const gl::Context *con
#if !defined(ANGLE_ENABLE_WINDOWS_STORE)
if (gl::DebugAnnotationsActive())
{
*sourcePath = getTempPath();
writeFile(sourcePath->c_str(), source.c_str(), source.length());
sourcePath = getTempPath();
writeFile(sourcePath.c_str(), source.c_str(), source.length());
additionalOptions |= SH_LINE_DIRECTIVES | SH_SOURCE_PATH;
}
#endif
additionalOptions |= mAdditionalOptions;
*shaderSourceStream << source;
return additionalOptions;
}
bool ShaderD3D::hasUniform(const std::string &name) const
{
return mUniformRegisterMap.find(name) != mUniformRegisterMap.end();
}
options |= additionalOptions;
const std::map<std::string, unsigned int> &GetUniformRegisterMap(
const std::map<std::string, unsigned int> *uniformRegisterMap)
{
ASSERT(uniformRegisterMap);
return *uniformRegisterMap;
}
const std::set<std::string> &GetUsedImage2DFunctionNames(
const std::set<std::string> *usedImage2DFunctionNames)
{
ASSERT(usedImage2DFunctionNames);
return *usedImage2DFunctionNames;
}
bool ShaderD3D::postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog)
{
auto postTranslateFunctor = [this](gl::ShCompilerInstance *compiler, std::string *infoLog) {
// TODO(jmadill): We shouldn't need to cache this.
mCompilerOutputType = compiler->getShaderOutputType();
......@@ -262,9 +321,24 @@ bool ShaderD3D::postTranslateCompile(gl::ShCompilerInstance *compiler, std::stri
mDebugInfo +=
std::string("// ") + gl::GetShaderTypeString(mData.getShaderType()) + " SHADER BEGIN\n";
mDebugInfo += "\n// GLSL BEGIN\n\n" + mData.getSource() + "\n\n// GLSL END\n\n\n";
mDebugInfo += "// INITIAL HLSL BEGIN\n\n" + translatedSource + "\n// INITIAL HLSL END\n\n\n";
mDebugInfo +=
"// INITIAL HLSL BEGIN\n\n" + translatedSource + "\n// INITIAL HLSL END\n\n\n";
// Successive steps will append more info
return true;
};
auto workerThreadPool = context->getWorkerThreadPool();
auto translateTask = std::make_shared<TranslateTaskD3D>(compilerInstance->getHandle(), options,
source, sourcePath);
return std::make_shared<WaitableCompileEventD3D>(
angle::WorkerThreadPool::PostWorkerTask(workerThreadPool, translateTask), compilerInstance,
std::move(postTranslateFunctor), translateTask);
}
bool ShaderD3D::hasUniform(const std::string &name) const
{
return mUniformRegisterMap.find(name) != mUniformRegisterMap.end();
}
} // namespace rx
......@@ -38,11 +38,10 @@ class ShaderD3D : public ShaderImpl
const gl::Extensions &extensions);
~ShaderD3D() override;
// ShaderImpl implementation
ShCompileOptions prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath) override;
bool postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog) override;
std::shared_ptr<WaitableCompileEvent> compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options) override;
std::string getDebugInfo() const override;
// D3D-specific methods
......
......@@ -608,4 +608,10 @@ angle::Result ContextGL::memoryBarrierByRegion(const gl::Context *context, GLbit
{
return mRenderer->memoryBarrierByRegion(barriers);
}
void ContextGL::setMaxShaderCompilerThreads(GLuint count)
{
mRenderer->setMaxShaderCompilerThreads(count);
}
} // namespace rx
......@@ -213,6 +213,8 @@ class ContextGL : public ContextImpl
angle::Result memoryBarrier(const gl::Context *context, GLbitfield barriers) override;
angle::Result memoryBarrierByRegion(const gl::Context *context, GLbitfield barriers) override;
void setMaxShaderCompilerThreads(GLuint count) override;
private:
angle::Result setDrawArraysState(const gl::Context *context,
GLint first,
......
......@@ -1128,6 +1128,11 @@ void DispatchTableGL::initProcsDesktopGL(const gl::Version &version,
ASSIGN("glIsQueryARB", isQuery);
}
if (extensions.count("GL_ARB_parallel_shader_compile") != 0)
{
ASSIGN("glMaxShaderCompilerThreadsARB", maxShaderCompilerThreadsARB);
}
if (extensions.count("GL_ARB_point_parameters") != 0)
{
ASSIGN("glPointParameterfARB", pointParameterf);
......@@ -2699,6 +2704,11 @@ void DispatchTableGL::initProcsSharedExtensions(const std::set<std::string> &ext
ASSIGN("glValidateProgramPipelineEXT", validateProgramPipeline);
}
if (extensions.count("GL_KHR_parallel_shader_compile") != 0)
{
ASSIGN("glMaxShaderCompilerThreadsKHR", maxShaderCompilerThreadsKHR);
}
if (extensions.count("GL_NV_fence") != 0)
{
ASSIGN("glDeleteFencesNV", deleteFencesNV);
......@@ -3849,6 +3859,11 @@ void DispatchTableGL::initProcsDesktopGLNULL(const gl::Version &version,
isQuery = &glIsQueryNULL;
}
if (extensions.count("GL_ARB_parallel_shader_compile") != 0)
{
maxShaderCompilerThreadsARB = &glMaxShaderCompilerThreadsARBNULL;
}
if (extensions.count("GL_ARB_point_parameters") != 0)
{
pointParameterf = &glPointParameterfNULL;
......@@ -5419,6 +5434,11 @@ void DispatchTableGL::initProcsSharedExtensionsNULL(const std::set<std::string>
validateProgramPipeline = &glValidateProgramPipelineNULL;
}
if (extensions.count("GL_KHR_parallel_shader_compile") != 0)
{
maxShaderCompilerThreadsKHR = &glMaxShaderCompilerThreadsKHRNULL;
}
if (extensions.count("GL_NV_fence") != 0)
{
deleteFencesNV = &glDeleteFencesNVNULL;
......
......@@ -730,6 +730,9 @@ class DispatchTableGL : angle::NonCopyable
PFNGLBLENDBARRIERPROC blendBarrier = nullptr;
PFNGLPRIMITIVEBOUNDINGBOXPROC primitiveBoundingBox = nullptr;
// GL_ARB_parallel_shader_compile
PFNGLMAXSHADERCOMPILERTHREADSARBPROC maxShaderCompilerThreadsARB = nullptr;
// GL_EXT_debug_marker
PFNGLINSERTEVENTMARKEREXTPROC insertEventMarkerEXT = nullptr;
PFNGLPOPGROUPMARKEREXTPROC popGroupMarkerEXT = nullptr;
......@@ -738,6 +741,9 @@ class DispatchTableGL : angle::NonCopyable
// GL_EXT_discard_framebuffer
PFNGLDISCARDFRAMEBUFFEREXTPROC discardFramebufferEXT = nullptr;
// GL_KHR_parallel_shader_compile
PFNGLMAXSHADERCOMPILERTHREADSKHRPROC maxShaderCompilerThreadsKHR = nullptr;
// GL_NV_internalformat_sample_query
PFNGLGETINTERNALFORMATSAMPLEIVNVPROC getInternalformatSampleivNV = nullptr;
......
......@@ -148,16 +148,51 @@ class ProgramGL::LinkTask final : public angle::Closure
};
using PostLinkImplFunctor = std::function<angle::Result(bool, const std::string &)>;
// The event for a parallelized linking using the native driver extension.
class ProgramGL::LinkEventNativeParallel final : public LinkEvent
{
public:
LinkEventNativeParallel(PostLinkImplFunctor &&functor,
const FunctionsGL *functions,
GLuint programID)
: mPostLinkImplFunctor(functor), mFunctions(functions), mProgramID(programID)
{}
angle::Result wait(const gl::Context *context) override
{
GLint linkStatus = GL_FALSE;
mFunctions->getProgramiv(mProgramID, GL_LINK_STATUS, &linkStatus);
if (linkStatus == GL_TRUE)
{
return mPostLinkImplFunctor(false, std::string());
}
return angle::Result::Incomplete;
}
bool isLinking() override
{
GLint completionStatus = GL_FALSE;
mFunctions->getProgramiv(mProgramID, GL_COMPLETION_STATUS, &completionStatus);
return completionStatus == GL_FALSE;
}
private:
PostLinkImplFunctor mPostLinkImplFunctor;
const FunctionsGL *mFunctions;
GLuint mProgramID;
};
// The event for a parallelized linking using the worker thread pool.
class ProgramGL::LinkEventGL final : public LinkEvent
{
public:
LinkEventGL(std::shared_ptr<angle::WorkerThreadPool> workerPool,
std::shared_ptr<ProgramGL::LinkTask> linkTask,
PostLinkImplFunctor &&functor)
: mWorkerPool(workerPool),
mLinkTask(linkTask),
mWaitableEvent(
std::shared_ptr<angle::WaitableEvent>(workerPool->postWorkerTask(mLinkTask))),
: mLinkTask(linkTask),
mWaitableEvent(std::shared_ptr<angle::WaitableEvent>(
angle::WorkerThreadPool::PostWorkerTask(workerPool, mLinkTask))),
mPostLinkImplFunctor(functor)
{}
......@@ -170,7 +205,6 @@ class ProgramGL::LinkEventGL final : public LinkEvent
bool isLinking() override { return !mWaitableEvent->isReady(); }
private:
std::shared_ptr<angle::WorkerThreadPool> mWorkerPool;
std::shared_ptr<ProgramGL::LinkTask> mLinkTask;
std::shared_ptr<angle::WaitableEvent> mWaitableEvent;
PostLinkImplFunctor mPostLinkImplFunctor;
......@@ -430,7 +464,13 @@ std::unique_ptr<LinkEvent> ProgramGL::link(const gl::Context *context,
return angle::Result::Continue;
};
if (workerPool->isAsync() && (!mWorkarounds.dontRelinkProgramsInParallel || !mLinkedInParallel))
if (mRenderer->hasNativeParallelCompile())
{
mFunctions->linkProgram(mProgramID);
return std::make_unique<LinkEventNativeParallel>(postLinkImplTask, mFunctions, mProgramID);
}
else if (workerPool->isAsync() &&
(!mWorkarounds.dontRelinkProgramsInParallel || !mLinkedInParallel))
{
mLinkedInParallel = true;
return std::make_unique<LinkEventGL>(workerPool, linkTask, postLinkImplTask);
......
......@@ -117,6 +117,7 @@ class ProgramGL : public ProgramImpl
private:
class LinkTask;
class LinkEventNativeParallel;
class LinkEventGL;
void preLink();
......
......@@ -55,6 +55,19 @@ std::vector<GLuint> GatherPaths(const std::vector<gl::Path *> &paths)
return ret;
}
void SetMaxShaderCompilerThreads(const rx::FunctionsGL *functions, GLuint count)
{
if (functions->maxShaderCompilerThreadsKHR != nullptr)
{
functions->maxShaderCompilerThreadsKHR(count);
}
else
{
ASSERT(functions->maxShaderCompilerThreadsARB != nullptr);
functions->maxShaderCompilerThreadsARB(count);
}
}
} // namespace
static void INTERNAL_GL_APIENTRY LogGLDebugMessage(GLenum source,
......@@ -175,7 +188,8 @@ RendererGL::RendererGL(std::unique_ptr<FunctionsGL> functions, const egl::Attrib
mMultiviewClearer(nullptr),
mUseDebugOutput(false),
mCapsInitialized(false),
mMultiviewImplementationType(MultiviewImplementationTypeGL::UNSPECIFIED)
mMultiviewImplementationType(MultiviewImplementationTypeGL::UNSPECIFIED),
mNativeParallelCompileEnabled(false)
{
ASSERT(mFunctions);
nativegl_gl::GenerateWorkarounds(mFunctions.get(), &mWorkarounds);
......@@ -215,6 +229,12 @@ RendererGL::RendererGL(std::unique_ptr<FunctionsGL> functions, const egl::Attrib
mFunctions->vertexAttrib4f(i, 0.0f, 0.0f, 0.0f, 1.0f);
}
}
if (hasNativeParallelCompile() && !mNativeParallelCompileEnabled)
{
SetMaxShaderCompilerThreads(mFunctions.get(), 0xffffffff);
mNativeParallelCompileEnabled = true;
}
}
RendererGL::~RendererGL()
......@@ -617,6 +637,20 @@ unsigned int RendererGL::getMaxWorkerContexts()
return std::min(16u, std::thread::hardware_concurrency());
}
bool RendererGL::hasNativeParallelCompile()
{
return mFunctions->maxShaderCompilerThreadsKHR != nullptr ||
mFunctions->maxShaderCompilerThreadsARB != nullptr;
}
void RendererGL::setMaxShaderCompilerThreads(GLuint count)
{
if (hasNativeParallelCompile())
{
SetMaxShaderCompilerThreads(mFunctions.get(), count);
}
}
ScopedWorkerContextGL::ScopedWorkerContextGL(RendererGL *renderer, std::string *infoLog)
: mRenderer(renderer)
{
......
......@@ -180,6 +180,10 @@ class RendererGL : angle::NonCopyable
bool bindWorkerContext(std::string *infoLog);
void unbindWorkerContext();
// Checks if the driver has the KHR_parallel_shader_compile or ARB_parallel_shader_compile
// extension.
bool hasNativeParallelCompile();
void setMaxShaderCompilerThreads(GLuint count);
static unsigned int getMaxWorkerContexts();
......@@ -218,6 +222,8 @@ class RendererGL : angle::NonCopyable
std::list<std::unique_ptr<WorkerContext>> mWorkerContextPool;
// Protect the concurrent accesses to worker contexts.
std::mutex mWorkerMutex;
bool mNativeParallelCompileEnabled;
};
} // namespace rx
......
......@@ -28,23 +28,24 @@ class ShaderGL : public ShaderImpl
void destroy() override;
// ShaderImpl implementation
ShCompileOptions prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath) override;
void compileAsync(const std::string &source, std::string &infoLog) override;
bool postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog) override;
std::shared_ptr<WaitableCompileEvent> compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options) override;
std::string getDebugInfo() const override;
GLuint getShaderID() const;
private:
void compileAndCheckShader(const char *source);
void compileShader(const char *source);
void checkShader();
bool peekCompletion();
bool compileAndCheckShaderInWorker(const char *source);
GLuint mShaderID;
MultiviewImplementationTypeGL mMultiviewImplementationType;
std::shared_ptr<RendererGL> mRenderer;
bool mFallbackToMainThread;
GLint mCompileStatus;
std::string mInfoLog;
};
......
......@@ -1371,5 +1371,6 @@
#define GL_TEXTURE_TARGET 0x1006
#define GL_UNKNOWN_CONTEXT_RESET 0x8255
#define GL_ZERO_TO_ONE 0x935F
#define GL_COMPLETION_STATUS 0x91B1
#endif // LIBANGLE_RENDERER_GL_FUNCTIONSGLENUMS_H_
......@@ -1857,6 +1857,13 @@ typedef void(INTERNAL_GL_APIENTRY *PFNGLINSERTEVENTMARKEREXTPROC)(GLsizei length
typedef void(INTERNAL_GL_APIENTRY *PFNGLPUSHGROUPMARKEREXTPROC)(GLsizei length,
const GLchar *marker);
typedef void(INTERNAL_GL_APIENTRY *PFNGLPOPGROUPMARKEREXTPROC)(void);
// KHR_parallel_shader_compile
typedef void(INTERNAL_GL_APIENTRY *PFNGLMAXSHADERCOMPILERTHREADSKHRPROC)(GLuint count);
// ARB_parallel_shader_compile
typedef void(INTERNAL_GL_APIENTRY *PFNGLMAXSHADERCOMPILERTHREADSARBPROC)(GLuint count);
} // namespace rx
#endif // LIBANGLE_RENDERER_GL_FUNCTIONSGLTYPEDEFS_H_
......@@ -297,7 +297,7 @@ def main():
command_name = command.attrib['name']
if 'gl' in support and 'gles2' in support:
# Special case for KHR extensions, since in GLES they are suffixed.
if '_KHR_' in extension_name:
if '_KHR_' in extension_name and not command_name.endswith('KHR'):
safe_append(gl_extension_commands, command_name, extension_name)
safe_append(gles2_extension_commands, command_name, extension_name)
else:
......
......@@ -790,5 +790,15 @@
"InsertEventMarkerEXT",
"PushGroupMarkerEXT",
"PopGroupMarkerEXT"
],
"GL_ARB_parallel_shader_compile":
[
"MaxShaderCompilerThreadsARB"
],
"GL_KHR_parallel_shader_compile":
[
"MaxShaderCompilerThreadsKHR"
]
}
......@@ -24,6 +24,17 @@
#include "libANGLE/renderer/gl/glx/WindowSurfaceGLX.h"
#include "libANGLE/renderer/gl/renderergl_utils.h"
namespace
{
bool HasParallelShaderCompileExtension(const rx::FunctionsGL *functions)
{
return functions->maxShaderCompilerThreadsKHR != nullptr ||
functions->maxShaderCompilerThreadsARB != nullptr;
}
} // anonymous namespace
namespace rx
{
......@@ -289,9 +300,17 @@ egl::Error DisplayGLX::initialize(egl::Display *display)
if (mSharedContext)
{
if (HasParallelShaderCompileExtension(functionsGL.get()))
{
mGLX.destroyContext(mSharedContext);
mSharedContext = nullptr;
}
else
{
for (unsigned int i = 0; i < RendererGL::getMaxWorkerContexts(); ++i)
{
glx::Pbuffer workerPbuffer = mGLX.createPbuffer(mContextConfig, dummyPbufferAttribs);
glx::Pbuffer workerPbuffer =
mGLX.createPbuffer(mContextConfig, dummyPbufferAttribs);
if (!workerPbuffer)
{
return egl::EglNotInitialized() << "Could not create the worker pbuffers.";
......@@ -299,6 +318,7 @@ egl::Error DisplayGLX::initialize(egl::Display *display)
mWorkerPbufferPool.push_back(workerPbuffer);
}
}
}
syncXCommands();
......
......@@ -1672,6 +1672,10 @@ void *INTERNAL_GL_APIENTRY glMapNamedBufferRangeNULL(GLuint buffer,
void INTERNAL_GL_APIENTRY glMatrixLoadfEXTNULL(GLenum mode, const GLfloat *m) {}
void INTERNAL_GL_APIENTRY glMaxShaderCompilerThreadsARBNULL(GLuint count) {}
void INTERNAL_GL_APIENTRY glMaxShaderCompilerThreadsKHRNULL(GLuint count) {}
void INTERNAL_GL_APIENTRY glMemoryBarrierNULL(GLbitfield barriers) {}
void INTERNAL_GL_APIENTRY glMemoryBarrierByRegionNULL(GLbitfield barriers) {}
......
......@@ -1036,6 +1036,8 @@ void *INTERNAL_GL_APIENTRY glMapNamedBufferRangeNULL(GLuint buffer,
GLsizeiptr length,
GLbitfield access);
void INTERNAL_GL_APIENTRY glMatrixLoadfEXTNULL(GLenum mode, const GLfloat *m);
void INTERNAL_GL_APIENTRY glMaxShaderCompilerThreadsARBNULL(GLuint count);
void INTERNAL_GL_APIENTRY glMaxShaderCompilerThreadsKHRNULL(GLuint count);
void INTERNAL_GL_APIENTRY glMemoryBarrierNULL(GLbitfield barriers);
void INTERNAL_GL_APIENTRY glMemoryBarrierByRegionNULL(GLbitfield barriers);
void INTERNAL_GL_APIENTRY glMinSampleShadingNULL(GLfloat value);
......
......@@ -10,6 +10,7 @@
#include "libANGLE/renderer/null/ShaderNULL.h"
#include "common/debug.h"
#include "libANGLE/Context.h"
namespace rx
{
......@@ -18,17 +19,11 @@ ShaderNULL::ShaderNULL(const gl::ShaderState &data) : ShaderImpl(data) {}
ShaderNULL::~ShaderNULL() {}
ShCompileOptions ShaderNULL::prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath)
std::shared_ptr<WaitableCompileEvent> ShaderNULL::compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options)
{
*sourceStream << mData.getSource();
return 0;
}
bool ShaderNULL::postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog)
{
return true;
return compileImpl(context, compilerInstance, mData.getSource(), options);
}
std::string ShaderNULL::getDebugInfo() const
......
......@@ -21,12 +21,9 @@ class ShaderNULL : public ShaderImpl
ShaderNULL(const gl::ShaderState &data);
~ShaderNULL() override;
// Returns additional sh::Compile options.
ShCompileOptions prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath) override;
// Returns success for compiling on the driver. Returns success.
bool postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog) override;
std::shared_ptr<WaitableCompileEvent> compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options) override;
std::string getDebugInfo() const override;
};
......
......@@ -21,12 +21,10 @@ ShaderVk::ShaderVk(const gl::ShaderState &data) : ShaderImpl(data) {}
ShaderVk::~ShaderVk() {}
ShCompileOptions ShaderVk::prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath)
std::shared_ptr<WaitableCompileEvent> ShaderVk::compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options)
{
*sourceStream << mData.getSource();
ShCompileOptions compileOptions = SH_INITIALIZE_UNINITIALIZED_LOCALS;
ContextVk *contextVk = vk::GetImpl(context);
......@@ -36,13 +34,7 @@ ShCompileOptions ShaderVk::prepareSourceAndReturnOptions(const gl::Context *cont
compileOptions |= SH_CLAMP_POINT_SIZE;
}
return compileOptions;
}
bool ShaderVk::postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog)
{
// No work to do here.
return true;
return compileImpl(context, compilerInstance, mData.getSource(), compileOptions | options);
}
std::string ShaderVk::getDebugInfo() const
......
......@@ -21,12 +21,9 @@ class ShaderVk : public ShaderImpl
ShaderVk(const gl::ShaderState &data);
~ShaderVk() override;
// Returns additional sh::Compile options.
ShCompileOptions prepareSourceAndReturnOptions(const gl::Context *context,
std::stringstream *sourceStream,
std::string *sourcePath) override;
// Returns success for compiling on the driver. Returns success.
bool postTranslateCompile(gl::ShCompilerInstance *compiler, std::string *infoLog) override;
std::shared_ptr<WaitableCompileEvent> compile(const gl::Context *context,
gl::ShCompilerInstance *compilerInstance,
ShCompileOptions options) override;
std::string getDebugInfo() const override;
};
......
......@@ -306,6 +306,7 @@ libangle_sources = [
"src/libANGLE/renderer/RenderbufferImpl.h",
"src/libANGLE/renderer/RenderTargetCache.h",
"src/libANGLE/renderer/SamplerImpl.h",
"src/libANGLE/renderer/ShaderImpl.cpp",
"src/libANGLE/renderer/ShaderImpl.h",
"src/libANGLE/renderer/StreamProducerImpl.h",
"src/libANGLE/renderer/SurfaceImpl.cpp",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment