Commit e332e621 by Geoff Lang Committed by Commit Bot

D3D: Asynchronously load program binaries.

Unpack as much of the binary steam as possible before passing the final loading of the shader programs off to a worker thread. Reporting as many possible link errors before becoming asynchronous means that linking should only fail due to unexpected system issues at that point. This also allows other backends to asynchronously load program binaries. BUG=angleproject:2857 Change-Id: I587917a3e54522114dabd41d1b14fc491c8fd18a Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/1473451 Commit-Queue: Jamie Madill <jmadill@google.com> Reviewed-by: 's avatarShahbaz Youssefi <syoussefi@chromium.org>
parent d838178d
......@@ -125,6 +125,11 @@ class BinaryInputStream : angle::NonCopyable
}
size_t offset() const { return mOffset; }
size_t remainingSize() const
{
ASSERT(mLength >= mOffset);
return mLength - mOffset;
}
bool error() const { return mError; }
......
......@@ -5898,23 +5898,7 @@ void Context::linkProgram(GLuint program)
Program *programObject = getProgramNoResolveLink(program);
ASSERT(programObject);
ANGLE_CONTEXT_TRY(programObject->link(this));
// Don't parallel link a program which is active in any GL contexts. With this assumption, we
// don't need to worry that:
// 1. Draw calls after link use the new executable code or the old one depending on the link
// result.
// 2. When a backend program, e.g., ProgramD3D is linking, other backend classes like
// StateManager11, Renderer11, etc., may have a chance to make unexpected calls to
// ProgramD3D.
if (programObject->isInUse())
{
programObject->resolveLink(this);
if (programObject->isLinked())
{
ANGLE_CONTEXT_TRY(mState.onProgramExecutableChange(this, programObject));
}
mStateCache.onProgramExecutableChange(this);
}
ANGLE_CONTEXT_TRY(onProgramLink(programObject));
}
void Context::releaseShaderCompiler()
......@@ -6155,11 +6139,7 @@ void Context::programBinary(GLuint program, GLenum binaryFormat, const void *bin
ASSERT(programObject != nullptr);
ANGLE_CONTEXT_TRY(programObject->loadBinary(this, binaryFormat, binary, length));
if (programObject->isInUse())
{
ANGLE_CONTEXT_TRY(mState.onProgramExecutableChange(this, programObject));
mStateCache.onProgramExecutableChange(this);
}
ANGLE_CONTEXT_TRY(onProgramLink(programObject));
}
void Context::uniform1ui(GLint location, GLuint v0)
......@@ -8039,6 +8019,28 @@ void Context::onSubjectStateChange(const Context *context,
}
}
angle::Result Context::onProgramLink(Program *programObject)
{
// Don't parallel link a program which is active in any GL contexts. With this assumption, we
// don't need to worry that:
// 1. Draw calls after link use the new executable code or the old one depending on the link
// result.
// 2. When a backend program, e.g., ProgramD3D is linking, other backend classes like
// StateManager11, Renderer11, etc., may have a chance to make unexpected calls to
// ProgramD3D.
if (programObject->isInUse())
{
programObject->resolveLink(this);
if (programObject->isLinked())
{
ANGLE_TRY(mState.onProgramExecutableChange(this, programObject));
}
mStateCache.onProgramExecutableChange(this);
}
return angle::Result::Continue;
}
// ErrorSet implementation.
ErrorSet::ErrorSet(Context *context) : mContext(context) {}
......
......@@ -1829,6 +1829,8 @@ class Context final : public egl::LabeledObject, angle::NonCopyable, public angl
VertexArray *checkVertexArrayAllocation(GLuint vertexArrayHandle);
TransformFeedback *checkTransformFeedbackAllocation(GLuint transformFeedback);
angle::Result onProgramLink(Program *programObject);
void detachBuffer(Buffer *buffer);
void detachTexture(GLuint texture);
void detachFramebuffer(GLuint framebuffer);
......
......@@ -133,9 +133,8 @@ angle::Result MemoryProgramCache::getProgram(const Context *context,
egl::BlobCache::Value binaryProgram;
if (get(context, *hashOut, &binaryProgram))
{
InfoLog infoLog;
angle::Result result =
program->deserialize(context, binaryProgram.data(), binaryProgram.size(), infoLog);
angle::Result result = program->loadBinary(context, GL_PROGRAM_BINARY_ANGLE,
binaryProgram.data(), binaryProgram.size());
ANGLE_HISTOGRAM_BOOLEAN("GPU.ANGLE.ProgramCache.LoadBinarySuccess",
result == angle::Result::Continue);
ANGLE_TRY(result);
......@@ -146,7 +145,7 @@ angle::Result MemoryProgramCache::getProgram(const Context *context,
// Cache load failed, evict.
if (mIssuedWarnings++ < kWarningLimit)
{
WARN() << "Failed to load binary from cache: " << infoLog.str();
WARN() << "Failed to load binary from cache.";
if (mIssuedWarnings == kWarningLimit)
{
......
......@@ -718,6 +718,7 @@ struct Program::LinkingState
std::unique_ptr<ProgramLinkedResources> resources;
egl::BlobCache::Key programHash;
std::unique_ptr<rx::LinkEvent> linkEvent;
bool linkingFromBinary;
};
const char *const g_fakepath = "C:\\fakepath";
......@@ -1234,17 +1235,19 @@ angle::Result Program::link(const Context *context)
if (cache)
{
angle::Result result = cache->getProgram(context, this, &programHash);
mLinked = (result == angle::Result::Continue);
ANGLE_TRY(result);
}
angle::Result cacheResult = cache->getProgram(context, this, &programHash);
ANGLE_TRY(cacheResult);
if (mLinked)
{
double delta = platform->currentTime(platform) - startTime;
int us = static_cast<int>(delta * 1000000.0);
ANGLE_HISTOGRAM_COUNTS("GPU.ANGLE.ProgramCache.ProgramCacheHitTimeUS", us);
return angle::Result::Continue;
// Check explicitly for Continue, Incomplete means a cache miss
if (cacheResult == angle::Result::Continue)
{
// Succeeded in loading the binaries in the front-end, back end may still be loading
// asynchronously
double delta = platform->currentTime(platform) - startTime;
int us = static_cast<int>(delta * 1000000.0);
ANGLE_HISTOGRAM_COUNTS("GPU.ANGLE.ProgramCache.ProgramCacheHitTimeUS", us);
return angle::Result::Continue;
}
}
// Cache load failed, fall through to normal linking.
......@@ -1375,6 +1378,7 @@ angle::Result Program::link(const Context *context)
mLinkingState.reset(new LinkingState());
mLinkingState->context = context;
mLinkingState->linkingFromBinary = false;
mLinkingState->programHash = programHash;
mLinkingState->linkEvent = mProgram->link(context, *resources, mInfoLog);
mLinkingState->resources = std::move(resources);
......@@ -1396,12 +1400,18 @@ void Program::resolveLinkImpl(const Context *context)
mLinked = result == angle::Result::Continue;
mLinkResolved = true;
auto linkingState = std::move(mLinkingState);
std::unique_ptr<LinkingState> linkingState = std::move(mLinkingState);
if (!mLinked)
{
return;
}
if (linkingState->linkingFromBinary)
{
// All internal Program state is already loaded from the binary.
return;
}
initInterfaceBlockBindings();
// According to GLES 3.0/3.1 spec for LinkProgram and UseProgram,
......@@ -1567,10 +1577,8 @@ angle::Result Program::loadBinary(const Context *context,
return angle::Result::Continue;
}
const uint8_t *bytes = reinterpret_cast<const uint8_t *>(binary);
angle::Result result = deserialize(context, bytes, length, mInfoLog);
mLinked = result == angle::Result::Continue;
ANGLE_TRY(result);
BinaryInputStream stream(binary, length);
ANGLE_TRY(deserialize(context, stream, mInfoLog));
// Currently we require the full shader text to compute the program hash.
// We could also store the binary in the internal program cache.
......@@ -1581,6 +1589,12 @@ angle::Result Program::loadBinary(const Context *context,
mDirtyBits.set(uniformBlockIndex);
}
mLinkingState.reset(new LinkingState());
mLinkingState->context = context;
mLinkingState->linkingFromBinary = true;
mLinkingState->linkEvent = mProgram->load(context, &stream, mInfoLog);
mLinkResolved = false;
return angle::Result::Continue;
#endif // #if ANGLE_PROGRAM_BINARY_LOAD == ANGLE_ENABLED
}
......@@ -4442,12 +4456,9 @@ void Program::serialize(const Context *context, angle::MemoryBuffer *binaryOut)
}
angle::Result Program::deserialize(const Context *context,
const uint8_t *binary,
size_t length,
BinaryInputStream &stream,
InfoLog &infoLog)
{
BinaryInputStream stream(binary, length);
unsigned char commitString[ANGLE_COMMIT_HASH_SIZE];
stream.readBytes(commitString, ANGLE_COMMIT_HASH_SIZE);
if (memcmp(commitString, ANGLE_COMMIT_HASH, sizeof(unsigned char) * ANGLE_COMMIT_HASH_SIZE) !=
......@@ -4678,7 +4689,7 @@ angle::Result Program::deserialize(const Context *context,
postResolveLink(context);
return mProgram->load(context, infoLog, &stream);
return angle::Result::Continue;
}
void Program::postResolveLink(const gl::Context *context)
......
......@@ -42,6 +42,7 @@ struct TranslatedAttribute;
namespace gl
{
class Buffer;
class BinaryInputStream;
struct Caps;
class Context;
struct Extensions;
......@@ -881,17 +882,15 @@ class Program final : angle::NonCopyable, public LabeledObject
// Writes a program's binary to the output memory buffer.
void serialize(const Context *context, angle::MemoryBuffer *binaryOut) const;
// Loads program state according to the specified binary blob.
angle::Result deserialize(const Context *context,
const uint8_t *binary,
size_t length,
InfoLog &infoLog);
private:
struct LinkingState;
~Program() override;
// Loads program state according to the specified binary blob.
angle::Result deserialize(const Context *context, BinaryInputStream &stream, InfoLog &infoLog);
void unlink();
void deleteSelf(const Context *context);
......
......@@ -75,9 +75,9 @@ class ProgramImpl : angle::NonCopyable
virtual ~ProgramImpl() {}
virtual void destroy(const gl::Context *context) {}
virtual angle::Result load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream) = 0;
virtual std::unique_ptr<LinkEvent> load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog) = 0;
virtual void save(const gl::Context *context, gl::BinaryOutputStream *stream) = 0;
virtual void setBinaryRetrievableHint(bool retrievable) = 0;
virtual void setSeparable(bool separable) = 0;
......
......@@ -8,6 +8,7 @@
#include "libANGLE/renderer/d3d/ProgramD3D.h"
#include "common/MemoryBuffer.h"
#include "common/bitset_utils.h"
#include "common/string_utils.h"
#include "common/utilities.h"
......@@ -562,6 +563,50 @@ const ShaderD3D *ProgramD3DMetadata::getFragmentShader() const
return mAttachedShaders[gl::ShaderType::Fragment];
}
// ProgramD3D::GetExecutableTask class
class ProgramD3D::GetExecutableTask : public Closure, public d3d::Context
{
public:
GetExecutableTask(ProgramD3D *program) : mProgram(program) {}
virtual angle::Result run() = 0;
void operator()() override { mResult = run(); }
angle::Result getResult() const { return mResult; }
const gl::InfoLog &getInfoLog() const { return mInfoLog; }
ShaderExecutableD3D *getExecutable() { return mExecutable; }
void handleResult(HRESULT hr,
const char *message,
const char *file,
const char *function,
unsigned int line) override
{
mStoredHR = hr;
mStoredMessage = message;
mStoredFile = file;
mStoredFunction = function;
mStoredLine = line;
}
void popError(d3d::Context *context)
{
context->handleResult(mStoredHR, mStoredMessage, mStoredFile, mStoredFunction, mStoredLine);
}
protected:
ProgramD3D *mProgram = nullptr;
angle::Result mResult = angle::Result::Continue;
gl::InfoLog mInfoLog;
ShaderExecutableD3D *mExecutable = nullptr;
HRESULT mStoredHR = S_OK;
const char *mStoredMessage = nullptr;
const char *mStoredFile = nullptr;
const char *mStoredFunction = nullptr;
unsigned int mStoredLine = 0;
};
// ProgramD3D Implementation
ProgramD3D::VertexExecutable::VertexExecutable(const gl::InputLayout &inputLayout,
......@@ -849,10 +894,92 @@ gl::RangeUI ProgramD3D::getUsedImageRange(gl::ShaderType type, bool readonly) co
}
}
angle::Result ProgramD3D::load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream)
class ProgramD3D::LoadBinaryTask : public ProgramD3D::GetExecutableTask
{
public:
LoadBinaryTask(const gl::Context *context,
ProgramD3D *program,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
: ProgramD3D::GetExecutableTask(program),
mContext(context),
mProgram(program),
mInfoLog(infoLog)
{
ASSERT(mContext);
ASSERT(mProgram);
ASSERT(stream);
// Copy the remaining data from the stream locally so that the client can't modify it when
// loading off thread.
size_t dataSize = stream->remainingSize();
mDataCopySucceeded = mStreamData.resize(dataSize);
if (mDataCopySucceeded)
{
memcpy(mStreamData.data(), stream->data() + stream->offset(), dataSize);
}
}
angle::Result run() override
{
if (!mDataCopySucceeded)
{
mInfoLog << "Failed to copy program binary data to local buffer.";
return angle::Result::Stop;
}
gl::BinaryInputStream stream(mStreamData.data(), mStreamData.size());
return mProgram->loadBinaryShaderExecutables(mContext, &stream, mInfoLog);
}
private:
const gl::Context *mContext;
ProgramD3D *mProgram;
gl::InfoLog &mInfoLog;
bool mDataCopySucceeded;
angle::MemoryBuffer mStreamData;
};
class ProgramD3D::LoadBinaryLinkEvent final : public LinkEvent
{
public:
LoadBinaryLinkEvent(std::shared_ptr<WorkerThreadPool> workerPool,
const gl::Context *context,
ProgramD3D *program,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
: mTask(std::make_shared<ProgramD3D::LoadBinaryTask>(context, program, stream, infoLog)),
mWaitableEvent(workerPool->postWorkerTask(mTask))
{}
angle::Result wait(const gl::Context *context) override
{
mWaitableEvent->wait();
// Continue and Incomplete are not errors. For Stop, pass the error to the ContextD3D.
if (mTask->getResult() != angle::Result::Stop)
{
return angle::Result::Continue;
}
ContextD3D *contextD3D = GetImplAs<ContextD3D>(context);
mTask->popError(contextD3D);
return angle::Result::Stop;
}
bool isLinking() override { return !mWaitableEvent->isReady(); }
private:
std::shared_ptr<ProgramD3D::LoadBinaryTask> mTask;
std::shared_ptr<WaitableEvent> mWaitableEvent;
};
std::unique_ptr<rx::LinkEvent> ProgramD3D::load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
{
// TODO(jmadill): Use Renderer from contextImpl.
reset();
......@@ -865,14 +992,14 @@ angle::Result ProgramD3D::load(const gl::Context *context,
if (memcmp(&identifier, &binaryDeviceIdentifier, sizeof(DeviceIdentifier)) != 0)
{
infoLog << "Invalid program binary, device configuration has changed.";
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
int compileFlags = stream->readInt<int>();
if (compileFlags != ANGLE_COMPILE_OPTIMIZATION_LEVEL)
{
infoLog << "Mismatched compilation flags.";
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
for (int &index : mAttribLocationToD3DSemantic)
......@@ -935,7 +1062,7 @@ angle::Result ProgramD3D::load(const gl::Context *context,
if (stream->error())
{
infoLog << "Invalid program binary.";
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
ASSERT(mD3DShaderStorageBlocks.empty());
......@@ -959,7 +1086,7 @@ angle::Result ProgramD3D::load(const gl::Context *context,
if (stream->error())
{
infoLog << "Invalid program binary.";
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
const auto &linkedUniforms = mState.getUniforms();
......@@ -986,7 +1113,7 @@ angle::Result ProgramD3D::load(const gl::Context *context,
if (stream->error())
{
infoLog << "Invalid program binary.";
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
ASSERT(mD3DUniformBlocks.empty());
......@@ -1038,6 +1165,14 @@ angle::Result ProgramD3D::load(const gl::Context *context,
stream->readString(&mGeometryShaderPreamble);
return std::make_unique<LoadBinaryLinkEvent>(context->getWorkerThreadPool(), context, this,
stream, infoLog);
}
angle::Result ProgramD3D::loadBinaryShaderExecutables(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
{
const unsigned char *binary = reinterpret_cast<const unsigned char *>(stream->data());
bool separateAttribs = (mState.getTransformFeedbackBufferMode() == GL_SEPARATE_ATTRIBS);
......@@ -1543,49 +1678,6 @@ angle::Result ProgramD3D::getGeometryExecutableForPrimitiveType(d3d::Context *co
return result;
}
class ProgramD3D::GetExecutableTask : public Closure, public d3d::Context
{
public:
GetExecutableTask(ProgramD3D *program) : mProgram(program) {}
virtual angle::Result run() = 0;
void operator()() override { mResult = run(); }
angle::Result getResult() const { return mResult; }
const gl::InfoLog &getInfoLog() const { return mInfoLog; }
ShaderExecutableD3D *getExecutable() { return mExecutable; }
void handleResult(HRESULT hr,
const char *message,
const char *file,
const char *function,
unsigned int line) override
{
mStoredHR = hr;
mStoredMessage = message;
mStoredFile = file;
mStoredFunction = function;
mStoredLine = line;
}
void popError(d3d::Context *context)
{
context->handleResult(mStoredHR, mStoredMessage, mStoredFile, mStoredFunction, mStoredLine);
}
protected:
ProgramD3D *mProgram = nullptr;
angle::Result mResult = angle::Result::Continue;
gl::InfoLog mInfoLog;
ShaderExecutableD3D *mExecutable = nullptr;
HRESULT mStoredHR = S_OK;
const char *mStoredMessage = nullptr;
const char *mStoredFile = nullptr;
const char *mStoredFunction = nullptr;
unsigned int mStoredLine = 0;
};
class ProgramD3D::GetVertexExecutableTask : public ProgramD3D::GetExecutableTask
{
public:
......
......@@ -183,9 +183,9 @@ class ProgramD3D : public ProgramImpl
bool usesGeometryShaderForPointSpriteEmulation() const;
bool usesInstancedPointSpriteEmulation() const;
angle::Result load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream) override;
std::unique_ptr<LinkEvent> load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog) override;
void save(const gl::Context *context, gl::BinaryOutputStream *stream) override;
void setBinaryRetrievableHint(bool retrievable) override;
void setSeparable(bool separable) override;
......@@ -332,6 +332,9 @@ class ProgramD3D : public ProgramImpl
class GetGeometryExecutableTask;
class GraphicsProgramLinkEvent;
class LoadBinaryTask;
class LoadBinaryLinkEvent;
class VertexExecutable
{
public:
......@@ -471,6 +474,10 @@ class ProgramD3D : public ProgramImpl
gl::InfoLog &infoLog);
angle::Result compileComputeExecutable(d3d::Context *context, gl::InfoLog &infoLog);
angle::Result loadBinaryShaderExecutables(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog);
void gatherTransformFeedbackVaryings(const gl::VaryingPacking &varyings,
const BuiltinInfo &builtins);
D3DUniform *getD3DUniformFromLocation(GLint location);
......
......@@ -57,9 +57,9 @@ ProgramGL::~ProgramGL()
mProgramID = 0;
}
angle::Result ProgramGL::load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream)
std::unique_ptr<LinkEvent> ProgramGL::load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
{
preLink();
......@@ -75,13 +75,13 @@ angle::Result ProgramGL::load(const gl::Context *context,
// Verify that the program linked
if (!checkLinkStatus(infoLog))
{
return angle::Result::Incomplete;
return std::make_unique<LinkEventDone>(angle::Result::Incomplete);
}
postLink();
reapplyUBOBindingsIfNeeded(context);
return angle::Result::Continue;
return std::make_unique<LinkEventDone>(angle::Result::Continue);
}
void ProgramGL::save(const gl::Context *context, gl::BinaryOutputStream *stream)
......
......@@ -33,9 +33,9 @@ class ProgramGL : public ProgramImpl
const std::shared_ptr<RendererGL> &renderer);
~ProgramGL() override;
angle::Result load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream) override;
std::unique_ptr<LinkEvent> load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog) override;
void save(const gl::Context *context, gl::BinaryOutputStream *stream) override;
void setBinaryRetrievableHint(bool retrievable) override;
void setSeparable(bool separable) override;
......
......@@ -18,11 +18,11 @@ ProgramNULL::ProgramNULL(const gl::ProgramState &state) : ProgramImpl(state) {}
ProgramNULL::~ProgramNULL() {}
angle::Result ProgramNULL::load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream)
std::unique_ptr<LinkEvent> ProgramNULL::load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
{
return angle::Result::Continue;
return std::make_unique<LinkEventDone>(angle::Result::Continue);
}
void ProgramNULL::save(const gl::Context *context, gl::BinaryOutputStream *stream) {}
......
......@@ -21,9 +21,9 @@ class ProgramNULL : public ProgramImpl
ProgramNULL(const gl::ProgramState &state);
~ProgramNULL() override;
angle::Result load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream) override;
std::unique_ptr<LinkEvent> load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog) override;
void save(const gl::Context *context, gl::BinaryOutputStream *stream) override;
void setBinaryRetrievableHint(bool retrievable) override;
void setSeparable(bool separable) override;
......
......@@ -221,12 +221,12 @@ void ProgramVk::reset(RendererVk *renderer)
}
}
angle::Result ProgramVk::load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream)
std::unique_ptr<rx::LinkEvent> ProgramVk::load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog)
{
UNIMPLEMENTED();
return angle::Result::Stop;
return std::make_unique<LinkEventDone>(angle::Result::Stop);
}
void ProgramVk::save(const gl::Context *context, gl::BinaryOutputStream *stream)
......
......@@ -32,9 +32,9 @@ class ProgramVk : public ProgramImpl
~ProgramVk() override;
void destroy(const gl::Context *context) override;
angle::Result load(const gl::Context *context,
gl::InfoLog &infoLog,
gl::BinaryInputStream *stream) override;
std::unique_ptr<LinkEvent> load(const gl::Context *context,
gl::BinaryInputStream *stream,
gl::InfoLog &infoLog) override;
void save(const gl::Context *context, gl::BinaryOutputStream *stream) override;
void setBinaryRetrievableHint(bool retrievable) override;
void setSeparable(bool separable) override;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment