Commit ccc0fbaa by Jamie Madill Committed by Commit Bot

Vulkan: Related fixes for buffer descriptor set cache.

Includes some stats counter gathering and a few related refactors and cleanups. Also includes a new overlay widget. Bug: angleproject:5736 Change-Id: Ida8d2cd815c5b598c6a442dd9bbfdf51e9c05180 Reviewed-on: https://chromium-review.googlesource.com/c/angle/angle/+/2785431 Commit-Queue: Jamie Madill <jmadill@chromium.org> Reviewed-by: 's avatarCharlie Lao <cclao@google.com> Reviewed-by: 's avatarTim Van Patten <timvp@google.com>
parent d7859d98
{
"src/libANGLE/Overlay_autogen.cpp":
"77cb799b7e7cd9cea1a04e143ce55162",
"687de489c4864b1204b2ea7f6477a4dd",
"src/libANGLE/Overlay_autogen.h":
"4e4b35f85231fdf717540eff5da6e388",
"b62df749c99e35421e40ee6e620c5c74",
"src/libANGLE/gen_overlay_widgets.py":
"d14bb9becb623817675e4ff758b6d4f4",
"src/libANGLE/overlay_widgets.json":
"3ee58a46e52247a5a366b47c1094e38d"
"0114af385f690a27937bae02341d9bdf"
}
\ No newline at end of file
......@@ -446,6 +446,21 @@ void AppendWidgetDataHelper::AppendVulkanDescriptorSetAllocations(const overlay:
AppendRunningGraphCommon(widget, imageExtent, textWidget, graphWidget, widgetCounts, format);
}
void AppendWidgetDataHelper::AppendVulkanShaderBufferDSHitRate(const overlay::Widget *widget,
const gl::Extents &imageExtent,
TextWidgetData *textWidget,
GraphWidgetData *graphWidget,
OverlayWidgetCounts *widgetCounts)
{
auto format = [](size_t maxValue) {
std::ostringstream text;
text << "Shader Buffer DS Hit Rate (Max: " << maxValue << "%)";
return text.str();
};
AppendRunningGraphCommon(widget, imageExtent, textWidget, graphWidget, widgetCounts, format);
}
void AppendWidgetDataHelper::AppendVulkanDynamicBufferAllocations(const overlay::Widget *widget,
const gl::Extents &imageExtent,
TextWidgetData *textWidget,
......
......@@ -319,6 +319,49 @@ void Overlay::initOverlayWidgets()
}
{
RunningGraph *widget = new RunningGraph(60);
{
const int32_t fontSize = GetFontSize(0, kLargeFont);
const int32_t offsetX = -50;
const int32_t offsetY = 360;
const int32_t width = 6 * static_cast<uint32_t>(widget->runningValues.size());
const int32_t height = 100;
widget->type = WidgetType::RunningGraph;
widget->fontSize = fontSize;
widget->coords[0] = offsetX - width;
widget->coords[1] = offsetY;
widget->coords[2] = offsetX;
widget->coords[3] = offsetY + height;
widget->color[0] = 1.0f;
widget->color[1] = 0.0f;
widget->color[2] = 0.294117647059f;
widget->color[3] = 0.78431372549f;
}
mState.mOverlayWidgets[WidgetId::VulkanShaderBufferDSHitRate].reset(widget);
{
const int32_t fontSize = GetFontSize(kFontLayerSmall, kLargeFont);
const int32_t offsetX =
mState.mOverlayWidgets[WidgetId::VulkanShaderBufferDSHitRate]->coords[0];
const int32_t offsetY =
mState.mOverlayWidgets[WidgetId::VulkanShaderBufferDSHitRate]->coords[1];
const int32_t width = 40 * kFontGlyphWidths[fontSize];
const int32_t height = kFontGlyphHeights[fontSize];
widget->description.type = WidgetType::Text;
widget->description.fontSize = fontSize;
widget->description.coords[0] = offsetX;
widget->description.coords[1] = std::max(offsetY - height, 1);
widget->description.coords[2] = std::min(offsetX + width, -1);
widget->description.coords[3] = offsetY;
widget->description.color[0] = 1.0f;
widget->description.color[1] = 0.0f;
widget->description.color[2] = 0.294117647059f;
widget->description.color[3] = 1.0f;
}
}
{
RunningGraph *widget = new RunningGraph(120);
{
const int32_t fontSize = GetFontSize(0, kLargeFont);
......
......@@ -28,6 +28,8 @@ enum class WidgetId
VulkanWriteDescriptorSetCount,
// Descriptor Set Allocations.
VulkanDescriptorSetAllocations,
// Shader Buffer Descriptor Set Cache Hit Rate.
VulkanShaderBufferDSHitRate,
// Buffer Allocations Made By vk::DynamicBuffer.
VulkanDynamicBufferAllocations,
......@@ -45,6 +47,7 @@ enum class WidgetId
PROC(VulkanSecondaryCommandBufferPoolWaste) \
PROC(VulkanWriteDescriptorSetCount) \
PROC(VulkanDescriptorSetAllocations) \
PROC(VulkanShaderBufferDSHitRate) \
PROC(VulkanDynamicBufferAllocations)
} // namespace gl
......@@ -491,6 +491,7 @@ class Program final : public LabeledObject, public angle::Subject, public HasAtt
// Peek whether there is any running linking tasks.
bool isLinking() const;
bool hasLinkingState() const { return mLinkingState != nullptr; }
bool isLinked() const
{
......
......@@ -134,6 +134,22 @@
}
},
{
"name": "VulkanShaderBufferDSHitRate",
"comment": "Shader Buffer Descriptor Set Cache Hit Rate.",
"type": "RunningGraph(60)",
"color": [255, 0, 75, 200],
"coords": [-50, 360],
"bar_width": 6,
"height": 100,
"description": {
"color": [255, 0, 75, 255],
"coords": ["VulkanShaderBufferDSHitRate.left.align",
"VulkanShaderBufferDSHitRate.top.adjacent"],
"font": "small",
"length": 40
}
},
{
"name": "VulkanDynamicBufferAllocations",
"comment": "Buffer Allocations Made By vk::DynamicBuffer.",
"type": "RunningGraph(120)",
......
......@@ -1956,16 +1956,19 @@ angle::Result ContextVk::handleDirtyDescriptorSetsImpl(vk::CommandBuffer *comman
void ContextVk::syncObjectPerfCounters()
{
uint32_t descriptorSetAllocations = 0;
mPerfCounters.descriptorSetAllocations = 0;
mPerfCounters.shaderBuffersDescriptorSetCacheHits = 0;
mPerfCounters.shaderBuffersDescriptorSetCacheMisses = 0;
// ContextVk's descriptor set allocations
ContextVkPerfCounters contextCounters = getAndResetObjectPerfCounters();
for (uint32_t count : contextCounters.descriptorSetsAllocated)
{
descriptorSetAllocations += count;
mPerfCounters.descriptorSetAllocations += count;
}
// UtilsVk's descriptor set allocations
descriptorSetAllocations += mUtils.getAndResetObjectPerfCounters().descriptorSetsAllocated;
mPerfCounters.descriptorSetAllocations +=
mUtils.getAndResetObjectPerfCounters().descriptorSetsAllocated;
// ProgramExecutableVk's descriptor set allocations
const gl::State &state = getState();
const gl::ShaderProgramManager &shadersAndPrograms = state.getShaderProgramManagerForCapture();
......@@ -1973,16 +1976,25 @@ void ContextVk::syncObjectPerfCounters()
shadersAndPrograms.getProgramsForCaptureAndPerf();
for (const std::pair<GLuint, gl::Program *> &resource : programs)
{
gl::Program *program = resource.second;
if (program->hasLinkingState())
{
continue;
}
ProgramVk *programVk = vk::GetImpl(resource.second);
ProgramExecutablePerfCounters progPerfCounters =
programVk->getExecutable().getAndResetObjectPerfCounters();
for (const uint32_t count : progPerfCounters.descriptorSetsAllocated)
for (uint32_t count : progPerfCounters.descriptorSetAllocations)
{
descriptorSetAllocations += count;
mPerfCounters.descriptorSetAllocations += count;
}
mPerfCounters.shaderBuffersDescriptorSetCacheHits +=
progPerfCounters.descriptorSetCacheHits[DescriptorSetIndex::ShaderResource];
mPerfCounters.shaderBuffersDescriptorSetCacheMisses +=
progPerfCounters.descriptorSetCacheMisses[DescriptorSetIndex::ShaderResource];
}
mPerfCounters.descriptorSetAllocations = descriptorSetAllocations;
}
void ContextVk::updateOverlayOnPresent()
......@@ -2017,6 +2029,22 @@ void ContextVk::updateOverlayOnPresent()
}
{
gl::RunningGraphWidget *shaderBufferHitRate =
overlay->getRunningGraphWidget(gl::WidgetId::VulkanShaderBufferDSHitRate);
size_t numCacheAccesses = mPerfCounters.shaderBuffersDescriptorSetCacheHits +
mPerfCounters.shaderBuffersDescriptorSetCacheMisses;
if (numCacheAccesses > 0)
{
float hitRateFloat =
static_cast<float>(mPerfCounters.shaderBuffersDescriptorSetCacheHits) /
static_cast<float>(numCacheAccesses);
size_t hitRate = static_cast<size_t>(hitRateFloat * 100.0f);
shaderBufferHitRate->add(hitRate);
shaderBufferHitRate->next();
}
}
{
gl::RunningGraphWidget *dynamicBufferAllocations =
overlay->getRunningGraphWidget(gl::WidgetId::VulkanDynamicBufferAllocations);
dynamicBufferAllocations->next();
......
......@@ -99,6 +99,24 @@ constexpr bool IsDynamicDescriptor(VkDescriptorType descriptorType)
}
constexpr VkDescriptorType kStorageBufferDescriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
DescriptorSetIndex CacheTypeToDescriptorSetIndex(VulkanCacheType cacheType)
{
switch (cacheType)
{
case VulkanCacheType::TextureDescriptors:
return DescriptorSetIndex::Texture;
case VulkanCacheType::ShaderBuffersDescriptors:
return DescriptorSetIndex::ShaderResource;
case VulkanCacheType::UniformsAndXfbDescriptors:
return DescriptorSetIndex::UniformsAndXfb;
case VulkanCacheType::DriverUniformsDescriptors:
return DescriptorSetIndex::Internal;
default:
UNREACHABLE();
return DescriptorSetIndex::InvalidEnum;
}
}
} // namespace
DefaultUniformBlock::DefaultUniformBlock() = default;
......@@ -495,7 +513,7 @@ angle::Result ProgramExecutableVk::allocateDescriptorSetAndGetInfo(
&mDescriptorSets[descriptorSetIndex], newPoolAllocatedOut));
mEmptyDescriptorSets[descriptorSetIndex] = VK_NULL_HANDLE;
++mPerfCounters.descriptorSetsAllocated[descriptorSetIndex];
++mPerfCounters.descriptorSetAllocations[descriptorSetIndex];
return angle::Result::Continue;
}
......@@ -1812,7 +1830,7 @@ angle::Result ProgramExecutableVk::updateDescriptorSets(ContextVk *contextVk,
&mDescriptorPoolBindings[descriptorSetIndex],
&mEmptyDescriptorSets[descriptorSetIndex]));
++mPerfCounters.descriptorSetsAllocated[descriptorSetIndex];
++mPerfCounters.descriptorSetAllocations[descriptorSetIndex];
}
descSet = mEmptyDescriptorSets[descriptorSetIndex];
}
......@@ -1856,7 +1874,7 @@ void ProgramExecutableVk::outputCumulativePerfCounters()
for (DescriptorSetIndex descriptorSetIndex : angle::AllEnums<DescriptorSetIndex>())
{
uint32_t count = mCumulativePerfCounters.descriptorSetsAllocated[descriptorSetIndex];
uint32_t count = mCumulativePerfCounters.descriptorSetAllocations[descriptorSetIndex];
if (count > 0)
{
text << " DescriptorSetIndex " << ToUnderlying(descriptorSetIndex) << ": " << count
......@@ -1883,10 +1901,29 @@ void ProgramExecutableVk::outputCumulativePerfCounters()
ProgramExecutablePerfCounters ProgramExecutableVk::getAndResetObjectPerfCounters()
{
mCumulativePerfCounters.descriptorSetsAllocated += mPerfCounters.descriptorSetsAllocated;
mUniformsAndXfbDescriptorsCache.accumulateCacheStats(this);
mTextureDescriptorsCache.accumulateCacheStats(this);
mShaderBufferDescriptorsCache.accumulateCacheStats(this);
mCumulativePerfCounters.descriptorSetAllocations += mPerfCounters.descriptorSetAllocations;
mCumulativePerfCounters.descriptorSetCacheHits += mPerfCounters.descriptorSetCacheHits;
mCumulativePerfCounters.descriptorSetCacheMisses += mPerfCounters.descriptorSetCacheMisses;
ProgramExecutablePerfCounters counters = mPerfCounters;
mPerfCounters.descriptorSetsAllocated = {};
mPerfCounters.descriptorSetAllocations = {};
mPerfCounters.descriptorSetCacheHits = {};
mPerfCounters.descriptorSetCacheMisses = {};
return counters;
}
void ProgramExecutableVk::accumulateCacheStats(VulkanCacheType cacheType,
const CacheStats &cacheStats)
{
DescriptorSetIndex dsIndex = CacheTypeToDescriptorSetIndex(cacheType);
mPerfCounters.descriptorSetCacheHits[dsIndex] +=
static_cast<uint32_t>(cacheStats.getHitCount());
mPerfCounters.descriptorSetCacheMisses[dsIndex] +=
static_cast<uint32_t>(cacheStats.getMissCount());
}
} // namespace rx
......@@ -105,7 +105,9 @@ using DescriptorSetCountList = angle::PackedEnumMap<DescriptorSetIndex, uint32_t
struct ProgramExecutablePerfCounters
{
DescriptorSetCountList descriptorSetsAllocated;
DescriptorSetCountList descriptorSetAllocations;
DescriptorSetCountList descriptorSetCacheHits;
DescriptorSetCountList descriptorSetCacheMisses;
};
class ProgramExecutableVk
......@@ -190,6 +192,7 @@ class ProgramExecutableVk
return mUniformBufferDescriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
}
void accumulateCacheStats(VulkanCacheType cacheType, const CacheStats &cacheStats);
ProgramExecutablePerfCounters getAndResetObjectPerfCounters();
private:
......
......@@ -3553,7 +3553,7 @@ GraphicsPipelineCache::~GraphicsPipelineCache()
void GraphicsPipelineCache::destroy(RendererVk *rendererVk)
{
rendererVk->accumulateCacheStats(VulkanCacheType::GraphicsPipeline, mCacheStats);
accumulateCacheStats(rendererVk);
VkDevice device = rendererVk->getDevice();
......@@ -3696,7 +3696,7 @@ PipelineLayoutCache::~PipelineLayoutCache()
void PipelineLayoutCache::destroy(RendererVk *rendererVk)
{
rendererVk->accumulateCacheStats(VulkanCacheType::PipelineLayout, mCacheStats);
accumulateCacheStats(rendererVk);
VkDevice device = rendererVk->getDevice();
......@@ -3904,20 +3904,22 @@ angle::Result SamplerCache::getSampler(ContextVk *contextVk,
// DriverUniformsDescriptorSetCache implementation.
void DriverUniformsDescriptorSetCache::destroy(RendererVk *rendererVk)
{
rendererVk->accumulateCacheStats(VulkanCacheType::DescriptorSet, mCacheStats);
accumulateCacheStats(rendererVk);
mPayload.clear();
}
// DescriptorSetCache implementation.
template <typename key, VulkanCacheType cacheType>
void DescriptorSetCache<key, cacheType>::destroy(RendererVk *rendererVk)
template <typename Key, VulkanCacheType CacheType>
void DescriptorSetCache<Key, CacheType>::destroy(RendererVk *rendererVk)
{
rendererVk->accumulateCacheStats(cacheType, mCacheStats);
this->accumulateCacheStats(rendererVk);
mPayload.clear();
}
// RendererVk's methods are not accessible in vk_cache_utils.h
// Below declarations are needed to avoid linker errors.
// Unclear why Clang warns about weak vtables in this case.
ANGLE_DISABLE_WEAK_TEMPLATE_VTABLES_WARNING
template class DescriptorSetCache<vk::TextureDescriptorDesc, VulkanCacheType::TextureDescriptors>;
template class DescriptorSetCache<vk::UniformsAndXfbDescriptorDesc,
......@@ -3925,4 +3927,5 @@ template class DescriptorSetCache<vk::UniformsAndXfbDescriptorDesc,
template class DescriptorSetCache<vk::ShaderBuffersDescriptorDesc,
VulkanCacheType::ShaderBuffersDescriptors>;
ANGLE_REENABLE_WEAK_TEMPLATE_VTABLES_WARNING
} // namespace rx
......@@ -1382,8 +1382,8 @@ enum class VulkanCacheType
PipelineLayout,
Sampler,
SamplerYcbcrConversion,
DescriptorSet,
DescriptorSetLayout,
DriverUniformsDescriptors,
TextureDescriptors,
UniformsAndXfbDescriptors,
ShaderBuffersDescriptors,
......@@ -1395,7 +1395,7 @@ enum class VulkanCacheType
class CacheStats final : angle::NonCopyable
{
public:
CacheStats() : mHitCount(0), mMissCount(0) {}
CacheStats() { reset(); }
~CacheStats() {}
ANGLE_INLINE void hit() { mHitCount++; }
......@@ -1406,6 +1406,9 @@ class CacheStats final : angle::NonCopyable
mMissCount += stats.mMissCount;
}
uint64_t getHitCount() const { return mHitCount; }
uint64_t getMissCount() const { return mMissCount; }
ANGLE_INLINE double getHitRatio() const
{
if (mHitCount + mMissCount == 0)
......@@ -1418,11 +1421,35 @@ class CacheStats final : angle::NonCopyable
}
}
void reset()
{
mHitCount = 0;
mMissCount = 0;
}
private:
uint64_t mHitCount;
uint64_t mMissCount;
};
template <VulkanCacheType CacheType>
class HasCacheStats : angle::NonCopyable
{
public:
template <typename Accumulator>
void accumulateCacheStats(Accumulator *accum)
{
accum->accumulateCacheStats(CacheType, mCacheStats);
mCacheStats.reset();
}
protected:
HasCacheStats() = default;
virtual ~HasCacheStats() = default;
CacheStats mCacheStats;
};
// TODO(jmadill): Add cache trimming/eviction.
class RenderPassCache final : angle::NonCopyable
{
......@@ -1479,11 +1506,11 @@ class RenderPassCache final : angle::NonCopyable
};
// TODO(jmadill): Add cache trimming/eviction.
class GraphicsPipelineCache final : angle::NonCopyable
class GraphicsPipelineCache final : public HasCacheStats<VulkanCacheType::GraphicsPipeline>
{
public:
GraphicsPipelineCache();
~GraphicsPipelineCache();
~GraphicsPipelineCache() override;
void destroy(RendererVk *rendererVk);
void release(ContextVk *context);
......@@ -1540,7 +1567,6 @@ class GraphicsPipelineCache final : angle::NonCopyable
vk::PipelineHelper **pipelineOut);
std::unordered_map<vk::GraphicsPipelineDesc, vk::PipelineHelper> mPayload;
CacheStats mCacheStats;
};
class DescriptorSetLayoutCache final : angle::NonCopyable
......@@ -1561,11 +1587,11 @@ class DescriptorSetLayoutCache final : angle::NonCopyable
CacheStats mCacheStats;
};
class PipelineLayoutCache final : angle::NonCopyable
class PipelineLayoutCache final : public HasCacheStats<VulkanCacheType::PipelineLayout>
{
public:
PipelineLayoutCache();
~PipelineLayoutCache();
~PipelineLayoutCache() override;
void destroy(RendererVk *rendererVk);
......@@ -1576,14 +1602,13 @@ class PipelineLayoutCache final : angle::NonCopyable
private:
std::unordered_map<vk::PipelineLayoutDesc, vk::RefCountedPipelineLayout> mPayload;
CacheStats mCacheStats;
};
class SamplerCache final : angle::NonCopyable
class SamplerCache final : public HasCacheStats<VulkanCacheType::Sampler>
{
public:
SamplerCache();
~SamplerCache();
~SamplerCache() override;
void destroy(RendererVk *rendererVk);
......@@ -1593,15 +1618,15 @@ class SamplerCache final : angle::NonCopyable
private:
std::unordered_map<vk::SamplerDesc, vk::RefCountedSampler> mPayload;
CacheStats mCacheStats;
};
// YuvConversion Cache
class SamplerYcbcrConversionCache final : angle::NonCopyable
class SamplerYcbcrConversionCache final
: public HasCacheStats<VulkanCacheType::SamplerYcbcrConversion>
{
public:
SamplerYcbcrConversionCache();
~SamplerYcbcrConversionCache();
~SamplerYcbcrConversionCache() override;
void destroy(RendererVk *rendererVk);
......@@ -1614,15 +1639,15 @@ class SamplerYcbcrConversionCache final : angle::NonCopyable
private:
std::unordered_map<uint64_t, vk::RefCountedSamplerYcbcrConversion> mPayload;
CacheStats mCacheStats;
};
// DescriptorSet Cache
class DriverUniformsDescriptorSetCache final : angle::NonCopyable
class DriverUniformsDescriptorSetCache final
: public HasCacheStats<VulkanCacheType::DriverUniformsDescriptors>
{
public:
DriverUniformsDescriptorSetCache() = default;
~DriverUniformsDescriptorSetCache() { ASSERT(mPayload.empty()); }
~DriverUniformsDescriptorSetCache() override { ASSERT(mPayload.empty()); }
void destroy(RendererVk *rendererVk);
......@@ -1646,40 +1671,38 @@ class DriverUniformsDescriptorSetCache final : angle::NonCopyable
private:
angle::FastIntegerMap<VkDescriptorSet> mPayload;
CacheStats mCacheStats;
};
// Templated Descriptors Cache
template <typename key, VulkanCacheType cacheType>
class DescriptorSetCache final : angle::NonCopyable
template <typename Key, VulkanCacheType CacheType>
class DescriptorSetCache final : public HasCacheStats<CacheType>
{
public:
DescriptorSetCache() = default;
~DescriptorSetCache() { ASSERT(mPayload.empty()); }
~DescriptorSetCache() override { ASSERT(mPayload.empty()); }
void destroy(RendererVk *rendererVk);
ANGLE_INLINE bool get(const key &desc, VkDescriptorSet *descriptorSet)
ANGLE_INLINE bool get(const Key &desc, VkDescriptorSet *descriptorSet)
{
auto iter = mPayload.find(desc);
if (iter != mPayload.end())
{
*descriptorSet = iter->second;
mCacheStats.hit();
this->mCacheStats.hit();
return true;
}
mCacheStats.miss();
this->mCacheStats.miss();
return false;
}
ANGLE_INLINE void insert(const key &desc, VkDescriptorSet descriptorSet)
ANGLE_INLINE void insert(const Key &desc, VkDescriptorSet descriptorSet)
{
mPayload.emplace(desc, descriptorSet);
}
private:
angle::HashMap<key, VkDescriptorSet> mPayload;
CacheStats mCacheStats;
angle::HashMap<Key, VkDescriptorSet> mPayload;
};
// Only 1 driver uniform binding is used.
......
......@@ -2504,7 +2504,7 @@ angle::Result DynamicDescriptorPool::allocateNewPool(ContextVk *contextVk)
// This pool is getting hot, so grow its max size to try and prevent allocating another pool in
// the future.
if (mMaxSetsPerPool < KMaxSetsPerPoolMax)
if (mMaxSetsPerPool < kMaxSetsPerPoolMax)
{
mMaxSetsPerPool *= mMaxSetsPerPoolMultiplier;
}
......
......@@ -332,7 +332,7 @@ class DynamicDescriptorPool final : angle::NonCopyable
private:
angle::Result allocateNewPool(ContextVk *contextVk);
static constexpr uint32_t KMaxSetsPerPoolMax = 512;
static constexpr uint32_t kMaxSetsPerPoolMax = 512;
static uint32_t mMaxSetsPerPool;
static uint32_t mMaxSetsPerPoolMultiplier;
size_t mCurrentPoolIndex;
......
......@@ -876,6 +876,8 @@ struct PerfCounters
uint32_t stencilAttachmentResolves;
uint32_t readOnlyDepthStencilRenderPasses;
uint32_t descriptorSetAllocations;
uint32_t shaderBuffersDescriptorSetCacheHits;
uint32_t shaderBuffersDescriptorSetCacheMisses;
};
// A Vulkan image level index.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment