Punt quadlayout image<->buffer copies to blitter

src/Device/Blitter.cpp

@@ -19,6 +19,7 @@
 #include "System/Memory.hpp"
 #include "Vulkan/VkDebug.hpp"
 #include "Vulkan/VkImage.hpp"
+#include "Vulkan/VkBuffer.hpp"
 
 #include <utility>
 
@@ -1549,6 +1550,138 @@ namespace sw
 		return blitRoutine;
 	}
 
+	void Blitter::blitToBuffer(const vk::Image *src, VkImageSubresourceLayers subresource, VkOffset3D offset, VkExtent3D extent, uint8_t *dst, int bufferRowPitch, int bufferSlicePitch)
+	{
+		auto aspect = static_cast<VkImageAspectFlagBits>(subresource.aspectMask);
+		auto format = src->getFormat(aspect);
+		State state(format, format.getNonQuadLayoutFormat(), VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_1_BIT,
+					{false, false});
+
+		Routine *blitRoutine = getRoutine(state);
+		if(!blitRoutine)
+		{
+			return;
+		}
+
+		void(*blitFunction)(const BlitData *data) = (void(*)(const BlitData*))blitRoutine->getEntry();
+
+		BlitData data =
+		{
+			nullptr, // source
+			dst, // dest
+			src->rowPitchBytes(aspect, subresource.mipLevel),   // sPitchB
+			bufferRowPitch,   // dPitchB
+			src->slicePitchBytes(aspect, subresource.mipLevel), // sSliceB
+			bufferSlicePitch, // dSliceB
+
+			0, 0, 1, 1,
+
+			0, // y0d
+			static_cast<int>(extent.height), // y1d
+			0, // x0d
+			static_cast<int>(extent.width), // x1d
+
+			static_cast<int>(extent.width), // sWidth
+			static_cast<int>(extent.height) // sHeight;
+		};
+
+		VkOffset3D srcOffset = { 0, 0, offset.z };
+
+		VkImageSubresourceLayers srcSubresLayers = subresource;
+		srcSubresLayers.layerCount = 1;
+
+		VkImageSubresourceRange srcSubresRange =
+		{
+			subresource.aspectMask,
+			subresource.mipLevel,
+			1,
+			subresource.baseArrayLayer,
+			subresource.layerCount
+		};
+
+		uint32_t lastLayer = src->getLastLayerIndex(srcSubresRange);
+
+		for(; srcSubresLayers.baseArrayLayer <= lastLayer; srcSubresLayers.baseArrayLayer++)
+		{
+			srcOffset.z = offset.z;
+
+			for(auto i = 0u; i < extent.depth; i++)
+			{
+				data.source = src->getTexelPointer(srcOffset, srcSubresLayers);
+				ASSERT(data.source < src->end());
+				blitFunction(&data);
+				srcOffset.z++;
+				data.dest = (dst += bufferSlicePitch);
+			}
+		}
+	}
+
+	void Blitter::blitFromBuffer(const vk::Image *dst, VkImageSubresourceLayers subresource, VkOffset3D offset, VkExtent3D extent, uint8_t *src, int bufferRowPitch, int bufferSlicePitch)
+	{
+		auto aspect = static_cast<VkImageAspectFlagBits>(subresource.aspectMask);
+		auto format = dst->getFormat(aspect);
+		State state(format.getNonQuadLayoutFormat(), format, VK_SAMPLE_COUNT_1_BIT, VK_SAMPLE_COUNT_1_BIT,
+					{false, false});
+
+		Routine *blitRoutine = getRoutine(state);
+		if(!blitRoutine)
+		{
+			return;
+		}
+
+		void(*blitFunction)(const BlitData *data) = (void(*)(const BlitData*))blitRoutine->getEntry();
+
+		BlitData data =
+		{
+			src, // source
+			nullptr, // dest
+			bufferRowPitch,   // sPitchB
+			dst->rowPitchBytes(aspect, subresource.mipLevel),   // dPitchB
+			bufferSlicePitch, // sSliceB
+			dst->slicePitchBytes(aspect, subresource.mipLevel), // dSliceB
+
+			0, 0, 1, 1,
+
+			offset.y, // y0d
+			static_cast<int>(offset.y + extent.height), // y1d
+			offset.x, // x0d
+			static_cast<int>(offset.x + extent.width), // x1d
+
+			static_cast<int>(extent.width), // sWidth
+			static_cast<int>(extent.height) // sHeight;
+		};
+
+		VkOffset3D dstOffset = { 0, 0, offset.z };
+
+		VkImageSubresourceLayers dstSubresLayers = subresource;
+		dstSubresLayers.layerCount = 1;
+
+		VkImageSubresourceRange dstSubresRange =
+		{
+			subresource.aspectMask,
+			subresource.mipLevel,
+			1,
+			subresource.baseArrayLayer,
+			subresource.layerCount
+		};
+
+		uint32_t lastLayer = dst->getLastLayerIndex(dstSubresRange);
+
+		for(; dstSubresLayers.baseArrayLayer <= lastLayer; dstSubresLayers.baseArrayLayer++)
+		{
+			dstOffset.z = offset.z;
+
+			for(auto i = 0u; i < extent.depth; i++)
+			{
+				data.dest = dst->getTexelPointer(dstOffset, dstSubresLayers);
+				ASSERT(data.dest < dst->end());
+				blitFunction(&data);
+				dstOffset.z++;
+				data.source = (src += bufferSlicePitch);
+			}
+		}
+	}
+
 	void Blitter::blit(const vk::Image *src, vk::Image *dst, VkImageBlit region, VkFilter filter)
 	{
 		if(dst->getFormat() == VK_FORMAT_UNDEFINED)

src/Device/Blitter.hpp

@@ -25,6 +25,7 @@
 namespace vk
 {
 	class Image;
+	class Buffer;
 }
 
 namespace sw
@@ -114,6 +115,8 @@ namespace sw
 		void clear(void *pixel, vk::Format format, vk::Image *dest, const vk::Format& viewFormat, const VkImageSubresourceRange& subresourceRange, const VkRect2D* renderArea = nullptr);
 
 		void blit(const vk::Image *src, vk::Image *dst, VkImageBlit region, VkFilter filter);
+		void blitToBuffer(const vk::Image *src, VkImageSubresourceLayers subresource, VkOffset3D offset, VkExtent3D extent, uint8_t *dst, int bufferRowPitch, int bufferSlicePitch);
+		void blitFromBuffer(const vk::Image *dst, VkImageSubresourceLayers subresource, VkOffset3D offset, VkExtent3D extent, uint8_t *src, int bufferRowPitch, int bufferSlicePitch);
 
 		void updateBorders(vk::Image* image, const VkImageSubresourceLayers& subresourceLayers);
 

src/Vulkan/VkFormat.cpp

@@ -131,6 +131,17 @@ bool Format::hasQuadLayout() const
 	}
 }
 
+VkFormat Format::getNonQuadLayoutFormat() const
+{
+	switch(format)
+	{
+	case VK_FORMAT_S8_UINT:
+		return VK_FORMAT_R8_UINT;
+	default:
+		return format;
+	}
+}
+
 bool Format::isSRGBformat() const
 {
 	switch(format)

src/Vulkan/VkFormat.h

@@ -39,10 +39,9 @@ public:
 	bool isStencil() const;
 	bool isDepth() const;
 	bool hasQuadLayout() const;
+	VkFormat getNonQuadLayoutFormat() const;
 
 	bool isSRGBformat() const;
-	bool isSRGBreadable() const;
-	bool isSRGBwritable() const;
 	bool isFloatFormat() const;
 
 	bool isCompatible(const Format& other) const;

src/Vulkan/VkImage.cpp

@@ -285,47 +285,59 @@ void Image::copy(VkBuffer buf, const VkBufferImageCopy& region, bool bufferIsSou
 		UNIMPLEMENTED("imageSubresource");
 	}
 
-	VkImageAspectFlagBits aspect = static_cast<VkImageAspectFlagBits>(region.imageSubresource.aspectMask);
+	auto aspect = static_cast<VkImageAspectFlagBits>(region.imageSubresource.aspectMask);
 
 	Format copyFormat = getFormat(aspect);
 
+	VkExtent3D imageExtent = imageExtentInBlocks(region.imageExtent, aspect);
+	VkExtent2D bufferExtent = bufferExtentInBlocks({ imageExtent.width, imageExtent.height }, region);
+	int bytesPerBlock = copyFormat.bytesPerBlock();
+	int bufferRowPitchBytes = bufferExtent.width * bytesPerBlock;
+	int bufferSlicePitchBytes = bufferExtent.height * bufferRowPitchBytes;
+
+	Buffer* buffer = Cast(buf);
+	uint8_t* bufferMemory = static_cast<uint8_t*>(buffer->getOffsetPointer(region.bufferOffset));
+
 	if (copyFormat.hasQuadLayout())
 	{
-		UNIMPLEMENTED("Quad layout copies");
+		if (bufferIsSource)
+		{
+			return device->getBlitter()->blitFromBuffer(this, region.imageSubresource, region.imageOffset,
+														region.imageExtent, bufferMemory, bufferRowPitchBytes,
+														bufferSlicePitchBytes);
+		}
+		else
+		{
+			return device->getBlitter()->blitToBuffer(this, region.imageSubresource, region.imageOffset,
+													  region.imageExtent, bufferMemory, bufferRowPitchBytes,
+													  bufferSlicePitchBytes);
+		}
 	}
 
-	VkExtent3D mipLevelExtent = getMipLevelExtent(region.imageSubresource.mipLevel);
-	VkExtent3D imageExtent = imageExtentInBlocks(region.imageExtent, aspect);
-	VkExtent2D bufferExtent = bufferExtentInBlocks({ imageExtent.width, imageExtent.height }, region);
-	int imageBytesPerBlock = copyFormat.bytesPerBlock();
+	uint8_t* imageMemory = static_cast<uint8_t*>(getTexelPointer(region.imageOffset, region.imageSubresource));
+	uint8_t* srcMemory = bufferIsSource ? bufferMemory : imageMemory;
+	uint8_t* dstMemory = bufferIsSource ? imageMemory : bufferMemory;
 	int imageRowPitchBytes = rowPitchBytes(aspect, region.imageSubresource.mipLevel);
 	int imageSlicePitchBytes = slicePitchBytes(aspect, region.imageSubresource.mipLevel);
-	int bufferRowPitchBytes = bufferExtent.width * imageBytesPerBlock;
-	int bufferSlicePitchBytes = bufferExtent.height * bufferRowPitchBytes;
 
 	int srcSlicePitchBytes = bufferIsSource ? bufferSlicePitchBytes : imageSlicePitchBytes;
 	int dstSlicePitchBytes = bufferIsSource ? imageSlicePitchBytes : bufferSlicePitchBytes;
 	int srcRowPitchBytes = bufferIsSource ? bufferRowPitchBytes : imageRowPitchBytes;
 	int dstRowPitchBytes = bufferIsSource ? imageRowPitchBytes : bufferRowPitchBytes;
 
+	VkExtent3D mipLevelExtent = getMipLevelExtent(region.imageSubresource.mipLevel);
 	bool isSinglePlane = (imageExtent.depth == 1);
 	bool isSingleLine  = (imageExtent.height == 1) && isSinglePlane;
 	bool isEntireLine  = (imageExtent.width == mipLevelExtent.width) &&
-	                     (imageRowPitchBytes == bufferRowPitchBytes);
+						 (imageRowPitchBytes == bufferRowPitchBytes);
 	bool isEntirePlane = isEntireLine && (imageExtent.height == mipLevelExtent.height) &&
-	                     (imageSlicePitchBytes == bufferSlicePitchBytes);
-
-	Buffer* buffer = Cast(buf);
-	uint8_t* bufferMemory = static_cast<uint8_t*>(buffer->getOffsetPointer(region.bufferOffset));
-	uint8_t* imageMemory = static_cast<uint8_t*>(getTexelPointer(region.imageOffset, region.imageSubresource));
-	uint8_t* srcMemory = bufferIsSource ? bufferMemory : imageMemory;
-	uint8_t* dstMemory = bufferIsSource ? imageMemory : bufferMemory;
+						 (imageSlicePitchBytes == bufferSlicePitchBytes);
 
 	VkDeviceSize copySize = 0;
 	VkDeviceSize bufferLayerSize = 0;
 	if(isSingleLine)
 	{
-		copySize = imageExtent.width * imageBytesPerBlock;
+		copySize = imageExtent.width * bytesPerBlock;
 		bufferLayerSize = copySize;
 	}
 	else if(isEntireLine && isSinglePlane)
@@ -345,7 +357,7 @@ void Image::copy(VkBuffer buf, const VkBufferImageCopy& region, bool bufferIsSou
 	}
 	else // Copy line by line
 	{
-		copySize = imageExtent.width * imageBytesPerBlock;
+		copySize = imageExtent.width * bytesPerBlock;
 		bufferLayerSize = copySize * imageExtent.depth * imageExtent.height;
 	}