Allocate executable memory backed by named mmaps on Linux.

src/Common/Memory.cpp

@@ -24,7 +24,9 @@
 	#include <windows.h>
 	#include <intrin.h>
 #else
+	#include <errno.h>
 	#include <sys/mman.h>
+	#include <stdlib.h>
 	#include <unistd.h>
 #endif
 
@@ -39,6 +41,93 @@
 
 namespace sw
 {
+namespace
+{
+struct Allocation
+{
+//	size_t bytes;
+	unsigned char *block;
+};
+
+void *allocateRaw(size_t bytes, size_t alignment)
+{
+	ASSERT((alignment & (alignment - 1)) == 0);   // Power of 2 alignment.
+
+	#if defined(LINUX_ENABLE_NAMED_MMAP)
+		void *allocation;
+		int result = posix_memalign(&allocation, alignment, bytes);
+		if(result != 0)
+		{
+			errno = result;
+			allocation = nullptr;
+		}
+		return allocation;
+	#else
+		unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
+		unsigned char *aligned = nullptr;
+
+		if(block)
+		{
+			aligned = (unsigned char*)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
+			Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
+
+		//	allocation->bytes = bytes;
+			allocation->block = block;
+		}
+
+		return aligned;
+	#endif
+}
+
+#if defined(LINUX_ENABLE_NAMED_MMAP)
+// Create a file descriptor for anonymous memory with the given
+// name. Returns -1 on failure.
+// TODO: remove once libc wrapper exists.
+int memfd_create(const char* name, unsigned int flags)
+{
+	#if __aarch64__
+	#define __NR_memfd_create 279
+	#elif __arm__
+	#define __NR_memfd_create 279
+	#elif __powerpc64__
+	#define __NR_memfd_create 360
+	#elif __i386__
+	#define __NR_memfd_create 356
+	#elif __x86_64__
+	#define __NR_memfd_create 319
+	#endif /* __NR_memfd_create__ */
+	#ifdef __NR_memfd_create
+		// In the event of no system call this returns -1 with errno set
+		// as ENOSYS.
+		return syscall(__NR_memfd_create, name, flags);
+	#else
+		return -1;
+	#endif
+}
+
+// Returns a file descriptor for use with an anonymous mmap, if
+// memfd_create fails, -1 is returned. Note, the mappings should be
+// MAP_PRIVATE so that underlying pages aren't shared.
+int anonymousFd()
+{
+	static int fd = memfd_create("SwiftShader JIT", 0);
+	return fd;
+}
+
+// Ensure there is enough space in the "anonymous" fd for length.
+void ensureAnonFileSize(int anonFd, size_t length)
+{
+	static size_t fileSize = 0;
+	if(length > fileSize)
+	{
+		ftruncate(anonFd, length);
+		fileSize = length;
+	}
+}
+#endif  // defined(LINUX_ENABLE_NAMED_MMAP)
+
+}  // anonymous namespace
+
 size_t memoryPageSize()
 {
 	static int pageSize = 0;
@@ -57,29 +146,6 @@ size_t memoryPageSize()
 	return pageSize;
 }
 
-struct Allocation
-{
-//	size_t bytes;
-	unsigned char *block;
-};
-
-inline void *allocateRaw(size_t bytes, size_t alignment)
-{
-	unsigned char *block = new unsigned char[bytes + sizeof(Allocation) + alignment];
-	unsigned char *aligned = nullptr;
-
-	if(block)
-	{
-		aligned = (unsigned char*)((uintptr_t)(block + sizeof(Allocation) + alignment - 1) & -(intptr_t)alignment);
-		Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
-
-	//	allocation->bytes = bytes;
-		allocation->block = block;
-	}
-
-	return aligned;
-}
-
 void *allocate(size_t bytes, size_t alignment)
 {
 	void *memory = allocateRaw(bytes, alignment);
@@ -94,20 +160,50 @@ void *allocate(size_t bytes, size_t alignment)
 
 void deallocate(void *memory)
 {
-	if(memory)
-	{
-		unsigned char *aligned = (unsigned char*)memory;
-		Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
+	#if defined(LINUX_ENABLE_NAMED_MMAP)
+		free(memory);
+	#else
+		if(memory)
+		{
+			unsigned char *aligned = (unsigned char*)memory;
+			Allocation *allocation = (Allocation*)(aligned - sizeof(Allocation));
 
-		delete[] allocation->block;
-	}
+			delete[] allocation->block;
+		}
+	#endif
 }
 
 void *allocateExecutable(size_t bytes)
 {
 	size_t pageSize = memoryPageSize();
+	size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
+	void *mapping;
+
+	#if defined(LINUX_ENABLE_NAMED_MMAP)
+		// Try to name the memory region for the executable code,
+		// to aid profilers.
+		int anonFd = anonymousFd();
+		if(anonFd == -1)
+		{
+			mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
+			               MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+		}
+		else
+		{
+			ensureAnonFileSize(anonFd, length);
+			mapping = mmap(nullptr, length, PROT_READ | PROT_WRITE,
+			               MAP_PRIVATE, anonFd, 0);
+		}
+
+		if(mapping == MAP_FAILED)
+		{
+			mapping = nullptr;
+		}
+	#else
+		mapping = allocate(length, pageSize);
+	#endif
 
-	return allocate((bytes + pageSize - 1) & ~(pageSize - 1), pageSize);
+	return mapping;
 }
 
 void markExecutable(void *memory, size_t bytes)
@@ -125,11 +221,15 @@ void deallocateExecutable(void *memory, size_t bytes)
 	#if defined(_WIN32)
 		unsigned long oldProtection;
 		VirtualProtect(memory, bytes, PAGE_READWRITE, &oldProtection);
+		deallocate(memory);
+	#elif defined(LINUX_ENABLE_NAMED_MMAP)
+		size_t pageSize = memoryPageSize();
+		size_t length = (bytes + pageSize - 1) & ~(pageSize - 1);
+		munmap(memory, length);
 	#else
 		mprotect(memory, bytes, PROT_READ | PROT_WRITE);
+		deallocate(memory);
 	#endif
-
-	deallocate(memory);
 }
 
 void clear(uint16_t *memory, uint16_t element, size_t count)