Vulkan: Allow pbuffer usage if no window support

src/libANGLE/renderer/vulkan/DisplayVk.h

@@ -83,9 +83,10 @@ class DisplayVk : public DisplayImpl, public vk::Context
     virtual const char *getWSILayer() const;
 
     // Determine if a config with given formats and sample counts is supported.  This callback may
-    // modify the config to add or remove platform specific attributes such as nativeVisualID before
-    // returning a bool to indicate if the config should be supported.
-    virtual bool checkConfigSupport(egl::Config *config) = 0;
+    // modify the config to add or remove platform specific attributes such as nativeVisualID.  If
+    // the config is not supported by the window system, it removes the EGL_WINDOW_BIT from
+    // surfaceType, which would still allow the config to be used for pbuffers.
+    virtual void checkConfigSupport(egl::Config *config) = 0;
 
     ANGLE_NO_DISCARD bool getScratchBuffer(size_t requestedSizeBytes,
                                            angle::MemoryBuffer **scratchBufferOut) const;

src/libANGLE/renderer/vulkan/android/DisplayVkAndroid.cpp

@@ -55,11 +55,10 @@ egl::ConfigSet DisplayVkAndroid::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkAndroid::checkConfigSupport(egl::Config *config)
+void DisplayVkAndroid::checkConfigSupport(egl::Config *config)
 {
     // TODO(geofflang): Test for native support and modify the config accordingly.
     // anglebug.com/2692
-    return true;
 }
 
 egl::Error DisplayVkAndroid::validateImageClientBuffer(const gl::Context *context,

src/libANGLE/renderer/vulkan/android/DisplayVkAndroid.h

@@ -27,7 +27,7 @@ class DisplayVkAndroid : public DisplayVk
                                        EGLNativeWindowType window) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     egl::Error validateImageClientBuffer(const gl::Context *context,
                                          EGLenum target,

src/libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.cpp

@@ -35,11 +35,10 @@ egl::ConfigSet DisplayVkFuchsia::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkFuchsia::checkConfigSupport(egl::Config *config)
+void DisplayVkFuchsia::checkConfigSupport(egl::Config *config)
 {
     // TODO(geofflang): Test for native support and modify the config accordingly.
     // anglebug.com/2692
-    return true;
 }
 
 const char *DisplayVkFuchsia::getWSIExtension() const

src/libANGLE/renderer/vulkan/fuchsia/DisplayVkFuchsia.h

@@ -26,7 +26,7 @@ class DisplayVkFuchsia : public DisplayVk
                                        EGLNativeWindowType window) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     const char *getWSIExtension() const override;
     const char *getWSILayer() const override;

src/libANGLE/renderer/vulkan/ggp/DisplayVkGGP.cpp

@@ -35,10 +35,7 @@ egl::ConfigSet DisplayVkGGP::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkGGP::checkConfigSupport(egl::Config *config)
-{
-    return true;
-}
+void DisplayVkGGP::checkConfigSupport(egl::Config *config) {}
 
 const char *DisplayVkGGP::getWSIExtension() const
 {

src/libANGLE/renderer/vulkan/ggp/DisplayVkGGP.h

@@ -25,7 +25,7 @@ class DisplayVkGGP : public DisplayVk
                                        EGLNativeWindowType window) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     const char *getWSIExtension() const override;
 };

src/libANGLE/renderer/vulkan/mac/DisplayVkMac.h

@@ -31,7 +31,7 @@ class DisplayVkMac : public DisplayVk
                                                const egl::AttributeMap &attribs) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     void generateExtensions(egl::DisplayExtensions *outExtensions) const override;
 

src/libANGLE/renderer/vulkan/mac/DisplayVkMac.mm

@@ -52,11 +52,10 @@ egl::ConfigSet DisplayVkMac::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkMac::checkConfigSupport(egl::Config *config)
+void DisplayVkMac::checkConfigSupport(egl::Config *config)
 {
     // TODO(geofflang): Test for native support and modify the config accordingly.
     // anglebug.com/2692
-    return true;
 }
 
 const char *DisplayVkMac::getWSIExtension() const

src/libANGLE/renderer/vulkan/vk_caps_utils.cpp

@@ -628,11 +628,13 @@ EGLint ComputeMaximumPBufferPixels(const VkPhysicalDeviceProperties &physicalDev
 
 // Generates a basic config for a combination of color format, depth stencil format and sample
 // count.
-egl::Config GenerateDefaultConfig(const RendererVk *renderer,
+egl::Config GenerateDefaultConfig(DisplayVk *display,
                                   const gl::InternalFormat &colorFormat,
                                   const gl::InternalFormat &depthStencilFormat,
                                   EGLint sampleCount)
 {
+    const RendererVk *renderer = display->getRenderer();
+
     const VkPhysicalDeviceProperties &physicalDeviceProperties =
         renderer->getPhysicalDeviceProperties();
     gl::Version maxSupportedESVersion = renderer->getMaxSupportedESVersion();
@@ -683,6 +685,11 @@ egl::Config GenerateDefaultConfig(const RendererVk *renderer,
     config.colorComponentType =
         gl_egl::GLComponentTypeToEGLColorComponentType(colorFormat.componentType);
 
+    // Vulkan always supports off-screen rendering.  Check the config with display to see if it can
+    // also have window support.  If not, the following call should automatically remove
+    // EGL_WINDOW_BIT.
+    display->checkConfigSupport(&config);
+
     return config;
 }
 
@@ -749,12 +756,9 @@ egl::ConfigSet GenerateConfigs(const GLenum *colorFormats,
 
             for (EGLint sampleCount : *configSampleCounts)
             {
-                egl::Config config = GenerateDefaultConfig(display->getRenderer(), colorFormatInfo,
+                egl::Config config = GenerateDefaultConfig(display, colorFormatInfo,
                                                            depthStencilFormatInfo, sampleCount);
-                if (display->checkConfigSupport(&config))
-                {
-                    configSet.add(config);
-                }
+                configSet.add(config);
             }
         }
     }

src/libANGLE/renderer/vulkan/win32/DisplayVkWin32.cpp

@@ -139,7 +139,7 @@ egl::ConfigSet DisplayVkWin32::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkWin32::checkConfigSupport(egl::Config *config)
+void DisplayVkWin32::checkConfigSupport(egl::Config *config)
 {
     const vk::Format &formatVk = this->getRenderer()->getFormat(config->renderTargetFormat);
     VkFormat nativeFormat      = formatVk.vkImageFormat;
@@ -149,18 +149,19 @@ bool DisplayVkWin32::checkConfigSupport(egl::Config *config)
     // supported format will be returned.
     if (mSurfaceFormats.size() == 1u && mSurfaceFormats[0].format == VK_FORMAT_UNDEFINED)
     {
-        return true;
+        return;
     }
 
     for (const VkSurfaceFormatKHR &surfaceFormat : mSurfaceFormats)
     {
         if (surfaceFormat.format == nativeFormat)
         {
-            return true;
+            return;
         }
     }
 
-    return false;
+    // No window support for this config.
+    config->surfaceType &= ~EGL_WINDOW_BIT;
 }
 
 const char *DisplayVkWin32::getWSIExtension() const

src/libANGLE/renderer/vulkan/win32/DisplayVkWin32.h

@@ -29,7 +29,7 @@ class DisplayVkWin32 : public DisplayVk
                                        EGLNativeWindowType window) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     const char *getWSIExtension() const override;
 

src/libANGLE/renderer/vulkan/xcb/DisplayVkXcb.cpp

@@ -12,6 +12,7 @@
 #include <X11/Xutil.h>
 #include <xcb/xcb.h>
 
+#include "common/system_utils.h"
 #include "libANGLE/Display.h"
 #include "libANGLE/renderer/vulkan/vk_caps_utils.h"
 #include "libANGLE/renderer/vulkan/xcb/WindowSurfaceVkXcb.h"
@@ -90,8 +91,17 @@ egl::ConfigSet DisplayVkXcb::generateConfigs()
     return egl_vk::GenerateConfigs(kColorFormats, egl_vk::kConfigDepthStencilFormats, this);
 }
 
-bool DisplayVkXcb::checkConfigSupport(egl::Config *config)
+void DisplayVkXcb::checkConfigSupport(egl::Config *config)
 {
+    // If no window system, cannot support windows.
+    static bool sNoX11Display = angle::GetEnvironmentVar("DISPLAY").empty();
+    if (sNoX11Display)
+    {
+        // No window support if no X11.
+        config->surfaceType &= ~EGL_WINDOW_BIT;
+        return;
+    }
+
     // TODO(geofflang): Test for native support and modify the config accordingly.
     // http://anglebug.com/2692
 
@@ -105,8 +115,6 @@ bool DisplayVkXcb::checkConfigSupport(egl::Config *config)
     // Visual id is root_visual of the screen
     config->nativeVisualID   = screen->root_visual;
     config->nativeVisualType = GetXcbVisualType(screen);
-
-    return true;
 }
 
 const char *DisplayVkXcb::getWSIExtension() const

src/libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h

@@ -31,7 +31,7 @@ class DisplayVkXcb : public DisplayVk
                                        EGLNativeWindowType window) override;
 
     egl::ConfigSet generateConfigs() override;
-    bool checkConfigSupport(egl::Config *config) override;
+    void checkConfigSupport(egl::Config *config) override;
 
     const char *getWSIExtension() const override;
     angle::Result waitNativeImpl() override;