Reland "Fix unit test for setting locale creating malformed HLSL shader code

src/compiler/translator/AtomicCounterFunctionHLSL.cpp

@@ -9,6 +9,7 @@
 
 #include "compiler/translator/AtomicCounterFunctionHLSL.h"
 
+#include "compiler/translator/Common.h"
 #include "compiler/translator/ImmutableStringBuilder.h"
 #include "compiler/translator/InfoSink.h"
 #include "compiler/translator/IntermNode.h"
@@ -92,7 +93,7 @@ void AtomicCounterFunctionHLSL::atomicCounterFunctionHeader(TInfoSinkBase &out)
 
 ImmutableString getAtomicCounterNameForBinding(int binding)
 {
-    std::stringstream counterName;
+    std::stringstream counterName = sh::InitializeStream<std::stringstream>();
     counterName << kAtomicCounterBaseName << binding;
     return ImmutableString(counterName.str());
 }

src/compiler/translator/BuiltInFunctionEmulatorGLSL.cpp

@@ -5,6 +5,7 @@
 //
 
 #include "compiler/translator/BuiltInFunctionEmulatorGLSL.h"
+
 #include "angle_gl.h"
 #include "compiler/translator/BuiltInFunctionEmulator.h"
 #include "compiler/translator/VersionGLSL.h"
@@ -87,7 +88,7 @@ void InitBuiltInAtanFunctionEmulatorForGLSLWorkarounds(BuiltInFunctionEmulator *
     };
     for (int dim = 2; dim <= 4; ++dim)
     {
-        std::stringstream ss;
+        std::stringstream ss = sh::InitializeStream<std::stringstream>();
         ss << "emu_precision vec" << dim << " atan_emu(emu_precision vec" << dim
            << " y, emu_precision vec" << dim << " x)\n"
            << "{\n"

src/compiler/translator/CallDAG.cpp

@@ -171,7 +171,7 @@ class CallDAG::CallDAGCreator : public TIntermTraverser
 
         InitResult result = INITDAG_SUCCESS;
 
-        std::stringstream errorStream;
+        std::stringstream errorStream = sh::InitializeStream<std::stringstream>();
 
         while (!functionsToProcess.empty())
         {

src/compiler/translator/Common.h

@@ -125,6 +125,15 @@ inline const char *AllocatePoolCharArray(const char *str, size_t strLength)
     return buffer;
 }
 
+// Initialize a new stream which must be imbued with the classic locale
+template <typename T>
+T InitializeStream()
+{
+    T stream;
+    stream.imbue(std::locale::classic());
+    return stream;
+}
+
 }  // namespace sh
 
 namespace std

src/compiler/translator/Compiler.cpp

@@ -68,7 +68,7 @@ void DumpFuzzerCase(char const *const *shaderStrings,
 {
     static int fileIndex = 0;
 
-    std::ostringstream o;
+    std::ostringstream o = sh::InitializeStream<std::ostringstream>();
     o << "corpus/" << fileIndex++ << ".sample";
     std::string s = o.str();
 
@@ -940,7 +940,7 @@ bool TCompiler::initBuiltInSymbolTable(const ShBuiltInResources &resources)
 
 void TCompiler::setResourceString()
 {
-    std::ostringstream strstream;
+    std::ostringstream strstream = sh::InitializeStream<std::ostringstream>();
 
     // clang-format off
     strstream << ":MaxVertexAttribs:" << mResources.MaxVertexAttribs
@@ -1094,7 +1094,7 @@ bool TCompiler::checkCallDepth()
 
     for (size_t i = 0; i < mCallDag.size(); i++)
     {
-        int depth    = 0;
+        int depth                     = 0;
         const CallDAG::Record &record = mCallDag.getRecordFromIndex(i);
 
         for (const int &calleeIndex : record.callees)
@@ -1107,7 +1107,7 @@ bool TCompiler::checkCallDepth()
         if (depth >= mResources.MaxCallStackDepth)
         {
             // Trace back the function chain to have a meaningful info log.
-            std::stringstream errorStream;
+            std::stringstream errorStream = sh::InitializeStream<std::stringstream>();
             errorStream << "Call stack too deep (larger than " << mResources.MaxCallStackDepth
                         << ") with the following call chain: "
                         << record.node->getFunction()->name();

src/compiler/translator/DirectiveHandler.cpp

@@ -10,6 +10,7 @@
 
 #include "angle_gl.h"
 #include "common/debug.h"
+#include "compiler/translator/Common.h"
 #include "compiler/translator/Diagnostics.h"
 
 namespace sh
@@ -190,7 +191,7 @@ void TDirectiveHandler::handleVersion(const angle::pp::SourceLocation &loc, int
     }
     else
     {
-        std::stringstream stream;
+        std::stringstream stream = sh::InitializeStream<std::stringstream>();
         stream << version;
         std::string str = stream.str();
         mDiagnostics.error(loc, "version number not supported", str.c_str());

src/compiler/translator/InfoSink.cpp

@@ -69,7 +69,7 @@ TInfoSinkBase &TInfoSinkBase::operator<<(const TType &type)
 
 void TInfoSinkBase::location(int file, int line)
 {
-    TPersistStringStream stream;
+    TPersistStringStream stream = sh::InitializeStream<TPersistStringStream>();
     if (line)
         stream << file << ":" << line;
     else

src/compiler/translator/InfoSink.h

@@ -41,7 +41,7 @@ class TInfoSinkBase
     template <typename T>
     TInfoSinkBase &operator<<(const T &t)
     {
-        TPersistStringStream stream;
+        TPersistStringStream stream = sh::InitializeStream<TPersistStringStream>();
         stream << t;
         sink.append(stream.str());
         return *this;
@@ -79,7 +79,7 @@ class TInfoSinkBase
         // does not have a fractional part, the default precision format does
         // not write the decimal portion which gets interpreted as integer by
         // the compiler.
-        TPersistStringStream stream;
+        TPersistStringStream stream = sh::InitializeStream<TPersistStringStream>();
         if (fractionalPart(f) == 0.0f)
         {
             stream.precision(1);

src/compiler/translator/OutputHLSL.cpp

@@ -39,7 +39,7 @@ constexpr const char kImage2DFunctionString[] = "// @@ IMAGE2D DECLARATION FUNCT
 
 TString ArrayHelperFunctionName(const char *prefix, const TType &type)
 {
-    TStringStream fnName;
+    TStringStream fnName = sh::InitializeStream<TStringStream>();
     fnName << prefix << "_";
     if (type.isArray())
     {
@@ -132,7 +132,7 @@ const char *kZeros       = "_ANGLE_ZEROS_";
 constexpr int kZeroCount = 256;
 std::string DefineZeroArray()
 {
-    std::stringstream ss;
+    std::stringstream ss = sh::InitializeStream<std::stringstream>();
     // For 'static', if the declaration does not include an initializer, the value is set to zero.
     // https://docs.microsoft.com/en-us/windows/desktop/direct3dhlsl/dx-graphics-hlsl-variable-syntax
     ss << "static uint " << kZeros << "[" << kZeroCount << "];\n";
@@ -141,9 +141,9 @@ std::string DefineZeroArray()
 
 std::string GetZeroInitializer(size_t size)
 {
-    std::stringstream ss;
-    size_t quotient = size / kZeroCount;
-    size_t reminder = size % kZeroCount;
+    std::stringstream ss = sh::InitializeStream<std::stringstream>();
+    size_t quotient      = size / kZeroCount;
+    size_t reminder      = size % kZeroCount;
 
     for (size_t i = 0; i < quotient; ++i)
     {
@@ -416,7 +416,7 @@ TString OutputHLSL::structInitializerString(int indent,
         init += indentString + "{\n";
         for (unsigned int arrayIndex = 0u; arrayIndex < type.getOutermostArraySize(); ++arrayIndex)
         {
-            TStringStream indexedString;
+            TStringStream indexedString = sh::InitializeStream<TStringStream>();
             indexedString << name << "[" << arrayIndex << "]";
             TType elementType = type;
             elementType.toArrayElementType();
@@ -891,8 +891,8 @@ void OutputHLSL::header(TInfoSinkBase &out,
 
         out << kImage2DFunctionString << "\n";
 
-        std::ostringstream systemValueDeclaration;
-        std::ostringstream glBuiltinInitialization;
+        std::ostringstream systemValueDeclaration  = sh::InitializeStream<std::ostringstream>();
+        std::ostringstream glBuiltinInitialization = sh::InitializeStream<std::ostringstream>();
 
         systemValueDeclaration << "\nstruct CS_INPUT\n{\n";
         glBuiltinInitialization << "\nvoid initGLBuiltins(CS_INPUT input)\n"
@@ -1900,7 +1900,7 @@ ImmutableString OutputHLSL::samplerNamePrefixFromStruct(TIntermTyped *node)
         {
             int index = nodeBinary->getRight()->getAsConstantUnion()->getIConst(0);
 
-            std::stringstream prefixSink;
+            std::stringstream prefixSink = sh::InitializeStream<std::stringstream>();
             prefixSink << samplerNamePrefixFromStruct(nodeBinary->getLeft()) << "_" << index;
             return ImmutableString(prefixSink.str());
         }
@@ -1910,7 +1910,7 @@ ImmutableString OutputHLSL::samplerNamePrefixFromStruct(TIntermTyped *node)
             int index           = nodeBinary->getRight()->getAsConstantUnion()->getIConst(0);
             const TField *field = s->fields()[index];
 
-            std::stringstream prefixSink;
+            std::stringstream prefixSink = sh::InitializeStream<std::stringstream>();
             prefixSink << samplerNamePrefixFromStruct(nodeBinary->getLeft()) << "_"
                        << field->name();
             return ImmutableString(prefixSink.str());

src/compiler/translator/ParseContext.cpp

@@ -611,7 +611,7 @@ bool TParseContext::checkCanBeLValue(const TSourceLoc &line, const char *op, TIn
         return true;
     }
 
-    std::stringstream reasonStream;
+    std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
     reasonStream << "l-value required";
     if (!message.empty())
     {
@@ -902,7 +902,7 @@ bool TParseContext::checkIsNotOpaqueType(const TSourceLoc &line,
     {
         if (ContainsSampler(pType.userDef))
         {
-            std::stringstream reasonStream;
+            std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
             reasonStream << reason << " (structure contains a sampler)";
             std::string reasonStr = reasonStream.str();
             error(line, reasonStr.c_str(), getBasicString(pType.type));
@@ -3066,7 +3066,7 @@ void TParseContext::parseGlobalLayoutQualifier(const TTypeQualifierBuilder &type
                 if (mComputeShaderLocalSize[i] < 1 ||
                     mComputeShaderLocalSize[i] > maxComputeWorkGroupSizeValue)
                 {
-                    std::stringstream reasonStream;
+                    std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
                     reasonStream << "invalid value: Value must be at least 1 and no greater than "
                                  << maxComputeWorkGroupSizeValue;
                     const std::string &reason = reasonStream.str();
@@ -3891,7 +3891,7 @@ void TParseContext::checkIsBelowStructNestingLimit(const TSourceLoc &line, const
     // one to the field's struct nesting.
     if (1 + field.type()->getDeepestStructNesting() > kWebGLMaxStructNesting)
     {
-        std::stringstream reasonStream;
+        std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
         if (field.type()->getStruct()->symbolType() == SymbolType::Empty)
         {
             // This may happen in case there are nested struct definitions. While they are also
@@ -4085,7 +4085,7 @@ int TParseContext::checkIndexLessThan(bool outOfRangeIndexIsError,
     ASSERT(index >= 0);
     if (index >= arraySize)
     {
-        std::stringstream reasonStream;
+        std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
         reasonStream << reason << " '" << index << "'";
         std::string token = reasonStream.str();
         outOfRangeError(outOfRangeIndexIsError, location, reason, "[]");
@@ -4387,7 +4387,7 @@ void TParseContext::parseLocalSize(const ImmutableString &qualifierType,
     checkLayoutQualifierSupported(qualifierTypeLine, qualifierType, 310);
     if (intValue < 1)
     {
-        std::stringstream reasonStream;
+        std::stringstream reasonStream = sh::InitializeStream<std::stringstream>();
         reasonStream << "out of range: " << getWorkGroupSizeString(index) << " must be positive";
         std::string reason = reasonStream.str();
         error(intValueLine, reason.c_str(), intValueString.c_str());
@@ -5672,7 +5672,7 @@ void TParseContext::checkTextureOffsetConst(TIntermAggregate *functionCall)
                 int offsetValue = values[i].getIConst();
                 if (offsetValue > maxOffsetValue || offsetValue < minOffsetValue)
                 {
-                    std::stringstream tokenStream;
+                    std::stringstream tokenStream = sh::InitializeStream<std::stringstream>();
                     tokenStream << offsetValue;
                     std::string token = tokenStream.str();
                     error(offset->getLine(), "Texture offset value out of valid range",

src/compiler/translator/Symbol.cpp

@@ -115,7 +115,7 @@ void TStructure::createSamplerSymbols(const char *namePrefix,
         const TType *fieldType = field->type();
         if (IsSampler(fieldType->getBasicType()) || fieldType->isStructureContainingSamplers())
         {
-            std::stringstream fieldName;
+            std::stringstream fieldName = sh::InitializeStream<std::stringstream>();
             fieldName << namePrefix << "_" << field->name();
             TString fieldApiName = apiNamePrefix + ".";
             fieldApiName += field->name().data();

src/compiler/translator/Types.cpp

@@ -756,7 +756,7 @@ void TType::createSamplerSymbols(const ImmutableString &namePrefix,
             elementType.toArrayElementType();
             for (unsigned int arrayIndex = 0u; arrayIndex < getOutermostArraySize(); ++arrayIndex)
             {
-                std::stringstream elementName;
+                std::stringstream elementName = sh::InitializeStream<std::stringstream>();
                 elementName << namePrefix << "_" << arrayIndex;
                 TStringStream elementApiName;
                 elementApiName << apiNamePrefix << "[" << arrayIndex << "]";

src/compiler/translator/ValidateOutputs.cpp

@@ -119,7 +119,7 @@ void ValidateOutputsTraverser::validate(TDiagnostics *diagnostics) const
                 const size_t offsetLocation = location + elementIndex;
                 if ((*validOutputsToUse)[offsetLocation])
                 {
-                    std::stringstream strstr;
+                    std::stringstream strstr = sh::InitializeStream<std::stringstream>();
                     strstr << "conflicting output locations with previously defined output '"
                            << (*validOutputsToUse)[offsetLocation]->getName() << "'";
                     error(*symbol, strstr.str().c_str(), diagnostics);

src/compiler/translator/ValidateVaryingLocations.cpp

@@ -83,7 +83,7 @@ void ValidateShaderInterface(TDiagnostics *diagnostics,
             const int offsetLocation = location + elementIndex;
             if (locationMap.find(offsetLocation) != locationMap.end())
             {
-                std::stringstream strstr;
+                std::stringstream strstr = sh::InitializeStream<std::stringstream>();
                 strstr << "'" << varying->getName()
                        << "' conflicting location with previously defined '"
                        << locationMap[offsetLocation]->getName() << "'";

src/compiler/translator/blocklayout.cpp

@@ -11,6 +11,7 @@
 
 #include "common/mathutil.h"
 #include "common/utilities.h"
+#include "compiler/translator/Common.h"
 
 namespace sh
 {
@@ -132,7 +133,7 @@ void TraverseArrayOfArraysVariable(const ShaderVariable &variable,
 
 std::string CollapseNameStack(const std::vector<std::string> &nameStack)
 {
-    std::stringstream strstr;
+    std::stringstream strstr = sh::InitializeStream<std::stringstream>();
     for (const std::string &part : nameStack)
     {
         strstr << part;
@@ -415,7 +416,7 @@ void VariableNameVisitor::exitArray(const ShaderVariable &arrayVar)
 void VariableNameVisitor::enterArrayElement(const ShaderVariable &arrayVar,
                                             unsigned int arrayElement)
 {
-    std::stringstream strstr;
+    std::stringstream strstr = sh::InitializeStream<std::stringstream>();
     strstr << "[" << arrayElement << "]";
     std::string elementString = strstr.str();
     mNameStack.push_back(elementString);

src/compiler/translator/tree_ops/EmulatePrecision.cpp

@@ -174,7 +174,7 @@ std::string RoundingHelperWriterGLSL::getTypeString(const char *glslType)
 
 std::string RoundingHelperWriterESSL::getTypeString(const char *glslType)
 {
-    std::stringstream typeStrStr;
+    std::stringstream typeStrStr = sh::InitializeStream<std::stringstream>();
     typeStrStr << "highp " << glslType;
     return typeStrStr.str();
 }
@@ -257,7 +257,7 @@ void RoundingHelperWriterGLSL::writeFloatRoundingHelpers(TInfoSinkBase &sink)
 void RoundingHelperWriterGLSL::writeVectorRoundingHelpers(TInfoSinkBase &sink,
                                                           const unsigned int size)
 {
-    std::stringstream vecTypeStrStr;
+    std::stringstream vecTypeStrStr = sh::InitializeStream<std::stringstream>();
     vecTypeStrStr << "vec" << size;
     std::string vecType = getTypeString(vecTypeStrStr.str().c_str());
 
@@ -287,7 +287,7 @@ void RoundingHelperWriterGLSL::writeMatrixRoundingHelper(TInfoSinkBase &sink,
                                                          const unsigned int rows,
                                                          const char *functionName)
 {
-    std::stringstream matTypeStrStr;
+    std::stringstream matTypeStrStr = sh::InitializeStream<std::stringstream>();
     matTypeStrStr << "mat" << columns;
     if (rows != columns)
     {
@@ -379,7 +379,7 @@ void RoundingHelperWriterHLSL::writeFloatRoundingHelpers(TInfoSinkBase &sink)
 void RoundingHelperWriterHLSL::writeVectorRoundingHelpers(TInfoSinkBase &sink,
                                                           const unsigned int size)
 {
-    std::stringstream vecTypeStrStr;
+    std::stringstream vecTypeStrStr = sh::InitializeStream<std::stringstream>();
     vecTypeStrStr << "float" << size;
     std::string vecType = vecTypeStrStr.str();
 
@@ -409,7 +409,7 @@ void RoundingHelperWriterHLSL::writeMatrixRoundingHelper(TInfoSinkBase &sink,
                                                          const unsigned int rows,
                                                          const char *functionName)
 {
-    std::stringstream matTypeStrStr;
+    std::stringstream matTypeStrStr = sh::InitializeStream<std::stringstream>();
     matTypeStrStr << "float" << columns << "x" << rows;
     std::string matType = matTypeStrStr.str();
 
@@ -744,7 +744,7 @@ TIntermAggregate *EmulatePrecision::createCompoundAssignmentFunctionCallNode(TIn
                                                                              TIntermTyped *right,
                                                                              const char *opNameStr)
 {
-    std::stringstream strstr;
+    std::stringstream strstr = sh::InitializeStream<std::stringstream>();
     if (left->getPrecision() == EbpMedium)
         strstr << "angle_compound_" << opNameStr << "_frm";
     else

src/tests/compiler_tests/ShCompile_test.cpp

@@ -117,39 +117,70 @@ TEST_F(ShCompileTest, TokensSplitInShaderStrings)
 }
 
 // Parsing floats in shaders can run afoul of locale settings.
-// In de_DE, `strtof("1.9")` will yield `1.0f`. (It's expecting "1,9")
+// Eg. in de_DE, `strtof("1.9")` will yield `1.0f`. (It's expecting "1,9")
 TEST_F(ShCompileTest, DecimalSepLocale)
 {
-    const auto defaultLocale = setlocale(LC_NUMERIC, nullptr);
-
-    const auto fnSetLocale = [](const char *const name) {
-        return bool(setlocale(LC_NUMERIC, name));
+    // Locale names are platform dependent, add platform-specific names of locales to be tested here
+    const std::string availableLocales[] = {
+        "de_DE", "de-DE", "de_DE.UTF-8", "de_DE.ISO8859-1", "de_DE.ISO8859-15", "de_DE@euro",
+        "de_DE.88591", "de_DE.88591.en", "de_DE.iso88591", "de_DE.ISO-8859-1", "de_DE.ISO_8859-1",
+        "de_DE.iso885915", "de_DE.ISO-8859-15", "de_DE.ISO_8859-15", "de_DE.8859-15",
+        "de_DE.8859-15@euro", "de_DE.ISO-8859-15@euro", "de_DE.UTF-8@euro", "de_DE.utf8",
+        "German_germany", "German_Germany", "German_Germany.1252", "German_Germany.UTF-8", "German",
+        // One ubuntu tester doesn't have a german locale, but da_DK.utf8 has similar float
+        // representation
+        "da_DK.utf8"};
+
+    const auto localeExists = [](const std::string name) {
+        return bool(setlocale(LC_ALL, name.c_str()));
     };
 
-    const bool setLocaleToDe =
-        fnSetLocale("de_DE") || fnSetLocale("de-DE");  // Windows doesn't like de_DE.
-
-// These configs don't support de_DE: android_angle_vk[32,64]_rel_ng, linux_angle_rel_ng
-// Just allow those platforms to quietly fail, but require other platforms to succeed.
-#if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_LINUX)
-    if (!setLocaleToDe)
+    const char kSource[] = R"(
+    void main()
     {
-        return;
-    }
-#endif
-    ASSERT_TRUE(setLocaleToDe);
+        gl_FragColor = vec4(1.9);
+    })";
+    const char *parts[]  = {kSource};
 
-    const char kSource[] = R"(
-        void main()
+    int testedLocales = 0;
+
+    for (const std::string &locale : availableLocales)
+    {
+        // If the locale doesn't exist on the testing platform, the locale constructor will fail,
+        // throwing an exception
+        // We use setlocale() (through localeExists) to test whether a locale
+        // exists before calling the locale constructor
+        if (localeExists(locale))
         {
-            gl_FragColor = vec4(1.9);
-        })";
-    const char *parts[]  = {kSource};
-    testCompile(parts, 1, true);
+            std::locale localizedLoc(locale);
 
-    const auto &translated = sh::GetObjectCode(mCompiler);
-    // printf("%s\n", translated.data());
-    EXPECT_NE(translated.find("1.9"), std::string::npos);
+            // std::locale::global() must be used instead of setlocale() to affect new streams'
+            // default locale
+            std::locale::global(std::locale::classic());
+            sh::Compile(mCompiler, parts, 1, SH_OBJECT_CODE);
+            std::string referenceOut = sh::GetObjectCode(mCompiler);
+            EXPECT_NE(referenceOut.find("1.9"), std::string::npos)
+                << "float formatted incorrectly with classic locale";
 
-    fnSetLocale(defaultLocale);
-}
+            sh::ClearResults(mCompiler);
+
+            std::locale::global(localizedLoc);
+            sh::Compile(mCompiler, parts, 1, SH_OBJECT_CODE);
+            std::string localizedOut = sh::GetObjectCode(mCompiler);
+            EXPECT_NE(localizedOut.find("1.9"), std::string::npos)
+                << "float formatted incorrectly with locale (" << localizedLoc.name() << ") set";
+
+            ASSERT_EQ(referenceOut, localizedOut)
+                << "different output with locale (" << localizedLoc.name() << ") set";
+
+            testedLocales++;
+        }
+    }
+
+// Ignore for Android which doesn't use these locales
+#if !defined(ANGLE_PLATFORM_ANDROID)
+    // This test must run on at least one different locale to pass
+    EXPECT_NE(testedLocales, 0)
+        << "Missing locales to test on this platform. Add them to availableLocales[]";
+#endif
+}
\ No newline at end of file