Proper support for token max size in WebGL+ES3.

src/compiler/preprocessor/Tokenizer.cpp

 #line 16 "./Tokenizer.l"
 //
-// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -527,7 +527,7 @@ static yyconst flex_int16_t yy_chk[224] =
 #define YY_RESTORE_YY_MORE_OFFSET
 /*
 //
-// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -2317,10 +2317,6 @@ void ppfree (void * ptr , yyscan_t yyscanner)
 
 namespace pp {
 
-// TODO(alokp): Maximum token length should ideally be specified by
-// the preprocessor client, i.e., the compiler.
-const size_t Tokenizer::kMaxTokenLength = MAX_SYMBOL_NAME_LEN;
-
 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
     mContext.diagnostics = diagnostics;
@@ -2354,11 +2350,11 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = pplex(&token->text,&token->location,mHandle);
-    if (token->text.size() > kMaxTokenLength)
+    if (token->text.size() > GetGlobalMaxTokenSize())
     {
         mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                      token->location, token->text);
-        token->text.erase(kMaxTokenLength);
+        token->text.erase(GetGlobalMaxTokenSize());
     }
 
     token->flags = 0;

src/compiler/preprocessor/Tokenizer.h

 //
-// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2012-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -32,7 +32,6 @@ class Tokenizer : public Lexer
         bool leadingSpace;
         bool lineStart;
     };
-    static const std::size_t kMaxTokenLength;
 
     Tokenizer(Diagnostics* diagnostics);
     ~Tokenizer();

src/compiler/preprocessor/Tokenizer.l

 /*
 //
-// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -14,7 +14,7 @@ IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh.
 
 %top{
 //
-// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -268,10 +268,6 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 namespace pp {
 
-// TODO(alokp): Maximum token length should ideally be specified by
-// the preprocessor client, i.e., the compiler.
-const size_t Tokenizer::kMaxTokenLength = MAX_SYMBOL_NAME_LEN;
-
 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
     mContext.diagnostics = diagnostics;
@@ -305,11 +301,11 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->text, &token->location, mHandle);
-    if (token->text.size() > kMaxTokenLength)
+    if (token->text.size() > GetGlobalMaxTokenSize())
     {
         mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG,
                                      token->location, token->text);
-        token->text.erase(kMaxTokenLength);
+        token->text.erase(GetGlobalMaxTokenSize());
     }
 
     token->flags = 0;

src/compiler/preprocessor/length_limits.h

 //
-// Copyright (c) 2011-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -14,7 +14,6 @@
 // These constants are factored out from the rest of the headers to
 // make it easier to reference them from the compiler sources.
 
-// These lengths do not include the NULL terminator.
-#define MAX_SYMBOL_NAME_LEN 1024
+size_t GetGlobalMaxTokenSize();
 
 #endif // !(defined(__LENGTH_LIMITS_H)

src/compiler/translator/Compiler.cpp

 //
-// Copyright (c) 2002-2013 The ANGLE Project Authors. All rights reserved.
+// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
@@ -29,6 +29,21 @@ bool isWebGLBasedSpec(ShShaderSpec spec)
      return spec == SH_WEBGL_SPEC || spec == SH_CSS_SHADERS_SPEC;
 }
 
+size_t GetGlobalMaxTokenSize()
+{
+    TParseContext *parseContext = GetGlobalParseContext();
+    // WebGL defines a max token legnth of 256, while ES2 leaves max token
+    // size undefined. ES3 defines a max size of 1024 characters.
+    if (isWebGLBasedSpec(parseContext->shaderSpec))
+    {
+        return 256;
+    }
+    else
+    {
+        return 1024;
+    }
+}
+
 namespace {
 class TScopedPoolAllocator
 {

src/compiler/translator/Diagnostics.h

@@ -22,8 +22,6 @@ class TDiagnostics : public pp::Diagnostics
     int numErrors() const { return mNumErrors; }
     int numWarnings() const { return mNumWarnings; }
 
-    void setShaderVersion(int version);
-
     void writeInfo(Severity severity,
                    const pp::SourceLocation& loc,
                    const std::string& reason,

src/compiler/translator/ShaderLang.cpp

@@ -180,27 +180,27 @@ void ShGetInfo(const ShHandle handle, ShShaderInfo pname, size_t* params)
         *params = compiler->getUniforms().size();
         break;
     case SH_ACTIVE_UNIFORM_MAX_LENGTH:
-        *params = 1 +  MAX_SYMBOL_NAME_LEN;
+        *params = 1 + GetGlobalMaxTokenSize();
         break;
     case SH_ACTIVE_ATTRIBUTES:
         *params = compiler->getAttribs().size();
         break;
     case SH_ACTIVE_ATTRIBUTE_MAX_LENGTH:
-        *params = 1 + MAX_SYMBOL_NAME_LEN;
+        *params = 1 + GetGlobalMaxTokenSize();
         break;
     case SH_VARYINGS:
         *params = compiler->getVaryings().size();
         break;
     case SH_VARYING_MAX_LENGTH:
-        *params = 1 + MAX_SYMBOL_NAME_LEN;
+        *params = 1 + GetGlobalMaxTokenSize();
         break;
     case SH_MAPPED_NAME_MAX_LENGTH:
         // Use longer length than MAX_SHORTENED_IDENTIFIER_SIZE to
         // handle array and struct dereferences.
-        *params = 1 + MAX_SYMBOL_NAME_LEN;
+        *params = 1 + GetGlobalMaxTokenSize();
         break;
     case SH_NAME_MAX_LENGTH:
-        *params = 1 + MAX_SYMBOL_NAME_LEN;
+        *params = 1 + GetGlobalMaxTokenSize();
         break;
     case SH_HASHED_NAME_MAX_LENGTH:
         if (compiler->getHashFunction() == NULL) {
@@ -307,14 +307,14 @@ void ShGetVariableInfo(const ShHandle handle,
     // This size must match that queried by
     // SH_ACTIVE_UNIFORM_MAX_LENGTH, SH_ACTIVE_ATTRIBUTE_MAX_LENGTH, SH_VARYING_MAX_LENGTH
     // in ShGetInfo, below.
-    size_t variableLength = 1 + MAX_SYMBOL_NAME_LEN;
+    size_t variableLength = 1 + GetGlobalMaxTokenSize();
     ASSERT(checkVariableMaxLengths(handle, variableLength));
     strncpy(name, varInfo.name.c_str(), variableLength);
     name[variableLength - 1] = 0;
     if (mappedName) {
         // This size must match that queried by
         // SH_MAPPED_NAME_MAX_LENGTH in ShGetInfo, below.
-        size_t maxMappedNameLength = 1 + MAX_SYMBOL_NAME_LEN;
+        size_t maxMappedNameLength = 1 + GetGlobalMaxTokenSize();
         ASSERT(checkMappedNameMaxLength(handle, maxMappedNameLength));
         strncpy(mappedName, varInfo.mappedName.c_str(), maxMappedNameLength);
         mappedName[maxMappedNameLength - 1] = 0;