WebGL spec specifies maximum length of all types of tokens - not just identifier…

src/compiler/preprocessor/new/Diagnostics.cpp

@@ -52,8 +52,8 @@ std::string Diagnostics::message(ID id)
           return "integer overflow";
       case FLOAT_OVERFLOW:
           return "float overflow";
-      case IDENTIFIER_OVERFLOW:
-          return "identifier buffer overflow";
+      case TOKEN_TOO_LONG:
+          return "token too long";
       case INVALID_EXPRESSION:
           return "invalid expression";
       case DIVISION_BY_ZERO:

src/compiler/preprocessor/new/Diagnostics.h

@@ -33,7 +33,7 @@ class Diagnostics
         INVALID_NUMBER,
         INTEGER_OVERFLOW,
         FLOAT_OVERFLOW,
-        IDENTIFIER_OVERFLOW,
+        TOKEN_TOO_LONG,
         INVALID_EXPRESSION,
         DIVISION_BY_ZERO,
         EOF_IN_COMMENT,

src/compiler/preprocessor/new/DirectiveParser.cpp

@@ -347,12 +347,6 @@ void DirectiveParser::parseDefine(Token* token)
                              token->location, token->text);
         return;
     }
-    if (token->text.size() > Token::kMaxIdentifierLength)
-    {
-        mDiagnostics->report(Diagnostics::IDENTIFIER_OVERFLOW,
-                             token->location, token->text);
-        return;
-    }
 
     Macro macro;
     macro.type = Macro::kTypeObj;

src/compiler/preprocessor/new/Preprocessor.cpp

@@ -123,19 +123,6 @@ void Preprocessor::lex(Token* token)
             validToken = true;
             break;
           }
-          case Token::IDENTIFIER:
-          {
-            if (token->text.size() > Token::kMaxIdentifierLength)
-            {
-                // Do not mark the token as invalid.
-                // Just emit the diagnostic and clamp string length.
-                mImpl->diagnostics->report(Diagnostics::IDENTIFIER_OVERFLOW,
-                                           token->location, token->text);
-                token->text.erase(Token::kMaxIdentifierLength);
-            }
-            validToken = true;
-            break;
-          }
           case Token::PP_NUMBER:
             mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
                                        token->location, token->text);

src/compiler/preprocessor/new/Token.cpp

@@ -31,10 +31,6 @@ static bool atoi_t(const std::string& str, IntType* value)
 namespace pp
 {
 
-// TODO(alokp): This should be specified by preprocessor client,
-// i.e., the compiler.
-const size_t Token::kMaxIdentifierLength = 256;
-
 void Token::reset()
 {
     type = 0;

src/compiler/preprocessor/new/Token.h

@@ -61,7 +61,6 @@ struct Token
         HAS_LEADING_SPACE  = 1 << 1,
         EXPANSION_DISABLED = 1 << 2
     };
-    static const size_t kMaxIdentifierLength;
 
     Token() : type(0), flags(0) { }
 

src/compiler/preprocessor/new/Tokenizer.cpp

@@ -2280,6 +2280,10 @@ void ppfree (void * ptr , yyscan_t yyscanner)
 
 namespace pp {
 
+// TODO(alokp): Maximum token length should ideally be specified by
+// the preprocessor client, i.e., the compiler.
+const size_t Tokenizer::kMaxTokenLength = 256;
+
 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
     mContext.diagnostics = diagnostics;
@@ -2314,6 +2318,13 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = pplex(&token->text,&token->location,mHandle);
+    if (token->text.size() > kMaxTokenLength)
+    {
+        mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
+                                     token->location, token->text);
+        token->text.erase(kMaxTokenLength);
+    }
+
     token->flags = 0;
 
     token->setAtStartOfLine(mContext.lineStart);

src/compiler/preprocessor/new/Tokenizer.h

@@ -32,6 +32,7 @@ class Tokenizer : public Lexer
         bool leadingSpace;
         bool lineStart;
     };
+    static const size_t kMaxTokenLength;
 
     Tokenizer(Diagnostics* diagnostics);
     ~Tokenizer();

src/compiler/preprocessor/new/Tokenizer.l

@@ -264,6 +264,10 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 namespace pp {
 
+// TODO(alokp): Maximum token length should ideally be specified by
+// the preprocessor client, i.e., the compiler.
+const size_t Tokenizer::kMaxTokenLength = 256;
+
 Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
 {
     mContext.diagnostics = diagnostics;
@@ -298,6 +302,13 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->text, &token->location, mHandle);
+    if (token->text.size() > kMaxTokenLength)
+    {
+        mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
+                                     token->location, token->text);
+        token->text.erase(kMaxTokenLength);
+    }
+
     token->flags = 0;
 
     token->setAtStartOfLine(mContext.lineStart);