Moved the definition of max-token length from preprocessor to compiler.

src/compiler/glslang.l

@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
     // Initialize preprocessor.
     if (!context->preprocessor.init(count, string, length))
         return 1;
+    context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
 
     // Define extension macros.
     const TExtensionBehavior& extBehavior = context->extensionBehavior();

src/compiler/glslang.y

@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h).
 %code requires {
 #define YYLTYPE TSourceLoc
 #define YYLTYPE_IS_DECLARED 1
+#define SH_MAX_TOKEN_LENGTH 256  // WebGL spec.
 }
 
 %union {

src/compiler/glslang_lex.cpp

@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
     // Initialize preprocessor.
     if (!context->preprocessor.init(count, string, length))
         return 1;
+    context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
 
     // Define extension macros.
     const TExtensionBehavior& extBehavior = context->extensionBehavior();

src/compiler/glslang_tab.cpp

@@ -126,6 +126,7 @@ extern int yydebug;
 
 #define YYLTYPE TSourceLoc
 #define YYLTYPE_IS_DECLARED 1
+#define SH_MAX_TOKEN_LENGTH 256  // WebGL spec.
 
 
 
@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] =
 /* YYRLINE[YYN] -- source line where rule number YYN was defined.  */
 static const yytype_uint16 yyrline[] =
 {
-       0,   179,   179,   180,   183,   226,   229,   242,   247,   252,
-     258,   261,   264,   267,   362,   372,   385,   393,   493,   496,
-     504,   507,   513,   517,   524,   530,   539,   547,   602,   612,
-     615,   625,   635,   656,   657,   658,   663,   664,   672,   683,
-     684,   692,   703,   707,   708,   718,   728,   738,   751,   752,
-     762,   775,   779,   783,   787,   788,   801,   802,   815,   816,
-     829,   830,   847,   848,   861,   862,   863,   864,   865,   869,
-     872,   883,   891,   918,   923,   937,   992,   995,  1002,  1010,
-    1031,  1052,  1062,  1090,  1095,  1105,  1110,  1120,  1123,  1126,
-    1129,  1135,  1142,  1145,  1167,  1185,  1209,  1232,  1236,  1254,
-    1262,  1294,  1314,  1335,  1344,  1367,  1370,  1376,  1384,  1392,
-    1400,  1410,  1417,  1420,  1423,  1429,  1432,  1447,  1451,  1455,
-    1459,  1463,  1468,  1473,  1478,  1483,  1488,  1493,  1498,  1503,
-    1508,  1513,  1518,  1523,  1527,  1531,  1539,  1547,  1551,  1564,
-    1564,  1578,  1578,  1587,  1590,  1606,  1639,  1643,  1649,  1656,
-    1671,  1675,  1679,  1680,  1686,  1687,  1688,  1689,  1690,  1694,
-    1695,  1695,  1695,  1705,  1706,  1710,  1710,  1711,  1711,  1716,
-    1719,  1729,  1732,  1738,  1739,  1743,  1751,  1755,  1765,  1770,
-    1787,  1787,  1792,  1792,  1799,  1799,  1807,  1810,  1816,  1819,
-    1825,  1829,  1836,  1843,  1850,  1857,  1868,  1877,  1881,  1888,
-    1891,  1897,  1897
+       0,   180,   180,   181,   184,   227,   230,   243,   248,   253,
+     259,   262,   265,   268,   363,   373,   386,   394,   494,   497,
+     505,   508,   514,   518,   525,   531,   540,   548,   603,   613,
+     616,   626,   636,   657,   658,   659,   664,   665,   673,   684,
+     685,   693,   704,   708,   709,   719,   729,   739,   752,   753,
+     763,   776,   780,   784,   788,   789,   802,   803,   816,   817,
+     830,   831,   848,   849,   862,   863,   864,   865,   866,   870,
+     873,   884,   892,   919,   924,   938,   993,   996,  1003,  1011,
+    1032,  1053,  1063,  1091,  1096,  1106,  1111,  1121,  1124,  1127,
+    1130,  1136,  1143,  1146,  1168,  1186,  1210,  1233,  1237,  1255,
+    1263,  1295,  1315,  1336,  1345,  1368,  1371,  1377,  1385,  1393,
+    1401,  1411,  1418,  1421,  1424,  1430,  1433,  1448,  1452,  1456,
+    1460,  1464,  1469,  1474,  1479,  1484,  1489,  1494,  1499,  1504,
+    1509,  1514,  1519,  1524,  1528,  1532,  1540,  1548,  1552,  1565,
+    1565,  1579,  1579,  1588,  1591,  1607,  1640,  1644,  1650,  1657,
+    1672,  1676,  1680,  1681,  1687,  1688,  1689,  1690,  1691,  1695,
+    1696,  1696,  1696,  1706,  1707,  1711,  1711,  1712,  1712,  1717,
+    1720,  1730,  1733,  1739,  1740,  1744,  1752,  1756,  1766,  1771,
+    1788,  1788,  1793,  1793,  1800,  1800,  1808,  1811,  1817,  1820,
+    1826,  1830,  1837,  1844,  1851,  1858,  1869,  1878,  1882,  1889,
+    1892,  1898,  1898
 };
 #endif
 

src/compiler/glslang_tab.h

@@ -44,6 +44,7 @@ extern int yydebug;
 
 #define YYLTYPE TSourceLoc
 #define YYLTYPE_IS_DECLARED 1
+#define SH_MAX_TOKEN_LENGTH 256  // WebGL spec.
 
 
 

src/compiler/preprocessor/Preprocessor.cpp

@@ -81,6 +81,11 @@ void Preprocessor::predefineMacro(const char* name, int value)
     mImpl->macroSet[name] = macro;
 }
 
+void Preprocessor::setMaxTokenLength(size_t maxLength)
+{
+    mImpl->tokenizer.setMaxTokenLength(maxLength);
+}
+
 void Preprocessor::lex(Token* token)
 {
     bool validToken = false;

src/compiler/preprocessor/Preprocessor.h

@@ -37,6 +37,12 @@ class Preprocessor
     bool init(size_t count, const char* const string[], const int length[]);
     // Adds a pre-defined macro.
     void predefineMacro(const char* name, int value);
+    // Sets maximum allowed token length.
+    // If token length exceeds this limit,
+    // the token text will be truncated to the given maximum length, and
+    // TOKEN_TOO_LONG diagnostic will be generated.
+    // The maximum length defaults to 256.
+    void setMaxTokenLength(size_t maxLength);
 
     void lex(Token* token);
 

src/compiler/preprocessor/Tokenizer.cpp

@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
 
 namespace pp {
 
-// TODO(alokp): Maximum token length should ideally be specified by
-// the preprocessor client, i.e., the compiler.
-const size_t Tokenizer::kMaxTokenLength = 256;
-
-Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
+Tokenizer::Tokenizer(Diagnostics* diagnostics)
+    : mHandle(0),
+      mMaxTokenLength(256)
 {
     mContext.diagnostics = diagnostics;
 }
@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = pplex(&token->text,&token->location,mHandle);
-    if (token->text.size() > kMaxTokenLength)
+    if (token->text.size() > mMaxTokenLength)
     {
         mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
                                      token->location, token->text);
-        token->text.erase(kMaxTokenLength);
+        token->text.erase(mMaxTokenLength);
     }
 
     token->flags = 0;

src/compiler/preprocessor/Tokenizer.h

@@ -32,13 +32,13 @@ class Tokenizer : public Lexer
         bool leadingSpace;
         bool lineStart;
     };
-    static const std::size_t kMaxTokenLength;
 
     Tokenizer(Diagnostics* diagnostics);
     ~Tokenizer();
 
     bool init(size_t count, const char* const string[], const int length[]);
 
+    void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; }
     void setFileNumber(int file);
     void setLineNumber(int line);
 
@@ -51,6 +51,7 @@ class Tokenizer : public Lexer
 
     void* mHandle;  // Scanner handle.
     Context mContext;  // Scanner extra.
+    size_t mMaxTokenLength;
 };
 
 }  // namespace pp

src/compiler/preprocessor/Tokenizer.l

@@ -267,11 +267,9 @@ FRACTIONAL_CONSTANT  ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".")
 
 namespace pp {
 
-// TODO(alokp): Maximum token length should ideally be specified by
-// the preprocessor client, i.e., the compiler.
-const size_t Tokenizer::kMaxTokenLength = 256;
-
-Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
+Tokenizer::Tokenizer(Diagnostics* diagnostics)
+    : mHandle(0),
+      mMaxTokenLength(256)
 {
     mContext.diagnostics = diagnostics;
 }
@@ -304,11 +302,11 @@ void Tokenizer::setLineNumber(int line)
 void Tokenizer::lex(Token* token)
 {
     token->type = yylex(&token->text, &token->location, mHandle);
-    if (token->text.size() > kMaxTokenLength)
+    if (token->text.size() > mMaxTokenLength)
     {
         mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
                                      token->location, token->text);
-        token->text.erase(kMaxTokenLength);
+        token->text.erase(mMaxTokenLength);
     }
 
     token->flags = 0;