Skip to content

A
angle
Commit ba2cfe7c by Alok Priyadarshi
Options

Moved the definition of max-token length from preprocessor to compiler.

R=kbr@chromium.org Review URL: https://codereview.appspot.com/12957043
parent 76985f3c
Showing with 294 additions and 279 deletions
src/compiler/glslang.l
...@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[], ...@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor. // Initialize preprocessor.
if (!context->preprocessor.init(count, string, length)) if (!context->preprocessor.init(count, string, length))
return 1; return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros. // Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior(); const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang.y
...@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h). ...@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h).
%code requires { %code requires {
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
} }
%union { %union {
......
src/compiler/glslang_lex.cpp
...@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[], ...@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor. // Initialize preprocessor.
if (!context->preprocessor.init(count, string, length)) if (!context->preprocessor.init(count, string, length))
return 1; return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros. // Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior(); const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang_tab.cpp
...@@ -126,6 +126,7 @@ extern int yydebug; ...@@ -126,6 +126,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
...@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] = ...@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */ /* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] = static const yytype_uint16 yyrline[] =
{ {
0, 179, 179, 180, 183, 226, 229, 242, 247, 252, 0, 180, 180, 181, 184, 227, 230, 243, 248, 253,
258, 261, 264, 267, 362, 372, 385, 393, 493, 496, 259, 262, 265, 268, 363, 373, 386, 394, 494, 497,
504, 507, 513, 517, 524, 530, 539, 547, 602, 612, 505, 508, 514, 518, 525, 531, 540, 548, 603, 613,
615, 625, 635, 656, 657, 658, 663, 664, 672, 683, 616, 626, 636, 657, 658, 659, 664, 665, 673, 684,
684, 692, 703, 707, 708, 718, 728, 738, 751, 752, 685, 693, 704, 708, 709, 719, 729, 739, 752, 753,
762, 775, 779, 783, 787, 788, 801, 802, 815, 816, 763, 776, 780, 784, 788, 789, 802, 803, 816, 817,
829, 830, 847, 848, 861, 862, 863, 864, 865, 869, 830, 831, 848, 849, 862, 863, 864, 865, 866, 870,
872, 883, 891, 918, 923, 937, 992, 995, 1002, 1010, 873, 884, 892, 919, 924, 938, 993, 996, 1003, 1011,
1031, 1052, 1062, 1090, 1095, 1105, 1110, 1120, 1123, 1126, 1032, 1053, 1063, 1091, 1096, 1106, 1111, 1121, 1124, 1127,
1129, 1135, 1142, 1145, 1167, 1185, 1209, 1232, 1236, 1254, 1130, 1136, 1143, 1146, 1168, 1186, 1210, 1233, 1237, 1255,
1262, 1294, 1314, 1335, 1344, 1367, 1370, 1376, 1384, 1392, 1263, 1295, 1315, 1336, 1345, 1368, 1371, 1377, 1385, 1393,
1400, 1410, 1417, 1420, 1423, 1429, 1432, 1447, 1451, 1455, 1401, 1411, 1418, 1421, 1424, 1430, 1433, 1448, 1452, 1456,
1459, 1463, 1468, 1473, 1478, 1483, 1488, 1493, 1498, 1503, 1460, 1464, 1469, 1474, 1479, 1484, 1489, 1494, 1499, 1504,
1508, 1513, 1518, 1523, 1527, 1531, 1539, 1547, 1551, 1564, 1509, 1514, 1519, 1524, 1528, 1532, 1540, 1548, 1552, 1565,
1564, 1578, 1578, 1587, 1590, 1606, 1639, 1643, 1649, 1656, 1565, 1579, 1579, 1588, 1591, 1607, 1640, 1644, 1650, 1657,
1671, 1675, 1679, 1680, 1686, 1687, 1688, 1689, 1690, 1694, 1672, 1676, 1680, 1681, 1687, 1688, 1689, 1690, 1691, 1695,
1695, 1695, 1695, 1705, 1706, 1710, 1710, 1711, 1711, 1716, 1696, 1696, 1696, 1706, 1707, 1711, 1711, 1712, 1712, 1717,
1719, 1729, 1732, 1738, 1739, 1743, 1751, 1755, 1765, 1770, 1720, 1730, 1733, 1739, 1740, 1744, 1752, 1756, 1766, 1771,
1787, 1787, 1792, 1792, 1799, 1799, 1807, 1810, 1816, 1819, 1788, 1788, 1793, 1793, 1800, 1800, 1808, 1811, 1817, 1820,
1825, 1829, 1836, 1843, 1850, 1857, 1868, 1877, 1881, 1888, 1826, 1830, 1837, 1844, 1851, 1858, 1869, 1878, 1882, 1889,
1891, 1897, 1897 1892, 1898, 1898
}; };
#endif #endif
......
src/compiler/glslang_tab.h
...@@ -44,6 +44,7 @@ extern int yydebug; ...@@ -44,6 +44,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc #define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1 #define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......
src/compiler/preprocessor/Preprocessor.cpp
// //
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#include "Preprocessor.h" #include "Preprocessor.h"
#include <cassert> #include <cassert>
#include <sstream> #include <sstream>
#include "DiagnosticsBase.h" #include "DiagnosticsBase.h"
#include "DirectiveParser.h" #include "DirectiveParser.h"
#include "Macro.h" #include "Macro.h"
#include "MacroExpander.h" #include "MacroExpander.h"
#include "Token.h" #include "Token.h"
#include "Tokenizer.h" #include "Tokenizer.h"
namespace pp namespace pp
{ {
struct PreprocessorImpl struct PreprocessorImpl
{ {
Diagnostics* diagnostics; Diagnostics* diagnostics;
MacroSet macroSet; MacroSet macroSet;
Tokenizer tokenizer; Tokenizer tokenizer;
DirectiveParser directiveParser; DirectiveParser directiveParser;
MacroExpander macroExpander; MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag, PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) : DirectiveHandler* directiveHandler) :
diagnostics(diag), diagnostics(diag),
tokenizer(diag), tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler), directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag) macroExpander(&directiveParser, &macroSet, diag)
{ {
} }
}; };
Preprocessor::Preprocessor(Diagnostics* diagnostics, Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler) DirectiveHandler* directiveHandler)
{ {
mImpl = new PreprocessorImpl(diagnostics, directiveHandler); mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
} }
Preprocessor::~Preprocessor() Preprocessor::~Preprocessor()
{ {
delete mImpl; delete mImpl;
} }
bool Preprocessor::init(size_t count, bool Preprocessor::init(size_t count,
const char* const string[], const char* const string[],
const int length[]) const int length[])
{ {
static const int kGLSLVersion = 100; static const int kGLSLVersion = 100;
// Add standard pre-defined macros. // Add standard pre-defined macros.
predefineMacro("__LINE__", 0); predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0); predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion); predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1); predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length); return mImpl->tokenizer.init(count, string, length);
} }
void Preprocessor::predefineMacro(const char* name, int value) void Preprocessor::predefineMacro(const char* name, int value)
{ {
std::ostringstream stream; std::ostringstream stream;
stream << value; stream << value;
Token token; Token token;
token.type = Token::CONST_INT; token.type = Token::CONST_INT;
token.text = stream.str(); token.text = stream.str();
Macro macro; Macro macro;
macro.predefined = true; macro.predefined = true;
macro.type = Macro::kTypeObj; macro.type = Macro::kTypeObj;
macro.name = name; macro.name = name;
macro.replacements.push_back(token); macro.replacements.push_back(token);
mImpl->macroSet[name] = macro; mImpl->macroSet[name] = macro;
} }
void Preprocessor::lex(Token* token) void Preprocessor::setMaxTokenLength(size_t maxLength)
{ {
bool validToken = false; mImpl->tokenizer.setMaxTokenLength(maxLength);
while (!validToken) }
{
mImpl->macroExpander.lex(token); void Preprocessor::lex(Token* token)
switch (token->type) {
{ bool validToken = false;
// We should not be returning internal preprocessing tokens. while (!validToken)
// Convert preprocessing tokens to compiler tokens or report {
// diagnostics. mImpl->macroExpander.lex(token);
case Token::PP_HASH: switch (token->type)
assert(false); {
break; // We should not be returning internal preprocessing tokens.
case Token::CONST_INT: // Convert preprocessing tokens to compiler tokens or report
{ // diagnostics.
int val = 0; case Token::PP_HASH:
if (!token->iValue(&val)) assert(false);
{ break;
// Do not mark the token as invalid. case Token::CONST_INT:
// Just emit the diagnostic and reset value to 0. {
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW, int val = 0;
token->location, token->text); if (!token->iValue(&val))
token->text.assign("0"); {
} // Do not mark the token as invalid.
validToken = true; // Just emit the diagnostic and reset value to 0.
break; mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
} token->location, token->text);
case Token::CONST_FLOAT: token->text.assign("0");
{ }
float val = 0; validToken = true;
if (!token->fValue(&val)) break;
{ }
// Do not mark the token as invalid. case Token::CONST_FLOAT:
// Just emit the diagnostic and reset value to 0.0. {
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW, float val = 0;
token->location, token->text); if (!token->fValue(&val))
token->text.assign("0.0"); {
} // Do not mark the token as invalid.
validToken = true; // Just emit the diagnostic and reset value to 0.0.
break; mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
} token->location, token->text);
case Token::PP_NUMBER: token->text.assign("0.0");
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER, }
token->location, token->text); validToken = true;
break; break;
case Token::PP_OTHER: }
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER, case Token::PP_NUMBER:
token->location, token->text); mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
break; token->location, token->text);
default: break;
validToken = true; case Token::PP_OTHER:
break; mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
} token->location, token->text);
} break;
} default:
validToken = true;
} // namespace pp break;
}
}
}
} // namespace pp
src/compiler/preprocessor/Preprocessor.h
// //
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_ #ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_ #define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h> #include <stddef.h>
#include "pp_utils.h" #include "pp_utils.h"
namespace pp namespace pp
{ {
class Diagnostics; class Diagnostics;
class DirectiveHandler; class DirectiveHandler;
struct PreprocessorImpl; struct PreprocessorImpl;
struct Token; struct Token;
class Preprocessor class Preprocessor
{ {
public: public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler); Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor(); ~Preprocessor();
// count: specifies the number of elements in the string and length arrays. // count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings. // string: specifies an array of pointers to strings.
// length: specifies an array of string lengths. // length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated. // If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing // If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string. // a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the // Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string // corresponding string or a value less than 0 to indicate that the string
// is null terminated. // is null terminated.
bool init(size_t count, const char* const string[], const int length[]); bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro. // Adds a pre-defined macro.
void predefineMacro(const char* name, int value); void predefineMacro(const char* name, int value);
// Sets maximum allowed token length.
void lex(Token* token); // If token length exceeds this limit,
// the token text will be truncated to the given maximum length, and
private: // TOKEN_TOO_LONG diagnostic will be generated.
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor); // The maximum length defaults to 256.
void setMaxTokenLength(size_t maxLength);
PreprocessorImpl* mImpl;
}; void lex(Token* token);
} // namespace pp private:
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_ PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
src/compiler/preprocessor/Tokenizer.cpp
...@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner) ...@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
namespace pp { namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by Tokenizer::Tokenizer(Diagnostics* diagnostics)
// the preprocessor client, i.e., the compiler. : mHandle(0),
const size_t Tokenizer::kMaxTokenLength = 256; mMaxTokenLength(256)
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
{ {
mContext.diagnostics = diagnostics; mContext.diagnostics = diagnostics;
} }
...@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line) ...@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line)
void Tokenizer::lex(Token* token) void Tokenizer::lex(Token* token)
{ {
token->type = pplex(&token->text,&token->location,mHandle); token->type = pplex(&token->text,&token->location,mHandle);
if (token->text.size() > kMaxTokenLength) if (token->text.size() > mMaxTokenLength)
{ {
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG, mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text); token->location, token->text);
token->text.erase(kMaxTokenLength); token->text.erase(mMaxTokenLength);
} }
token->flags = 0; token->flags = 0;
......
src/compiler/preprocessor/Tokenizer.h
// //
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved. // Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. // found in the LICENSE file.
// //
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_ #define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h" #include "Input.h"
#include "Lexer.h" #include "Lexer.h"
#include "pp_utils.h" #include "pp_utils.h"
namespace pp namespace pp
{ {
class Diagnostics; class Diagnostics;
class Tokenizer : public Lexer class Tokenizer : public Lexer
{ {
public: public:
struct Context struct Context
{ {
Diagnostics* diagnostics; Diagnostics* diagnostics;
Input input; Input input;
// The location where yytext points to. Token location should track // The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same // scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer. // if text is buffered up in the scanner input buffer.
Input::Location scanLoc; Input::Location scanLoc;
bool leadingSpace; bool leadingSpace;
bool lineStart; bool lineStart;
}; };
static const std::size_t kMaxTokenLength;
Tokenizer(Diagnostics* diagnostics);
Tokenizer(Diagnostics* diagnostics); ~Tokenizer();
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
bool init(size_t count, const char* const string[], const int length[]);
void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; }
void setFileNumber(int file); void setFileNumber(int file);
void setLineNumber(int line); void setLineNumber(int line);
virtual void lex(Token* token); virtual void lex(Token* token);
private: private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner(); bool initScanner();
void destroyScanner(); void destroyScanner();
void* mHandle; // Scanner handle. void* mHandle; // Scanner handle.
Context mContext; // Scanner extra. Context mContext; // Scanner extra.
}; size_t mMaxTokenLength;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ } // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment