Skip to content

A
angle
Commit ba2cfe7c by Alok Priyadarshi
Options

Moved the definition of max-token length from preprocessor to compiler.

R=kbr@chromium.org Review URL: https://codereview.appspot.com/12957043
parent 76985f3c
Showing with 294 additions and 279 deletions
src/compiler/glslang.l
......@@ -321,6 +321,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor.
if (!context->preprocessor.init(count, string, length))
return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang.y
......@@ -51,6 +51,7 @@ WHICH GENERATES THE GLSL ES PARSER (glslang_tab.cpp AND glslang_tab.h).
%code requires {
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
}
%union {
......
src/compiler/glslang_lex.cpp
......@@ -2930,6 +2930,7 @@ int glslang_scan(size_t count, const char* const string[], const int length[],
// Initialize preprocessor.
if (!context->preprocessor.init(count, string, length))
return 1;
context->preprocessor.setMaxTokenLength(SH_MAX_TOKEN_LENGTH);
// Define extension macros.
const TExtensionBehavior& extBehavior = context->extensionBehavior();
......
src/compiler/glslang_tab.cpp
......@@ -126,6 +126,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......@@ -713,27 +714,27 @@ static const yytype_int16 yyrhs[] =
/* YYRLINE[YYN] -- source line where rule number YYN was defined. */
static const yytype_uint16 yyrline[] =
{
0, 179, 179, 180, 183, 226, 229, 242, 247, 252,
258, 261, 264, 267, 362, 372, 385, 393, 493, 496,
504, 507, 513, 517, 524, 530, 539, 547, 602, 612,
615, 625, 635, 656, 657, 658, 663, 664, 672, 683,
684, 692, 703, 707, 708, 718, 728, 738, 751, 752,
762, 775, 779, 783, 787, 788, 801, 802, 815, 816,
829, 830, 847, 848, 861, 862, 863, 864, 865, 869,
872, 883, 891, 918, 923, 937, 992, 995, 1002, 1010,
1031, 1052, 1062, 1090, 1095, 1105, 1110, 1120, 1123, 1126,
1129, 1135, 1142, 1145, 1167, 1185, 1209, 1232, 1236, 1254,
1262, 1294, 1314, 1335, 1344, 1367, 1370, 1376, 1384, 1392,
1400, 1410, 1417, 1420, 1423, 1429, 1432, 1447, 1451, 1455,
1459, 1463, 1468, 1473, 1478, 1483, 1488, 1493, 1498, 1503,
1508, 1513, 1518, 1523, 1527, 1531, 1539, 1547, 1551, 1564,
1564, 1578, 1578, 1587, 1590, 1606, 1639, 1643, 1649, 1656,
1671, 1675, 1679, 1680, 1686, 1687, 1688, 1689, 1690, 1694,
1695, 1695, 1695, 1705, 1706, 1710, 1710, 1711, 1711, 1716,
1719, 1729, 1732, 1738, 1739, 1743, 1751, 1755, 1765, 1770,
1787, 1787, 1792, 1792, 1799, 1799, 1807, 1810, 1816, 1819,
1825, 1829, 1836, 1843, 1850, 1857, 1868, 1877, 1881, 1888,
1891, 1897, 1897
0, 180, 180, 181, 184, 227, 230, 243, 248, 253,
259, 262, 265, 268, 363, 373, 386, 394, 494, 497,
505, 508, 514, 518, 525, 531, 540, 548, 603, 613,
616, 626, 636, 657, 658, 659, 664, 665, 673, 684,
685, 693, 704, 708, 709, 719, 729, 739, 752, 753,
763, 776, 780, 784, 788, 789, 802, 803, 816, 817,
830, 831, 848, 849, 862, 863, 864, 865, 866, 870,
873, 884, 892, 919, 924, 938, 993, 996, 1003, 1011,
1032, 1053, 1063, 1091, 1096, 1106, 1111, 1121, 1124, 1127,
1130, 1136, 1143, 1146, 1168, 1186, 1210, 1233, 1237, 1255,
1263, 1295, 1315, 1336, 1345, 1368, 1371, 1377, 1385, 1393,
1401, 1411, 1418, 1421, 1424, 1430, 1433, 1448, 1452, 1456,
1460, 1464, 1469, 1474, 1479, 1484, 1489, 1494, 1499, 1504,
1509, 1514, 1519, 1524, 1528, 1532, 1540, 1548, 1552, 1565,
1565, 1579, 1579, 1588, 1591, 1607, 1640, 1644, 1650, 1657,
1672, 1676, 1680, 1681, 1687, 1688, 1689, 1690, 1691, 1695,
1696, 1696, 1696, 1706, 1707, 1711, 1711, 1712, 1712, 1717,
1720, 1730, 1733, 1739, 1740, 1744, 1752, 1756, 1766, 1771,
1788, 1788, 1793, 1793, 1800, 1800, 1808, 1811, 1817, 1820,
1826, 1830, 1837, 1844, 1851, 1858, 1869, 1878, 1882, 1889,
1892, 1898, 1898
};
#endif
......
src/compiler/glslang_tab.h
......@@ -44,6 +44,7 @@ extern int yydebug;
#define YYLTYPE TSourceLoc
#define YYLTYPE_IS_DECLARED 1
#define SH_MAX_TOKEN_LENGTH 256 // WebGL spec.
......
src/compiler/preprocessor/Preprocessor.cpp
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Preprocessor.h"
#include <cassert>
#include <sstream>
#include "DiagnosticsBase.h"
#include "DirectiveParser.h"
#include "Macro.h"
#include "MacroExpander.h"
#include "Token.h"
#include "Tokenizer.h"
namespace pp
{
struct PreprocessorImpl
{
Diagnostics* diagnostics;
MacroSet macroSet;
Tokenizer tokenizer;
DirectiveParser directiveParser;
MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) :
diagnostics(diag),
tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag)
{
}
};
Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler)
{
mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
}
Preprocessor::~Preprocessor()
{
delete mImpl;
}
bool Preprocessor::init(size_t count,
const char* const string[],
const int length[])
{
static const int kGLSLVersion = 100;
// Add standard pre-defined macros.
predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length);
}
void Preprocessor::predefineMacro(const char* name, int value)
{
std::ostringstream stream;
stream << value;
Token token;
token.type = Token::CONST_INT;
token.text = stream.str();
Macro macro;
macro.predefined = true;
macro.type = Macro::kTypeObj;
macro.name = name;
macro.replacements.push_back(token);
mImpl->macroSet[name] = macro;
}
void Preprocessor::lex(Token* token)
{
bool validToken = false;
while (!validToken)
{
mImpl->macroExpander.lex(token);
switch (token->type)
{
// We should not be returning internal preprocessing tokens.
// Convert preprocessing tokens to compiler tokens or report
// diagnostics.
case Token::PP_HASH:
assert(false);
break;
case Token::CONST_INT:
{
int val = 0;
if (!token->iValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
token->location, token->text);
token->text.assign("0");
}
validToken = true;
break;
}
case Token::CONST_FLOAT:
{
float val = 0;
if (!token->fValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.0.
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
token->location, token->text);
token->text.assign("0.0");
}
validToken = true;
break;
}
case Token::PP_NUMBER:
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
token->location, token->text);
break;
case Token::PP_OTHER:
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
token->location, token->text);
break;
default:
validToken = true;
break;
}
}
}
} // namespace pp
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#include "Preprocessor.h"
#include <cassert>
#include <sstream>
#include "DiagnosticsBase.h"
#include "DirectiveParser.h"
#include "Macro.h"
#include "MacroExpander.h"
#include "Token.h"
#include "Tokenizer.h"
namespace pp
{
struct PreprocessorImpl
{
Diagnostics* diagnostics;
MacroSet macroSet;
Tokenizer tokenizer;
DirectiveParser directiveParser;
MacroExpander macroExpander;
PreprocessorImpl(Diagnostics* diag,
DirectiveHandler* directiveHandler) :
diagnostics(diag),
tokenizer(diag),
directiveParser(&tokenizer, &macroSet, diag, directiveHandler),
macroExpander(&directiveParser, &macroSet, diag)
{
}
};
Preprocessor::Preprocessor(Diagnostics* diagnostics,
DirectiveHandler* directiveHandler)
{
mImpl = new PreprocessorImpl(diagnostics, directiveHandler);
}
Preprocessor::~Preprocessor()
{
delete mImpl;
}
bool Preprocessor::init(size_t count,
const char* const string[],
const int length[])
{
static const int kGLSLVersion = 100;
// Add standard pre-defined macros.
predefineMacro("__LINE__", 0);
predefineMacro("__FILE__", 0);
predefineMacro("__VERSION__", kGLSLVersion);
predefineMacro("GL_ES", 1);
return mImpl->tokenizer.init(count, string, length);
}
void Preprocessor::predefineMacro(const char* name, int value)
{
std::ostringstream stream;
stream << value;
Token token;
token.type = Token::CONST_INT;
token.text = stream.str();
Macro macro;
macro.predefined = true;
macro.type = Macro::kTypeObj;
macro.name = name;
macro.replacements.push_back(token);
mImpl->macroSet[name] = macro;
}
void Preprocessor::setMaxTokenLength(size_t maxLength)
{
mImpl->tokenizer.setMaxTokenLength(maxLength);
}
void Preprocessor::lex(Token* token)
{
bool validToken = false;
while (!validToken)
{
mImpl->macroExpander.lex(token);
switch (token->type)
{
// We should not be returning internal preprocessing tokens.
// Convert preprocessing tokens to compiler tokens or report
// diagnostics.
case Token::PP_HASH:
assert(false);
break;
case Token::CONST_INT:
{
int val = 0;
if (!token->iValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.
mImpl->diagnostics->report(Diagnostics::INTEGER_OVERFLOW,
token->location, token->text);
token->text.assign("0");
}
validToken = true;
break;
}
case Token::CONST_FLOAT:
{
float val = 0;
if (!token->fValue(&val))
{
// Do not mark the token as invalid.
// Just emit the diagnostic and reset value to 0.0.
mImpl->diagnostics->report(Diagnostics::FLOAT_OVERFLOW,
token->location, token->text);
token->text.assign("0.0");
}
validToken = true;
break;
}
case Token::PP_NUMBER:
mImpl->diagnostics->report(Diagnostics::INVALID_NUMBER,
token->location, token->text);
break;
case Token::PP_OTHER:
mImpl->diagnostics->report(Diagnostics::INVALID_CHARACTER,
token->location, token->text);
break;
default:
validToken = true;
break;
}
}
}
} // namespace pp
src/compiler/preprocessor/Preprocessor.h
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h>
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class DirectiveHandler;
struct PreprocessorImpl;
struct Token;
class Preprocessor
{
public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor();
// count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings.
// length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string
// is null terminated.
bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro.
void predefineMacro(const char* name, int value);
void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
//
// Copyright (c) 2011 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#define COMPILER_PREPROCESSOR_PREPROCESSOR_H_
#include <stddef.h>
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class DirectiveHandler;
struct PreprocessorImpl;
struct Token;
class Preprocessor
{
public:
Preprocessor(Diagnostics* diagnostics, DirectiveHandler* directiveHandler);
~Preprocessor();
// count: specifies the number of elements in the string and length arrays.
// string: specifies an array of pointers to strings.
// length: specifies an array of string lengths.
// If length is NULL, each string is assumed to be null terminated.
// If length is a value other than NULL, it points to an array containing
// a string length for each of the corresponding elements of string.
// Each element in the length array may contain the length of the
// corresponding string or a value less than 0 to indicate that the string
// is null terminated.
bool init(size_t count, const char* const string[], const int length[]);
// Adds a pre-defined macro.
void predefineMacro(const char* name, int value);
// Sets maximum allowed token length.
// If token length exceeds this limit,
// the token text will be truncated to the given maximum length, and
// TOKEN_TOO_LONG diagnostic will be generated.
// The maximum length defaults to 256.
void setMaxTokenLength(size_t maxLength);
void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Preprocessor);
PreprocessorImpl* mImpl;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_PREPROCESSOR_H_
src/compiler/preprocessor/Tokenizer.cpp
......@@ -2290,11 +2290,9 @@ void ppfree (void * ptr , yyscan_t yyscanner)
namespace pp {
// TODO(alokp): Maximum token length should ideally be specified by
// the preprocessor client, i.e., the compiler.
const size_t Tokenizer::kMaxTokenLength = 256;
Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0)
Tokenizer::Tokenizer(Diagnostics* diagnostics)
: mHandle(0),
mMaxTokenLength(256)
{
mContext.diagnostics = diagnostics;
}
......@@ -2327,11 +2325,11 @@ void Tokenizer::setLineNumber(int line)
void Tokenizer::lex(Token* token)
{
token->type = pplex(&token->text,&token->location,mHandle);
if (token->text.size() > kMaxTokenLength)
if (token->text.size() > mMaxTokenLength)
{
mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG,
token->location, token->text);
token->text.erase(kMaxTokenLength);
token->text.erase(mMaxTokenLength);
}
token->flags = 0;
......
src/compiler/preprocessor/Tokenizer.h
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h"
#include "Lexer.h"
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class Tokenizer : public Lexer
{
public:
struct Context
{
Diagnostics* diagnostics;
Input input;
// The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer.
Input::Location scanLoc;
bool leadingSpace;
bool lineStart;
};
static const std::size_t kMaxTokenLength;
Tokenizer(Diagnostics* diagnostics);
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
void setFileNumber(int file);
void setLineNumber(int line);
virtual void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner();
void destroyScanner();
void* mHandle; // Scanner handle.
Context mContext; // Scanner extra.
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
//
// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_
#define COMPILER_PREPROCESSOR_TOKENIZER_H_
#include "Input.h"
#include "Lexer.h"
#include "pp_utils.h"
namespace pp
{
class Diagnostics;
class Tokenizer : public Lexer
{
public:
struct Context
{
Diagnostics* diagnostics;
Input input;
// The location where yytext points to. Token location should track
// scanLoc instead of Input::mReadLoc because they may not be the same
// if text is buffered up in the scanner input buffer.
Input::Location scanLoc;
bool leadingSpace;
bool lineStart;
};
Tokenizer(Diagnostics* diagnostics);
~Tokenizer();
bool init(size_t count, const char* const string[], const int length[]);
void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; }
void setFileNumber(int file);
void setLineNumber(int line);
virtual void lex(Token* token);
private:
PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer);
bool initScanner();
void destroyScanner();
void* mHandle; // Scanner handle.
Context mContext; // Scanner extra.
size_t mMaxTokenLength;
};
} // namespace pp
#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment